HONEYCOMB-207 : Configurable modules list for distributions

Export list of modules for built distribution on compile time according
to distribution.modules property to ***module-config.txt
Load aggregated set of modules on start from all descriptors in /modules
folder

Change-Id: Icdeb23536aee3a243a221d3f2ec5f340d387764e
Signed-off-by: Jan Srnicek <jsrnicek@cisco.com>

7 files changed, 330 insertions, 26 deletions

diff --git a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/ActiveModuleProvider.java b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/ActiveModuleProvider.java
new file mode 100644
index 000000000..03c56969d
--- /dev/null
+++ b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/ActiveModuleProvider.java
@@ -0,0 +1,154 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.honeycomb.infra.distro;
+
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Module;
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import javax.annotation.Nonnull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Provides list of active modules for distribution
+ */
+public class ActiveModuleProvider {
+
+    public static final String STANDARD_MODULES_RELATIVE_PATH = "../modules/";
+    private static final Logger LOG = LoggerFactory.getLogger(ActiveModuleProvider.class);
+
+    /**
+     * Provide unique set of active modules filtered from provided resources
+     */
+    public static Set<Module> loadActiveModules(@Nonnull final List<String> moduleNames) {
+        final ClassLoader classLoader = ActiveModuleProvider.class.getClassLoader();
+        LOG.info("Reading active modules configuration for distribution");
+
+        // process resources to resource modules
+        return moduleNames.stream()
+                .map(String::trim)
+                .filter(trimmedLine -> trimmedLine.length() != 0)
+                // filter out commented lines
+                .filter(nonEmptyLine -> !nonEmptyLine.startsWith("//"))
+                // filter duplicates
+                .distinct()
+                .map(validLine -> nameToClass(validLine, classLoader))
+                // filters out classes that are not modules
+                .filter(ActiveModuleProvider::filterNonModules)
+                .map(ActiveModuleProvider::classToInstance)
+                .collect(Collectors.toSet());
+    }
+
+    /**
+     * Aggregate all resources from provided relative path into a {@code List<String>}
+     */
+    public static List<String> aggregateResources(final String relativePath, final ClassLoader classLoader) {
+        try {
+            return Collections.list(classLoader.getResources(relativePath)).stream()
+                    .map(ActiveModuleProvider::toURI)
+                    .flatMap(ActiveModuleProvider::folderToFile)
+                    .map(File::toURI)
+                    .map(Paths::get)
+                    // readAll lines and add them to iteration
+                    .flatMap(ActiveModuleProvider::readLines)
+                    .collect(Collectors.toList());
+        } catch (IOException e) {
+            LOG.error("Unable to load resources from relative path {}", relativePath, e);
+            throw new IllegalStateException("Unable to load resources from relative path " + relativePath, e);
+        }
+    }
+
+    private static Stream<File> folderToFile(final URI uri) {
+        final File[] files = new File(uri).listFiles(File::isFile);
+
+        return files != null
+                ? ImmutableList.copyOf(files).stream()
+                : Collections.<File>emptyList().stream();
+    }
+
+    private static boolean filterNonModules(final Class<?> clazz) {
+        final boolean isModule = Module.class.isAssignableFrom(clazz);
+        if (!isModule) {
+            LOG.warn("Class {} is provided in modules configuration, but is not a Module and will be ignored", clazz);
+        }
+        return isModule;
+    }
+
+    /**
+     * Read lines from {@code Path}
+     */
+    private static Stream<String> readLines(final Path path) {
+        try {
+            return Files.readAllLines(path).stream();
+        } catch (IOException e) {
+            LOG.error("Unable to read content of {}", path, e);
+            throw new IllegalStateException("Unable to read content of " + path, e);
+        }
+    }
+
+    /**
+     * Converts {@code URL} to {@code URI}
+     */
+    private static URI toURI(final URL url) {
+        try {
+            return url.toURI();
+        } catch (URISyntaxException e) {
+            LOG.error("Unable to convert {} to uri", url);
+            throw new IllegalStateException("Unable to convert " + url + " to uri", e);
+        }
+    }
+
+    /**
+     * Loads class by provided name
+     */
+    private static Class<?> nameToClass(final String name,
+                                        final ClassLoader classLoader) {
+        try {
+            LOG.info("Loading module class {}", name);
+            return classLoader.loadClass(name);
+        } catch (ClassNotFoundException e) {
+            LOG.error("Unable to convert {} to class, make sure you've provided sources to classpath", name);
+            throw new IllegalStateException(
+                    "Unable to convert " + name + " to class, make sure you've provided sources to classpath", e);
+        }
+    }
+
+    /**
+     * Creates instance of module class
+     */
+    private static Module classToInstance(final Class<?> moduleClass) {
+        try {
+            LOG.info("Creating instance for module {}", moduleClass);
+            return Module.class.cast(moduleClass.newInstance());
+        } catch (InstantiationException | IllegalAccessException e) {
+            LOG.error("Unable to create instance for class {}", moduleClass, e);
+            throw new IllegalStateException("Unable to create instance for class" + moduleClass, e);
+        }
+    }
+}
diff --git a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/Main.java b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/Main.java
index d3a562d3a..3c62382c2 100644
--- a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/Main.java
+++ b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/Main.java
@@ -16,8 +16,11 @@
 
 package io.fd.honeycomb.infra.distro;
 
+import static io.fd.honeycomb.infra.distro.ActiveModuleProvider.STANDARD_MODULES_RELATIVE_PATH;
+import static io.fd.honeycomb.infra.distro.ActiveModuleProvider.aggregateResources;
+import static io.fd.honeycomb.infra.distro.ActiveModuleProvider.loadActiveModules;
+
 import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
 import com.google.inject.ConfigurationException;
 import com.google.inject.CreationException;
 import com.google.inject.Guice;
@@ -28,20 +31,14 @@ import com.google.inject.ProvisionException;
 import com.google.inject.name.Names;
 import io.fd.honeycomb.data.init.DataTreeInitializer;
 import io.fd.honeycomb.data.init.InitializerRegistry;
-import io.fd.honeycomb.infra.distro.cfgattrs.CfgAttrsModule;
 import io.fd.honeycomb.infra.distro.cfgattrs.HoneycombConfiguration;
-import io.fd.honeycomb.infra.distro.data.ConfigAndOperationalPipelineModule;
-import io.fd.honeycomb.infra.distro.data.context.ContextPipelineModule;
 import io.fd.honeycomb.infra.distro.initializer.InitializerPipelineModule;
 import io.fd.honeycomb.infra.distro.netconf.HoneycombNotification2NetconfProvider;
 import io.fd.honeycomb.infra.distro.netconf.NetconfModule;
-import io.fd.honeycomb.infra.distro.netconf.NetconfReadersModule;
 import io.fd.honeycomb.infra.distro.netconf.NetconfSshServerProvider;
 import io.fd.honeycomb.infra.distro.netconf.NetconfTcpServerProvider;
 import io.fd.honeycomb.infra.distro.restconf.RestconfModule;
-import io.fd.honeycomb.infra.distro.schema.SchemaModule;
-import io.fd.honeycomb.infra.distro.schema.YangBindingProviderModule;
-import java.util.List;
+import java.util.Set;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
 import org.opendaylight.netconf.mapping.api.NetconfOperationServiceFactory;
@@ -53,29 +50,18 @@ public final class Main {
 
     private static final Logger LOG = LoggerFactory.getLogger(Main.class);
 
-    public static final List<Module> BASE_MODULES = ImmutableList.of(
-            // Infra
-            new YangBindingProviderModule(),
-            new SchemaModule(),
-            new ConfigAndOperationalPipelineModule(),
-            new ContextPipelineModule(),
-            new InitializerPipelineModule(),
-            new NetconfModule(),
-            new NetconfReadersModule(),
-            new RestconfModule(),
-            // Json config attributes
-            new CfgAttrsModule());
-
-    private Main() {}
+    private Main() {
+    }
 
     public static void main(String[] args) {
-        init(BASE_MODULES);
+        final ClassLoader classLoader = Main.class.getClassLoader();
+        init(loadActiveModules(aggregateResources(STANDARD_MODULES_RELATIVE_PATH, classLoader)));
     }
 
     /**
-     * Initialize the Honeycomb infrastructure + all wired plugins.
+     * Initialize the Honeycomb with provided modules
      */
-    public static Injector init(final List<? extends Module> modules) {
+    public static Injector init(final Set<? extends Module> modules) {
         try {
             LOG.info("Starting honeycomb");
             Injector injector = Guice.createInjector(modules);
diff --git a/infra/minimal-distribution/src/test/java/io/fd/honeycomb/infra/distro/ActiveModuleProviderTest.java b/infra/minimal-distribution/src/test/java/io/fd/honeycomb/infra/distro/ActiveModuleProviderTest.java
new file mode 100644
index 000000000..bdadc5bd8
--- /dev/null
+++ b/infra/minimal-distribution/src/test/java/io/fd/honeycomb/infra/distro/ActiveModuleProviderTest.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.honeycomb.infra.distro;
+
+
+import static com.google.common.collect.ImmutableList.of;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.empty;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.hasItems;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.isA;
+import static org.hamcrest.core.Is.is;
+
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Module;
+import java.util.List;
+import java.util.Set;
+import org.junit.Test;
+
+public class ActiveModuleProviderTest {
+
+    @Test
+    public void testLoadActiveModulesSuccessfull() {
+        final ImmutableList rawResources = of(
+                "// this should be skipped",
+                "// io.fd.honeycomb.infra.distro.Modules$ChildModule1",
+                "               io.fd.honeycomb.infra.distro.Modules$ChildModule2",
+                "io.fd.honeycomb.infra.distro.Modules$ChildModule3     ",
+                "io.fd.honeycomb.infra.distro.Modules$ChildModule3",
+                "io.fd.honeycomb.infra.distro.Modules$NonModule"
+        );
+
+        final Set<Module> activeModules = ActiveModuleProvider.loadActiveModules(rawResources);
+
+        // first and second line should be ignored due to comment
+        // second,third,and fourth are valid,but should reduce module count to 2,because of duplicity
+        // last one does is not ancestor of Module, so it should be ignored/skipped
+        assertThat(activeModules, hasSize(2));
+        //hasItems or containsInAnyOrder does not have/is deprecated in variant with matcher
+        assertThat(activeModules, hasItem(isA(io.fd.honeycomb.infra.distro.Modules.ChildModule2.class)));
+        assertThat(activeModules, hasItem(isA(io.fd.honeycomb.infra.distro.Modules.ChildModule3.class)));
+    }
+
+    @Test(expected = IllegalStateException.class)
+    public void testLoadActiveModulesFailed() {
+        final ImmutableList rawResources = of(
+                "// this should be skipped",
+                "// io.fd.honeycomb.infra.distro.Modules$ChildModule1",
+                "               io.fd.honeycomb.infra.distro.Modules$ChildModule2",
+                "### io.fd.honeycomb.infra.distro.Modules$ChildModule3     ",// it should fail because of this
+                "io.fd.honeycomb.infra.distro.Modules$ChildModule3",
+                "io.fd.honeycomb.infra.distro.Modules$NonModule"
+        );
+
+        ActiveModuleProvider.loadActiveModules(rawResources);
+    }
+
+    @Test
+    public void testAggregateResourcesNonEmpty() {
+        final List<String> aggregatedResources =
+                ActiveModuleProvider.aggregateResources("./modules", this.getClass().getClassLoader());
+        assertThat(aggregatedResources, hasSize(5));
+        assertThat(aggregatedResources, hasItems("        Non-commented non-trimmed",
+                "//Commented",
+                "// Commented non-trimmed",
+                "Not commented",
+                "// Line from second file"));
+    }
+
+    @Test
+    public void testAggregateResourcesEmpty() {
+        assertThat(ActiveModuleProvider.aggregateResources("/non-existing-folder", this.getClass().getClassLoader()),
+                is(empty()));
+    }
+
+}
diff --git a/infra/minimal-distribution/src/test/java/io/fd/honeycomb/infra/distro/BaseMinimalDistributionTest.java b/infra/minimal-distribution/src/test/java/io/fd/honeycomb/infra/distro/BaseMinimalDistributionTest.java
index 61042868f..8a8ddc3ef 100644
--- a/infra/minimal-distribution/src/test/java/io/fd/honeycomb/infra/distro/BaseMinimalDistributionTest.java
+++ b/infra/minimal-distribution/src/test/java/io/fd/honeycomb/infra/distro/BaseMinimalDistributionTest.java
@@ -16,22 +16,35 @@
 
 package io.fd.honeycomb.infra.distro;
 
+import static com.google.common.collect.ImmutableSet.of;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 
 import com.google.common.base.Charsets;
 import com.google.common.io.ByteStreams;
+import com.google.inject.Module;
 import com.jcraft.jsch.Channel;
 import com.jcraft.jsch.ChannelSubsystem;
 import com.jcraft.jsch.JSch;
 import com.jcraft.jsch.Session;
 import com.mashape.unirest.http.HttpResponse;
 import com.mashape.unirest.http.Unirest;
+import io.fd.honeycomb.infra.distro.cfgattrs.CfgAttrsModule;
+import io.fd.honeycomb.infra.distro.data.ConfigAndOperationalPipelineModule;
+import io.fd.honeycomb.infra.distro.data.context.ContextPipelineModule;
+import io.fd.honeycomb.infra.distro.initializer.InitializerPipelineModule;
+import io.fd.honeycomb.infra.distro.netconf.NetconfModule;
+import io.fd.honeycomb.infra.distro.netconf.NetconfReadersModule;
+import io.fd.honeycomb.infra.distro.restconf.RestconfModule;
+import io.fd.honeycomb.infra.distro.schema.SchemaModule;
+import io.fd.honeycomb.infra.distro.schema.YangBindingProviderModule;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.Socket;
+import java.util.List;
 import java.util.Properties;
+import java.util.Set;
 import javax.net.ssl.SSLContext;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
@@ -57,6 +70,17 @@ public class BaseMinimalDistributionTest {
     private static final String NETCONF_NAMESPACE = "urn:ietf:params:xml:ns:netconf:base:1.0";
     private static final int NETCONF_HELLO_WAIT = 2500;
 
+    public static final Set<Module> BASE_MODULES = of(
+            new YangBindingProviderModule(),
+            new SchemaModule(),
+            new ConfigAndOperationalPipelineModule(),
+            new ContextPipelineModule(),
+            new InitializerPipelineModule(),
+            new NetconfModule(),
+            new NetconfReadersModule(),
+            new RestconfModule(),
+            new CfgAttrsModule());
+
     @Before
     public void setUp() throws Exception {
         SSLContext sslcontext = SSLContexts.custom()
@@ -76,7 +100,7 @@ public class BaseMinimalDistributionTest {
      */
     @Test(timeout = 60000)
     public void test() throws Exception {
-        Main.init(Main.BASE_MODULES);
+        Main.init(BASE_MODULES);
 
         LOG.info("Testing Honeycomb base distribution");
         LOG.info("Testing NETCONF TCP");
diff --git a/infra/minimal-distribution/src/test/java/io/fd/honeycomb/infra/distro/Modules.java b/infra/minimal-distribution/src/test/java/io/fd/honeycomb/infra/distro/Modules.java
new file mode 100644
index 000000000..e30fb87ca
--- /dev/null
+++ b/infra/minimal-distribution/src/test/java/io/fd/honeycomb/infra/distro/Modules.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.honeycomb.infra.distro;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Binder;
+import com.google.inject.Module;
+
+public class Modules {
+
+    public static class ChildModule1 implements Module {
+        @Override
+        public void configure(final Binder binder) {
+        }
+    }
+
+    public static class ChildModule2 implements Module {
+        @Override
+        public void configure(final Binder binder) {
+        }
+    }
+
+    public static class ChildModule3 extends AbstractModule {
+        @Override
+        protected void configure() {
+        }
+    }
+
+    public static class NonModule{}
+}
diff --git a/infra/minimal-distribution/src/test/resources/modules/module-config-one.txt b/infra/minimal-distribution/src/test/resources/modules/module-config-one.txt
new file mode 100644
index 000000000..8d48a3c9d
--- /dev/null
+++ b/infra/minimal-distribution/src/test/resources/modules/module-config-one.txt
@@ -0,0 +1,4 @@
+//Commented
+// Commented non-trimmed
+Not commented
+        Non-commented non-trimmed
\ No newline at end of file
diff --git a/infra/minimal-distribution/src/test/resources/modules/module-config-two.txt b/infra/minimal-distribution/src/test/resources/modules/module-config-two.txt
new file mode 100644
index 000000000..ef11829e9
--- /dev/null
+++ b/infra/minimal-distribution/src/test/resources/modules/module-config-two.txt
@@ -0,0 +1 @@
+// Line from second file
\ No newline at end of file