HONEYCOMB-358 - Conditional module loading

Change-Id: Ic9b7182cc77bf2f73cf5edd3ee19f25f53711cda
Signed-off-by: Jan Srnicek <jsrnicek@cisco.com>

9 files changed, 320 insertions, 63 deletions

diff --git a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/Main.java b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/Main.java
index 265740bcc..582fcf687 100644
--- a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/Main.java
+++ b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/Main.java
@@ -22,7 +22,6 @@ import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableSet;
 import com.google.inject.ConfigurationException;
 import com.google.inject.CreationException;
-import com.google.inject.Guice;
 import com.google.inject.Injector;
 import com.google.inject.Key;
 import com.google.inject.Module;
diff --git a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/activation/ActivationConfig.java b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/activation/ActivationConfig.java
index 8777c32a2..a17f5d632 100644
--- a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/activation/ActivationConfig.java
+++ b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/activation/ActivationConfig.java
@@ -27,7 +27,15 @@ public class ActivationConfig {
     @InjectConfig("modules-resource-path")
     private String modulesResourcePath;
 
+    @InjectConfig("yang-modules-index-path")
+    private String yangModulesIndexPath;
+
     public String getModulesResourcePath() {
         return Optional.ofNullable(modulesResourcePath).orElse("../modules/");
     }
+
+    public String getYangModulesIndexPath() {
+        return Optional.ofNullable(yangModulesIndexPath).orElse("../yang-mapping/");
+    }
+
 }
diff --git a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/activation/ActiveModuleProvider.java b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/activation/ActiveModuleProvider.java
index 21d108082..d31024ce9 100644
--- a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/activation/ActiveModuleProvider.java
+++ b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/activation/ActiveModuleProvider.java
@@ -139,7 +139,7 @@ class ActiveModuleProvider implements Provider<ActiveModules> {
     private static Class<? extends Module> moduleNameToClass(final String name,
                                                              final ClassLoader classLoader) {
         try {
-            LOG.info("Loading module class {}", name);
+            LOG.debug("Loading module class {}", name);
             return (Class<? extends Module>) classLoader.loadClass(name);
         } catch (ClassNotFoundException e) {
             LOG.error("Unable to convert {} to class, make sure you've provided sources to classpath", name);
diff --git a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/ModuleInfoBackedCtxProvider.java b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/ModuleInfoBackedCtxProvider.java
index f82cdd0b1..e2d0fbfaa 100644
--- a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/ModuleInfoBackedCtxProvider.java
+++ b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/ModuleInfoBackedCtxProvider.java
@@ -16,11 +16,11 @@
 
 package io.fd.honeycomb.infra.distro.schema;
 
+import static io.fd.honeycomb.infra.distro.schema.YangModulesProvider.YangModules;
+
 import com.google.common.base.MoreObjects;
 import com.google.inject.Inject;
 import io.fd.honeycomb.infra.distro.ProviderTrait;
-import java.util.HashSet;
-import java.util.Set;
 import java.util.stream.Collectors;
 import org.opendaylight.yangtools.sal.binding.generator.impl.ModuleInfoBackedContext;
 import org.opendaylight.yangtools.yang.binding.YangModelBindingProvider;
@@ -30,13 +30,14 @@ import org.slf4j.LoggerFactory;
 public final class ModuleInfoBackedCtxProvider extends ProviderTrait<ModuleInfoBackedContext> {
     private static final Logger LOG = LoggerFactory.getLogger(ModuleInfoBackedCtxProvider.class);
 
-    @Inject(optional = true)
-    private Set<YangModelBindingProvider> moduleInfos = new HashSet<>();
+    // optional in sense that list of modules inside can be empty if none was found
+    @Inject
+    private YangModules moduleInfos;
 
     @Override
     protected ModuleInfoBackedContext create() {
         ModuleInfoBackedContext create = ModuleInfoBackedContext.create();
-        create.addModuleInfos(moduleInfos.stream()
+        create.addModuleInfos(moduleInfos.getYangBindings().stream()
                 .map(YangModelBindingProvider::getModuleInfo)
                 .collect(Collectors.toList()));
         LOG.debug("ModuleInfoBackedContext created from {}", moduleInfos);
diff --git a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/ResourceLoader.java b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/ResourceLoader.java
new file mode 100644
index 000000000..c851dab1b
--- /dev/null
+++ b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/ResourceLoader.java
@@ -0,0 +1,135 @@
+/*
+ * Copyright (c) 2017 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.honeycomb.infra.distro.schema;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.lang.String.format;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Strings;
+import com.google.common.io.Resources;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Set;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.stream.Collectors;
+import org.apache.commons.io.IOUtils;
+
+interface ResourceLoader {
+
+    default Set<String> loadResourceContentsOnPath(final String path) {
+        final URL folderUrl = getClass().getClassLoader().getResource(path);
+        checkNotNull(folderUrl, "Resources %s not found", path);
+
+        if (ResourceLoaderIml.urlToUri(folderUrl).getScheme().equals("jar")) {
+            return ResourceLoaderIml.readFromJar(path, folderUrl);
+        } else {
+            return ResourceLoaderIml.readFromFolder(folderUrl);
+        }
+
+    }
+
+    final class ResourceLoaderIml {
+
+        private static Set<String> readFromFolder(final URL folderUrl) {
+            final File folder = new File(folderUrl.getPath());
+            final File[] files = checkNotNull(folder.listFiles(), "No files present on path %s", folderUrl);
+            return Arrays.stream(files)
+                    .map(ResourceLoaderIml::fileToUrl)
+                    .map(ResourceLoaderIml::urlToContentString)
+                    .flatMap(content -> Arrays.stream(content.split(System.lineSeparator())))
+                    .filter(ResourceLoaderIml::filterNonEmpty)
+                    .collect(Collectors.toSet());
+        }
+
+        private static Set<String> readFromJar(final String path, final URL url) {
+            final String uriString = urlToUri(url).toString();
+            final String fileReference = extractJarFilePath(uriString);
+            try (JarFile jar = new JarFile(new File(fileReference))) {
+                return Collections.list(jar.entries())
+                        .stream()
+                        .filter(jarEntry -> jarEntry.getName().contains(path))
+                        .map(jarEntry -> getJarEntryStream(jar, jarEntry))
+                        .map(ResourceLoaderIml::readJarEntryStream)
+                        .flatMap(content -> Arrays.stream(content.split(System.lineSeparator())))
+                        .filter(ResourceLoaderIml::filterNonEmpty)
+                        .collect(Collectors.toSet());
+            } catch (IOException e) {
+                throw new IllegalStateException(e);
+            }
+        }
+
+        private static String extractJarFilePath(final String uriString) {
+            return uriString.substring(0, uriString.indexOf("!")).replace("jar:file:", "");
+        }
+
+        private static boolean filterNonEmpty(final String line) {
+            return !Strings.isNullOrEmpty(line.trim());
+        }
+
+        private static String readJarEntryStream(final InputStream inputStream) {
+            try {
+                final String value = IOUtils.toString(inputStream, StandardCharsets.UTF_8);
+                IOUtils.closeQuietly(inputStream);
+                return value;
+            } catch (IOException e) {
+                throw new IllegalStateException(e);
+            }
+        }
+
+        private static InputStream getJarEntryStream(final JarFile jar, final JarEntry jarEntry) {
+            try {
+                return jar.getInputStream(jarEntry);
+            } catch (IOException e) {
+                throw new IllegalStateException(format("Unable to get stream for entry %s | jar %s", jar, jarEntry));
+            }
+        }
+
+        private static URI urlToUri(final URL url) {
+            try {
+                return url.toURI();
+            } catch (URISyntaxException e) {
+                throw new IllegalStateException(format("Unable to convert URL %s to URI", url));
+            }
+        }
+
+        private static String urlToContentString(final URL url) {
+            try {
+                return Resources.toString(url, Charsets.UTF_8);
+            } catch (IOException e) {
+                throw new IllegalArgumentException("Unable to read resource from: " + url, e);
+            }
+        }
+
+        private static URL fileToUrl(final File file) {
+            try {
+                return file.toURI().toURL();
+            } catch (MalformedURLException e) {
+                throw new IllegalStateException(e);
+            }
+        }
+    }
+}
diff --git a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/YangBindingProviderModule.java b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/YangBindingProviderModule.java
index 8e402808b..d705226bf 100644
--- a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/YangBindingProviderModule.java
+++ b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/YangBindingProviderModule.java
@@ -16,68 +16,15 @@
 
 package io.fd.honeycomb.infra.distro.schema;
 
-import com.google.common.base.Charsets;
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-import com.google.common.io.Resources;
 import com.google.inject.AbstractModule;
-import com.google.inject.Singleton;
-import com.google.inject.multibindings.Multibinder;
-import java.io.IOException;
-import java.net.URL;
-import java.util.Collections;
-import java.util.List;
-import javax.annotation.Nonnull;
-import org.opendaylight.yangtools.yang.binding.YangModelBindingProvider;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-/**
- * Load all YangModelBindingProvider classes from classpath.
- * <p/>
- * Relying on /META-INF/services/ metadata.
- */
 public class YangBindingProviderModule extends AbstractModule {
     private static final Logger LOG = LoggerFactory.getLogger(YangBindingProviderModule.class);
 
-    private static final String YANG_BA_PROVIDER_PATH = "META-INF/services/" + YangModelBindingProvider.class.getName();
-
     protected void configure() {
-        final Multibinder<YangModelBindingProvider> binder =
-                Multibinder.newSetBinder(binder(), YangModelBindingProvider.class);
-        final List<URL> resources;
-        try {
-            resources = Collections.list(getClass().getClassLoader().getResources(YANG_BA_PROVIDER_PATH));
-        } catch (IOException e) {
-            throw new IllegalStateException("Unable to load binding providers from path: " + YANG_BA_PROVIDER_PATH, e);
-        }
-        LOG.debug("ModuleProviders found at {}", resources);
-        resources.stream()
-                .map(YangBindingProviderModule::urlToString)
-                .flatMap(content -> Lists.newArrayList(content.split("\n")).stream())
-                .filter(line -> !Strings.isNullOrEmpty(line.trim()))
-                .distinct()
-                .map(YangBindingProviderModule::loadClass)
-                .forEach(providerClass -> {
-                    LOG.debug("ModuleProvider found for {}", providerClass);
-                    binder.addBinding().to((Class<? extends YangModelBindingProvider>) providerClass)
-                            .in(Singleton.class);
-                });
-    }
-
-    private static Class<?> loadClass(@Nonnull final String className) {
-        try {
-            return Class.forName(className);
-        } catch (ClassNotFoundException e) {
-            throw new IllegalArgumentException("Unable to load class: " + className, e);
-        }
-    }
-
-    private static String urlToString(@Nonnull final URL url) {
-        try {
-            return Resources.toString(url, Charsets.UTF_8);
-        } catch (IOException e) {
-            throw new IllegalArgumentException("Unable to read resource from: " + url, e);
-        }
+        LOG.info("Configuring YangBindingProviderModule");
+        bind(YangModulesProvider.YangModules.class).toProvider(YangModulesProvider.class).asEagerSingleton();
     }
 }
diff --git a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/YangModuleMappingIndex.java b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/YangModuleMappingIndex.java
new file mode 100644
index 000000000..a483cfd9c
--- /dev/null
+++ b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/YangModuleMappingIndex.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.honeycomb.infra.distro.schema;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.LinkedListMultimap;
+import com.google.common.collect.Multimap;
+import java.util.Arrays;
+import java.util.Set;
+import java.util.stream.Stream;
+import javax.annotation.Nonnull;
+
+/**
+ * Index from guice module to yang module providers
+ */
+class YangModuleMappingIndex implements ResourceLoader {
+
+    private static final String G_MODULE_TOKEN = "GUICE_MODULE:";
+    private static final String KEY_VALUE_SEPARATOR = "|";
+    private static final String Y_MODULE_TOKEN = "YANG_MODULES:";
+    private static final String Y_MODULE_SEPARATOR = ",";
+
+    /**
+     * key - module class name
+     * value  - yang module provider
+     */
+    private final Multimap<String, String> index;
+
+    YangModuleMappingIndex(final String indexPath) {
+        this.index = LinkedListMultimap.create();
+        loadResourceContentsOnPath(indexPath)
+                .forEach(line -> {
+                    final String moduleName = parseModuleName(line);
+                    parseYangModules(line).forEach(yModuleProvider -> index.put(moduleName, yModuleProvider));
+                });
+    }
+
+    Set<String> getByModuleName(@Nonnull final String moduleName) {
+        return ImmutableSet.copyOf(index.get(moduleName));
+    }
+
+    int applicationModulesCount() {
+        return index.keySet().size();
+    }
+
+    private static String parseModuleName(final String rawLine) {
+        return rawLine.substring(rawLine.indexOf(G_MODULE_TOKEN) + G_MODULE_TOKEN.length(),
+                rawLine.indexOf(KEY_VALUE_SEPARATOR));
+    }
+
+    private static Stream<String> parseYangModules(final String rawLine) {
+        return Arrays.stream(rawLine.substring(rawLine.indexOf(Y_MODULE_TOKEN) + Y_MODULE_TOKEN.length())
+                .split(Y_MODULE_SEPARATOR));
+    }
+}
diff --git a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/YangModulesProvider.java b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/YangModulesProvider.java
new file mode 100644
index 000000000..f6d8ea0c1
--- /dev/null
+++ b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/YangModulesProvider.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.honeycomb.infra.distro.schema;
+
+
+import static java.lang.String.format;
+
+import com.google.common.base.Charsets;
+import com.google.common.io.Resources;
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import io.fd.honeycomb.infra.distro.activation.ActivationConfig;
+import io.fd.honeycomb.infra.distro.activation.ActiveModules;
+import java.io.IOException;
+import java.net.URL;
+import java.util.Collection;
+import java.util.Set;
+import java.util.stream.Collectors;
+import javax.annotation.Nonnull;
+import org.opendaylight.yangtools.yang.binding.YangModelBindingProvider;
+
+/**
+ * Loads active yang modules
+ * Relying on generate yang-module-index
+ */
+public class YangModulesProvider implements Provider<YangModulesProvider.YangModules> {
+
+    @Inject
+    private ActiveModules activeModules;
+
+    @Inject
+    private ActivationConfig config;
+
+    @Override
+    public YangModules get() {
+        // no need to bind this, pretty big map and its needed just here
+        final YangModuleMappingIndex index = new YangModuleMappingIndex(config.getYangModulesIndexPath());
+
+        return new YangModules(activeModules.getActiveModulesClasses().stream()
+                .map(Class::getName)
+                .map(index::getByModuleName)
+                .flatMap(Collection::stream)
+                .map(YangModulesProvider::loadClass)
+                .map(aClass -> (Class<? extends YangModelBindingProvider>) aClass)
+                .collect(Collectors.toSet()));
+    }
+
+    static class YangModules {
+        private final Set<Class<? extends YangModelBindingProvider>> yangBindings;
+
+        YangModules(final Set<Class<? extends YangModelBindingProvider>> yangBindings) {
+            this.yangBindings = yangBindings;
+        }
+
+        Set<YangModelBindingProvider> getYangBindings() {
+            return yangBindings.stream()
+                    .map(providerClass -> {
+                        try {
+                            return providerClass.newInstance();
+                        } catch (InstantiationException | IllegalAccessException e) {
+                            throw new IllegalStateException(format("Unable to create instance of %s", providerClass),
+                                    e);
+                        }
+                    }).collect(Collectors.toSet());
+        }
+    }
+
+    private static Class<?> loadClass(@Nonnull final String className) {
+        try {
+            return Class.forName(className);
+        } catch (ClassNotFoundException e) {
+            throw new IllegalArgumentException("Unable to load class: " + className, e);
+        }
+    }
+
+    static String urlToString(@Nonnull final URL url) {
+        try {
+            return Resources.toString(url, Charsets.UTF_8);
+        } catch (IOException e) {
+            throw new IllegalArgumentException("Unable to read resource from: " + url, e);
+        }
+    }
+}
diff --git a/infra/minimal-distribution/src/main/resources/honeycomb-minimal-resources/config/activation.json b/infra/minimal-distribution/src/main/resources/honeycomb-minimal-resources/config/activation.json
index 7b6f44a20..8dd168702 100644
--- a/infra/minimal-distribution/src/main/resources/honeycomb-minimal-resources/config/activation.json
+++ b/infra/minimal-distribution/src/main/resources/honeycomb-minimal-resources/config/activation.json
@@ -1,3 +1,4 @@
 {
-  "modules-resource-path": "../modules/"
+  "modules-resource-path": "../modules/",
+  "yang-modules-index-path": "../yang-mapping/"
 }
\ No newline at end of file