HONEYCOMB-358 - Generate guice to yang modules index

Generates two descriptor files
yang-modules-binding/yang-modules - List of Yang modules by project(classpath + deps)
yang-mapping/FULL_PROJECT_NAME-yang-modules-index - Index from Guice modules to Yang modules
that are used by respective Guice module

These files are included in jar files and distribution resources

Change-Id: Iafc178219245df9129fb426a5876215c6fd1837e
Signed-off-by: Jan Srnicek <jsrnicek@cisco.com>

6 files changed, 380 insertions, 17 deletions

diff --git a/common/common-scripts/asciidoc/Readme.adoc b/common/common-scripts/asciidoc/Readme.adoc
new file mode 100644
index 000000000..d8d5f8e3e
--- /dev/null
+++ b/common/common-scripts/asciidoc/Readme.adoc
@@ -0,0 +1,3 @@
+= common-scripts
+
+Overview of common-scripts
\ No newline at end of file
diff --git a/common/common-scripts/pom.xml b/common/common-scripts/pom.xml
index e41364154..7be29b662 100644
--- a/common/common-scripts/pom.xml
+++ b/common/common-scripts/pom.xml
@@ -31,6 +31,8 @@
         <groovy.version>2.4.7</groovy.version>
         <groovy.eclipse.compiler.version>2.9.2-01</groovy.eclipse.compiler.version>
         <groovy.eclipse.batch.version>2.4.3-01</groovy.eclipse.batch.version>
+        <commons-io.version>2.5</commons-io.version>
+        <yang-binding.version>0.9.3-Boron-SR3</yang-binding.version>
     </properties>
 
     <build>
@@ -82,6 +84,16 @@
             <artifactId>groovy-all</artifactId>
             <version>${groovy.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.opendaylight.mdsal</groupId>
+            <artifactId>yang-binding</artifactId>
+            <version>${yang-binding.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+            <version>${commons-io.version}</version>
+        </dependency>
     </dependencies>
 
 </project>
\ No newline at end of file
diff --git a/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModuleYangIndexGenerator.groovy b/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModuleYangIndexGenerator.groovy
new file mode 100644
index 000000000..21877c51c
--- /dev/null
+++ b/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModuleYangIndexGenerator.groovy
@@ -0,0 +1,262 @@
+/*
+ * Copyright (c) 2017 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.honeycomb.common.scripts
+
+import com.google.common.base.Strings
+import com.google.common.io.Files
+import org.apache.commons.io.FileUtils
+import org.opendaylight.yangtools.yang.binding.YangModelBindingProvider
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Paths
+import java.util.jar.JarFile
+import java.util.stream.Collectors
+
+/**
+ * Provides logic to generate:
+ * <li><b>generateIndexForPresentModules()</b> - yang-modules-binding/yang-modules -
+ * List of Yang modules used by project(classpath + deps)</li>
+ * <li><b>pairDistributionModulesWithYangModules()</b> -  yang-mapping/FULL_PROJECT_NAME-yang-modules-index -
+ * Index from Guice modules to Yang modules that are used by respective Guice module</li>
+ * <br>
+ * These files can be then included in jars and distribution resources to allow
+ * conditional yang module loading according to list of Guice modules that are started by distribution
+ * */
+class ModuleYangIndexGenerator {
+
+    private static final String YANG_MODEL_PROVIDER_NAME = YangModelBindingProvider.class.getName()
+    private static
+    final YANG_PROVIDERS_PATH = "META-INF/services/" + YANG_MODEL_PROVIDER_NAME
+    private static final MODULES_DELIMITER = ","
+    private static final CLASS_EXT = "class"
+    private static final String[] EXTENSIONS = [CLASS_EXT]
+    private static final YANG_MODULES_FOLDER = "yang-modules-binding"
+    private static final YANG_MODULES_FILE_NAME = "yang-modules"
+    private static final YANG_MAPPING_FOLDER = "yang-mapping"
+    private static final YANG_MODULES_INDEX_FILE_NAME = "yang-modules-index"
+
+    public static void generateIndexForPresentModules(project, log) {
+        log.info "Checking module providers for project ${project.getName()}"
+        // checks module provides from dependencies
+        // folder with extracted libs
+        def libsFolder = Paths.get(project.getBuild().getDirectory(), "lib")
+        if (!libsFolder.toFile().exists()) {
+            // Plugin for collecting dependencies is executed from parent project,
+            // therefore it will run also for parent, that does not have any depedencies(just dep management)
+            // so lib folder wont be created
+            log.info "Folder ${libsFolder} does not exist - No dependencies to process"
+            return;
+        }
+
+        String yangModules = java.nio.file.Files.walk(libsFolder)
+                .map { path -> path.toFile() }
+                .filter { file -> file.isFile() }
+                .filter { file -> file.getName().endsWith(".jar") }
+                .map { file -> getModuleProviderContentFromApiJar(new JarFile(file), log) }
+                .filter { content -> !Strings.isNullOrEmpty(content.trim()) }
+                .collect().join(MODULES_DELIMITER)
+        log.info "Yang yangModules found : $yangModules"
+        def outputDir = Paths.get(project.getBuild().getOutputDirectory(), YANG_MODULES_FOLDER).toFile()
+        outputDir.mkdirs()
+        def outputFile = Paths.get(outputDir.getPath(), YANG_MODULES_FILE_NAME).toFile()
+        outputFile.createNewFile()
+        Files.write(yangModules, outputFile, StandardCharsets.UTF_8)
+        log.info "Yang yangModules configuration successfully written to ${outputFile.getPath()}"
+    }
+
+    /**
+     * Loads module list of current distribution, and attempts
+     * to pair them with yang module providers from either their classpath or direct/indirect dependencies.
+     * */
+    public static void pairDistributionModulesWithYangModules(project, log) {
+        def modules = modulesList(project)
+        if (modules.isEmpty()) {
+            log.warn "No distribution modules defined, skipping"
+            return
+        }
+
+        log.info "Pairing distribution modules ${modules} to yang modules"
+        def moduleToYangModulesIndex = new HashMap<String, String>()
+        def outputDir = project.getBuild().getOutputDirectory()
+
+        // TODO - HONEYCOMB-373 - eliminate local matching after distribution modules are moved to separate project
+        // first ,matches modules against local files, helps to filter local classpath modules, to reduce scope
+        // of dependency scanning that is more performance heavy
+        log.info "Pairing against local classpath"
+        pairAgainsLocalFiles(outputDir, modules, moduleToYangModulesIndex, log)
+
+        // go to dependencies only if some modules are left. this occurs for modules
+        // started by distribution that are not part of its classpath(basically all plugin modules)
+        if (!modules.isEmpty()) {
+            log.info "Pairing against dependencies"
+            // The rest of the modules is looked up in dependencies
+            pairAgainsDependencyArtifacts(project, modules, log, moduleToYangModulesIndex)
+        }
+
+        // for ex.: /target/honeycomb-minimal-resources/yang-mapping
+        def yangMappingFolder = Paths.get(project.getBuild().getOutputDirectory(), StartupScriptGenerator.MINIMAL_RESOURCES_FOLDER, YANG_MAPPING_FOLDER).toFile()
+
+        if (!yangMappingFolder.exists()) {
+            yangMappingFolder.mkdir()
+        }
+        def outputFileName = "${ModulesListGenerator.pathFriendlyProjectName(project.artifact)}_$YANG_MODULES_INDEX_FILE_NAME"
+
+        def outputFile = Paths.get(yangMappingFolder.getPath(), outputFileName).toFile()
+        outputFile.createNewFile()
+
+        def indexFileContent = moduleToYangModulesIndex.entrySet()
+                .stream()
+                .map { entry -> "GUICE_MODULE:${entry.getKey()}|YANG_MODULES:${entry.getValue()}${System.lineSeparator()}" }
+                .collect(Collectors.joining())
+
+        Files.write(indexFileContent, outputFile, StandardCharsets.UTF_8)
+        if (!modules.isEmpty()) {
+            log.warn "No yang configuration found for modules ${modules}"
+        }
+        log.info "Distribution to yang modules index successfully generated to $outputFile"
+
+    }
+
+    // provides list of modules for distribution, not from property, but already processed list from /modules folder.
+    // this allows us to skip all validation that is present in modules list generation, and just take final list of modules
+    private static Set<String> modulesList(project) {
+        def modulesFolder = ModulesListGenerator.modulesConfigFolder(project).toFile()
+        Arrays.stream(modulesFolder.listFiles())
+        // picks up only file for currently processed distribution
+                .filter { file -> file.getName().contains(ModulesListGenerator.pathFriendlyProjectName(project.artifact)) }
+                .map { file -> FileUtils.readLines(file, StandardCharsets.UTF_8) }
+                .flatMap { lines -> lines.stream() }
+                .map { line -> line.replace("//", "") }
+                .map { line -> line.trim() }
+                .collect(Collectors.toSet())
+    }
+
+    private static void pairAgainsDependencyArtifacts(project, modules, log, index) {
+        // loads jar file
+        def artifacts = project.getDependencyArtifacts()
+        log.info "Artifacts used for pairing $artifacts"
+        artifacts.stream()
+                .map { artifact -> artifact.getFile() }
+                .map { file -> new JarFile(file) }
+                .forEach { jar ->
+            // first tries to find content of yang module provides file,
+            // if not found, skip's this jar
+            def moduleProvidersContent = getModuleProviderContentFromImplJar(jar, log)
+            if (Strings.isNullOrEmpty(moduleProvidersContent.trim())) {
+                log.debug "No yang module configuration found in ${jar.getName()}"
+                return
+            }
+
+            def entryNames = Collections.list(jar.entries()).stream()
+                    .map { entry -> entry.getName() }
+                    .filter { name -> name.endsWith(CLASS_EXT) }
+                    .map { name -> pathToClassName(name) }
+                    .collect(Collectors.toSet())
+
+            log.info "Entries $entryNames"
+            log.info "Modules $modules"
+            for (String module : modules) {
+                if (entryNames.contains(module)) {
+                    log.info "Module $module found in artifact ${jar.getName()}"
+                    index.put(module, moduleProvidersContent)
+                }
+            }
+        }
+        modules.removeAll(index.keySet());
+        log.info "Modules left after dependency pairing $modules"
+    }
+
+    // TODO - HONEYCOMB-373 - eliminate local matching
+    private static void pairAgainsLocalFiles(outputDir, modules, HashMap<String, String> index, log) {
+        // Pairs modules that are part of distribution classpath
+        def yangModulesLocalConfig = Paths.get(outputDir, YANG_MODULES_FOLDER, YANG_MODULES_FILE_NAME).toFile()
+        if (!yangModulesLocalConfig.exists()) {
+            log.debug "Local configuration for yang modules does not exist, skiping local matching"
+            return
+        }
+
+        log.info "Local file ${yangModulesLocalConfig}"
+        def localYangModules = fixDelimiters(FileUtils.readFileToString(yangModulesLocalConfig, StandardCharsets.UTF_8))
+
+        log.info "Output dir $outputDir"
+        FileUtils.listFiles(Paths.get(outputDir).toFile(), EXTENSIONS, true)
+                .stream()
+                .map { file -> file.getPath() }
+                .map { path -> relativizePath(path, outputDir) }
+                .forEach { path ->
+            for (String module : modules) {
+                if (path.equals(classNameToPath(module))) {
+                    log.info "Module $module found in local classpath"
+                    // mapping by standard class name
+                    index.put(module, localYangModules)
+                }
+            }
+        }
+
+        // remove all matching modules to reduce scope of search
+        modules.removeAll(index.keySet());
+        log.info "Modules left after local classpath pairing $modules"
+    }
+
+    private static String relativizePath(String path, String outputDir) {
+        return path.replace(outputDir, "").substring(1).trim();
+    }
+
+    private static String pathToClassName(String path) {
+        return path.replace("/", ".").replace(".class", "").trim()
+    }
+
+    private static String classNameToPath(String className) {
+        return className.replace(".", "/").concat(".class").trim()
+    }
+
+    private static String getModuleProviderContentFromImplJar(JarFile jarFile, log) {
+        def moduleProviderEntry = jarFile.getJarEntry(YANG_MODULES_FOLDER + "/" + YANG_MODULES_FILE_NAME)
+        if (moduleProviderEntry == null) {
+            return "";
+        }
+        // module provider files are in general a couple of lines, so should'nt be a problem
+        // to read at once
+        InputStream input = jarFile.getInputStream(moduleProviderEntry)
+        byte[] data = new byte[(int) moduleProviderEntry.getSize()]
+        input.read(data)
+        input.close()
+
+        return fixDelimiters(new String(data, StandardCharsets.UTF_8));
+    }
+
+    private static String getModuleProviderContentFromApiJar(JarFile jarFile, log) {
+        def moduleProviderEntry = jarFile.getJarEntry(YANG_PROVIDERS_PATH)
+        if (moduleProviderEntry == null) {
+            return "";
+        }
+        // module provider files are in general a couple of lines, so should'nt be a problem
+        // to read at once
+        InputStream input = jarFile.getInputStream(moduleProviderEntry)
+        byte[] data = new byte[(int) moduleProviderEntry.getSize()]
+        input.read(data)
+        input.close()
+
+        return fixDelimiters(new String(data, StandardCharsets.UTF_8))
+    }
+
+    private static String fixDelimiters(String data) {
+        return Arrays.stream(data.split(System.lineSeparator()))
+                .map { line -> line.trim() }
+                .collect().join(MODULES_DELIMITER)
+    }
+}
diff --git a/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModulesListGenerator.groovy b/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModulesListGenerator.groovy
index c7a74d20e..525a77e66 100644
--- a/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModulesListGenerator.groovy
+++ b/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModulesListGenerator.groovy
@@ -18,6 +18,7 @@ package io.fd.honeycomb.common.scripts
 
 import groovy.text.SimpleTemplateEngine
 
+import java.nio.file.Path
 import java.nio.file.Paths
 
 /**
@@ -37,7 +38,7 @@ class ModulesListGenerator {
         // builds project name from group,artifact and version to prevent overwriting
         // while building multiple distribution project
         def artifact = project.artifact
-        def projectName = "${artifact.getGroupId()}_${artifact.getArtifactId()}_${artifact.getVersion()}".replace(".","-")
+        def projectName = pathFriendlyProjectName(artifact)
 
         log.info "Generating list of modules started by distribution ${projectName}"
 
@@ -48,7 +49,7 @@ class ModulesListGenerator {
         log.info "Project ${projectName} : Found modules ${activeModules}"
         //creates folder modules
 
-        def outputPath = Paths.get(project.build.outputDirectory, StartupScriptGenerator.MINIMAL_RESOURCES_FOLDER, MODULES_FOLDER)
+        def outputPath = modulesConfigFolder(project)
         //creates module folder
         outputPath.toFile().mkdirs()
 
@@ -66,4 +67,12 @@ class ModulesListGenerator {
             outputFile.text = activeModules.join(System.lineSeparator)
         }
     }
+
+    public static Path modulesConfigFolder(project) {
+        return Paths.get(project.build.outputDirectory, StartupScriptGenerator.MINIMAL_RESOURCES_FOLDER, MODULES_FOLDER)
+    }
+
+    public static String pathFriendlyProjectName(artifact) {
+        return "${artifact.getGroupId()}_${artifact.getArtifactId()}_${artifact.getVersion()}".replace(".", "-")
+    }
 }
diff --git a/common/impl-parent/pom.xml b/common/impl-parent/pom.xml
index 7d35518d7..65fb573c3 100644
--- a/common/impl-parent/pom.xml
+++ b/common/impl-parent/pom.xml
@@ -33,6 +33,7 @@
     <guice.version>4.1.0</guice.version>
     <guice.config.version>1.2.0</guice.config.version>
     <skinny.logback.version>1.0.8</skinny.logback.version>
+    <maven-resources-plugin.version>3.0.2</maven-resources-plugin.version>
   </properties>
 
   <dependencyManagement>
@@ -68,4 +69,81 @@
       </dependency>
     </dependencies>
   </dependencyManagement>
+
+  <build>
+    <pluginManagement>
+      <!-- Must be done in parent, to unpack jars for all projects that we generate yang module index for -->
+      <plugins> <!-- Copy all dependencies -->
+        <plugin>
+          <groupId>org.apache.maven.plugins</groupId>
+          <artifactId>maven-dependency-plugin</artifactId>
+          <version>2.10</version>
+          <executions>
+            <execution>
+              <id>copy-dependencies</id>
+              <!-- Must be done before generating yang to module index -->
+              <phase>process-sources</phase>
+              <goals>
+                <goal>copy-dependencies</goal>
+              </goals>
+              <configuration>
+                <outputDirectory>${project.build.directory}/lib</outputDirectory>
+                <useBaseVersion>true</useBaseVersion>
+                <useRepositoryLayout>true</useRepositoryLayout>
+                <excludeArtifactIds>yang-jmx-generator,test-api</excludeArtifactIds>
+              </configuration>
+            </execution>
+          </executions>
+        </plugin>
+        <!-- Generate module to yang provider index -->
+        <plugin>
+          <groupId>org.codehaus.gmaven</groupId>
+          <artifactId>groovy-maven-plugin</artifactId>
+          <executions>
+            <execution>
+              <id>generate-yang-index</id>
+              <phase>generate-resources</phase>
+              <goals>
+                <goal>execute</goal>
+              </goals>
+              <configuration>
+                <source>
+                  io.fd.honeycomb.common.scripts.ModuleYangIndexGenerator.generateIndexForPresentModules(project, log)
+                </source>
+              </configuration>
+            </execution>
+          </executions>
+          <dependencies>
+            <dependency>
+              <groupId>io.fd.honeycomb.common</groupId>
+              <artifactId>common-scripts</artifactId>
+              <version>${project.version}</version>
+            </dependency>
+          </dependencies>
+        </plugin>
+      </plugins>
+    </pluginManagement>
+
+    <plugins>
+      <plugin>
+        <groupId>org.codehaus.gmaven</groupId>
+        <artifactId>groovy-maven-plugin</artifactId>
+      </plugin>
+    </plugins>
+
+    <resources>
+      <resource>
+        <directory>src/main/resources</directory>
+        <includes>
+          <include>**/*</include>
+        </includes>
+      </resource>
+      <resource>
+        <directory>${project.build.outputDirectory}</directory>
+        <includes>
+          <include>**/yang-modules-binding/yang-modules</include>
+        </includes>
+      </resource>
+    </resources>
+  </build>
 </project>
diff --git a/common/minimal-distribution-parent/pom.xml b/common/minimal-distribution-parent/pom.xml
index f191fda52..617509b96 100644
--- a/common/minimal-distribution-parent/pom.xml
+++ b/common/minimal-distribution-parent/pom.xml
@@ -111,31 +111,18 @@
                                 <classpathMavenRepositoryLayout>true</classpathMavenRepositoryLayout>
                             </manifest>
                             <manifestEntries>
-                                <Class-Path>config/ cert/ modules/</Class-Path>
+                                <Class-Path>config/ cert/ modules/ yang-mapping/</Class-Path>
                             </manifestEntries>
                         </archive>
                     </configuration>
                 </plugin>
 
-                <!-- Copy all dependencies -->
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-dependency-plugin</artifactId>
                     <version>2.10</version>
                     <executions>
-                        <execution>
-                            <id>copy-dependencies</id>
-                            <phase>package</phase>
-                            <goals>
-                                <goal>copy-dependencies</goal>
-                            </goals>
-                            <configuration>
-                                <outputDirectory>${project.build.directory}/lib</outputDirectory>
-                                <useBaseVersion>true</useBaseVersion>
-                                <useRepositoryLayout>true</useRepositoryLayout>
-                                <excludeArtifactIds>yang-jmx-generator</excludeArtifactIds>
-                            </configuration>
-                        </execution>
+                        <!-- Dependencies are copied by parent project -->
                         <execution>
                             <id>unpack-configuration</id>
                             <phase>prepare-package</phase>
@@ -182,6 +169,18 @@
                                 </source>
                             </configuration>
                         </execution>
+                        <execution>
+                            <id>generate-module-to-yang-index</id>
+                            <phase>prepare-package</phase>
+                            <goals>
+                                <goal>execute</goal>
+                            </goals>
+                            <configuration>
+                                <source>
+                                    io.fd.honeycomb.common.scripts.ModuleYangIndexGenerator.pairDistributionModulesWithYangModules(project, log)
+                                </source>
+                            </configuration>
+                        </execution>
                     </executions>
                     <dependencies>
                         <dependency>