From 62dd4d32fd270d3a6b7bb47c972bbcd5dc7b9f43 Mon Sep 17 00:00:00 2001 From: Jan Srnicek Date: Fri, 30 Jun 2017 10:08:14 +0200 Subject: HONEYCOMB-358 - Generate guice to yang modules index Generates two descriptor files yang-modules-binding/yang-modules - List of Yang modules by project(classpath + deps) yang-mapping/FULL_PROJECT_NAME-yang-modules-index - Index from Guice modules to Yang modules that are used by respective Guice module These files are included in jar files and distribution resources Change-Id: Iafc178219245df9129fb426a5876215c6fd1837e Signed-off-by: Jan Srnicek --- common/common-scripts/asciidoc/Readme.adoc | 3 + common/common-scripts/pom.xml | 12 + .../common/scripts/ModuleYangIndexGenerator.groovy | 262 +++++++++++++++++++++ .../common/scripts/ModulesListGenerator.groovy | 13 +- common/impl-parent/pom.xml | 78 ++++++ common/minimal-distribution-parent/pom.xml | 29 ++- 6 files changed, 380 insertions(+), 17 deletions(-) create mode 100644 common/common-scripts/asciidoc/Readme.adoc create mode 100644 common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModuleYangIndexGenerator.groovy (limited to 'common') diff --git a/common/common-scripts/asciidoc/Readme.adoc b/common/common-scripts/asciidoc/Readme.adoc new file mode 100644 index 000000000..d8d5f8e3e --- /dev/null +++ b/common/common-scripts/asciidoc/Readme.adoc @@ -0,0 +1,3 @@ += common-scripts + +Overview of common-scripts \ No newline at end of file diff --git a/common/common-scripts/pom.xml b/common/common-scripts/pom.xml index e41364154..7be29b662 100644 --- a/common/common-scripts/pom.xml +++ b/common/common-scripts/pom.xml @@ -31,6 +31,8 @@ 2.4.7 2.9.2-01 2.4.3-01 + 2.5 + 0.9.3-Boron-SR3 @@ -82,6 +84,16 @@ groovy-all ${groovy.version} + + org.opendaylight.mdsal + yang-binding + ${yang-binding.version} + + + commons-io + commons-io + ${commons-io.version} + \ No newline at end of file diff --git a/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModuleYangIndexGenerator.groovy b/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModuleYangIndexGenerator.groovy new file mode 100644 index 000000000..21877c51c --- /dev/null +++ b/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModuleYangIndexGenerator.groovy @@ -0,0 +1,262 @@ +/* + * Copyright (c) 2017 Cisco and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fd.honeycomb.common.scripts + +import com.google.common.base.Strings +import com.google.common.io.Files +import org.apache.commons.io.FileUtils +import org.opendaylight.yangtools.yang.binding.YangModelBindingProvider + +import java.nio.charset.StandardCharsets +import java.nio.file.Paths +import java.util.jar.JarFile +import java.util.stream.Collectors + +/** + * Provides logic to generate: + *
  • generateIndexForPresentModules() - yang-modules-binding/yang-modules - + * List of Yang modules used by project(classpath + deps)
    • + *
  • pairDistributionModulesWithYangModules() - yang-mapping/FULL_PROJECT_NAME-yang-modules-index - + * Index from Guice modules to Yang modules that are used by respective Guice module
    • + *
    + * These files can be then included in jars and distribution resources to allow + * conditional yang module loading according to list of Guice modules that are started by distribution + * */ +class ModuleYangIndexGenerator { + + private static final String YANG_MODEL_PROVIDER_NAME = YangModelBindingProvider.class.getName() + private static + final YANG_PROVIDERS_PATH = "META-INF/services/" + YANG_MODEL_PROVIDER_NAME + private static final MODULES_DELIMITER = "," + private static final CLASS_EXT = "class" + private static final String[] EXTENSIONS = [CLASS_EXT] + private static final YANG_MODULES_FOLDER = "yang-modules-binding" + private static final YANG_MODULES_FILE_NAME = "yang-modules" + private static final YANG_MAPPING_FOLDER = "yang-mapping" + private static final YANG_MODULES_INDEX_FILE_NAME = "yang-modules-index" + + public static void generateIndexForPresentModules(project, log) { + log.info "Checking module providers for project ${project.getName()}" + // checks module provides from dependencies + // folder with extracted libs + def libsFolder = Paths.get(project.getBuild().getDirectory(), "lib") + if (!libsFolder.toFile().exists()) { + // Plugin for collecting dependencies is executed from parent project, + // therefore it will run also for parent, that does not have any depedencies(just dep management) + // so lib folder wont be created + log.info "Folder ${libsFolder} does not exist - No dependencies to process" + return; + } + + String yangModules = java.nio.file.Files.walk(libsFolder) + .map { path -> path.toFile() } + .filter { file -> file.isFile() } + .filter { file -> file.getName().endsWith(".jar") } + .map { file -> getModuleProviderContentFromApiJar(new JarFile(file), log) } + .filter { content -> !Strings.isNullOrEmpty(content.trim()) } + .collect().join(MODULES_DELIMITER) + log.info "Yang yangModules found : $yangModules" + def outputDir = Paths.get(project.getBuild().getOutputDirectory(), YANG_MODULES_FOLDER).toFile() + outputDir.mkdirs() + def outputFile = Paths.get(outputDir.getPath(), YANG_MODULES_FILE_NAME).toFile() + outputFile.createNewFile() + Files.write(yangModules, outputFile, StandardCharsets.UTF_8) + log.info "Yang yangModules configuration successfully written to ${outputFile.getPath()}" + } + + /** + * Loads module list of current distribution, and attempts + * to pair them with yang module providers from either their classpath or direct/indirect dependencies. + * */ + public static void pairDistributionModulesWithYangModules(project, log) { + def modules = modulesList(project) + if (modules.isEmpty()) { + log.warn "No distribution modules defined, skipping" + return + } + + log.info "Pairing distribution modules ${modules} to yang modules" + def moduleToYangModulesIndex = new HashMap() + def outputDir = project.getBuild().getOutputDirectory() + + // TODO - HONEYCOMB-373 - eliminate local matching after distribution modules are moved to separate project + // first ,matches modules against local files, helps to filter local classpath modules, to reduce scope + // of dependency scanning that is more performance heavy + log.info "Pairing against local classpath" + pairAgainsLocalFiles(outputDir, modules, moduleToYangModulesIndex, log) + + // go to dependencies only if some modules are left. this occurs for modules + // started by distribution that are not part of its classpath(basically all plugin modules) + if (!modules.isEmpty()) { + log.info "Pairing against dependencies" + // The rest of the modules is looked up in dependencies + pairAgainsDependencyArtifacts(project, modules, log, moduleToYangModulesIndex) + } + + // for ex.: /target/honeycomb-minimal-resources/yang-mapping + def yangMappingFolder = Paths.get(project.getBuild().getOutputDirectory(), StartupScriptGenerator.MINIMAL_RESOURCES_FOLDER, YANG_MAPPING_FOLDER).toFile() + + if (!yangMappingFolder.exists()) { + yangMappingFolder.mkdir() + } + def outputFileName = "${ModulesListGenerator.pathFriendlyProjectName(project.artifact)}_$YANG_MODULES_INDEX_FILE_NAME" + + def outputFile = Paths.get(yangMappingFolder.getPath(), outputFileName).toFile() + outputFile.createNewFile() + + def indexFileContent = moduleToYangModulesIndex.entrySet() + .stream() + .map { entry -> "GUICE_MODULE:${entry.getKey()}|YANG_MODULES:${entry.getValue()}${System.lineSeparator()}" } + .collect(Collectors.joining()) + + Files.write(indexFileContent, outputFile, StandardCharsets.UTF_8) + if (!modules.isEmpty()) { + log.warn "No yang configuration found for modules ${modules}" + } + log.info "Distribution to yang modules index successfully generated to $outputFile" + + } + + // provides list of modules for distribution, not from property, but already processed list from /modules folder. + // this allows us to skip all validation that is present in modules list generation, and just take final list of modules + private static Set modulesList(project) { + def modulesFolder = ModulesListGenerator.modulesConfigFolder(project).toFile() + Arrays.stream(modulesFolder.listFiles()) + // picks up only file for currently processed distribution + .filter { file -> file.getName().contains(ModulesListGenerator.pathFriendlyProjectName(project.artifact)) } + .map { file -> FileUtils.readLines(file, StandardCharsets.UTF_8) } + .flatMap { lines -> lines.stream() } + .map { line -> line.replace("//", "") } + .map { line -> line.trim() } + .collect(Collectors.toSet()) + } + + private static void pairAgainsDependencyArtifacts(project, modules, log, index) { + // loads jar file + def artifacts = project.getDependencyArtifacts() + log.info "Artifacts used for pairing $artifacts" + artifacts.stream() + .map { artifact -> artifact.getFile() } + .map { file -> new JarFile(file) } + .forEach { jar -> + // first tries to find content of yang module provides file, + // if not found, skip's this jar + def moduleProvidersContent = getModuleProviderContentFromImplJar(jar, log) + if (Strings.isNullOrEmpty(moduleProvidersContent.trim())) { + log.debug "No yang module configuration found in ${jar.getName()}" + return + } + + def entryNames = Collections.list(jar.entries()).stream() + .map { entry -> entry.getName() } + .filter { name -> name.endsWith(CLASS_EXT) } + .map { name -> pathToClassName(name) } + .collect(Collectors.toSet()) + + log.info "Entries $entryNames" + log.info "Modules $modules" + for (String module : modules) { + if (entryNames.contains(module)) { + log.info "Module $module found in artifact ${jar.getName()}" + index.put(module, moduleProvidersContent) + } + } + } + modules.removeAll(index.keySet()); + log.info "Modules left after dependency pairing $modules" + } + + // TODO - HONEYCOMB-373 - eliminate local matching + private static void pairAgainsLocalFiles(outputDir, modules, HashMap index, log) { + // Pairs modules that are part of distribution classpath + def yangModulesLocalConfig = Paths.get(outputDir, YANG_MODULES_FOLDER, YANG_MODULES_FILE_NAME).toFile() + if (!yangModulesLocalConfig.exists()) { + log.debug "Local configuration for yang modules does not exist, skiping local matching" + return + } + + log.info "Local file ${yangModulesLocalConfig}" + def localYangModules = fixDelimiters(FileUtils.readFileToString(yangModulesLocalConfig, StandardCharsets.UTF_8)) + + log.info "Output dir $outputDir" + FileUtils.listFiles(Paths.get(outputDir).toFile(), EXTENSIONS, true) + .stream() + .map { file -> file.getPath() } + .map { path -> relativizePath(path, outputDir) } + .forEach { path -> + for (String module : modules) { + if (path.equals(classNameToPath(module))) { + log.info "Module $module found in local classpath" + // mapping by standard class name + index.put(module, localYangModules) + } + } + } + + // remove all matching modules to reduce scope of search + modules.removeAll(index.keySet()); + log.info "Modules left after local classpath pairing $modules" + } + + private static String relativizePath(String path, String outputDir) { + return path.replace(outputDir, "").substring(1).trim(); + } + + private static String pathToClassName(String path) { + return path.replace("/", ".").replace(".class", "").trim() + } + + private static String classNameToPath(String className) { + return className.replace(".", "/").concat(".class").trim() + } + + private static String getModuleProviderContentFromImplJar(JarFile jarFile, log) { + def moduleProviderEntry = jarFile.getJarEntry(YANG_MODULES_FOLDER + "/" + YANG_MODULES_FILE_NAME) + if (moduleProviderEntry == null) { + return ""; + } + // module provider files are in general a couple of lines, so should'nt be a problem + // to read at once + InputStream input = jarFile.getInputStream(moduleProviderEntry) + byte[] data = new byte[(int) moduleProviderEntry.getSize()] + input.read(data) + input.close() + + return fixDelimiters(new String(data, StandardCharsets.UTF_8)); + } + + private static String getModuleProviderContentFromApiJar(JarFile jarFile, log) { + def moduleProviderEntry = jarFile.getJarEntry(YANG_PROVIDERS_PATH) + if (moduleProviderEntry == null) { + return ""; + } + // module provider files are in general a couple of lines, so should'nt be a problem + // to read at once + InputStream input = jarFile.getInputStream(moduleProviderEntry) + byte[] data = new byte[(int) moduleProviderEntry.getSize()] + input.read(data) + input.close() + + return fixDelimiters(new String(data, StandardCharsets.UTF_8)) + } + + private static String fixDelimiters(String data) { + return Arrays.stream(data.split(System.lineSeparator())) + .map { line -> line.trim() } + .collect().join(MODULES_DELIMITER) + } +} diff --git a/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModulesListGenerator.groovy b/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModulesListGenerator.groovy index c7a74d20e..525a77e66 100644 --- a/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModulesListGenerator.groovy +++ b/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModulesListGenerator.groovy @@ -18,6 +18,7 @@ package io.fd.honeycomb.common.scripts import groovy.text.SimpleTemplateEngine +import java.nio.file.Path import java.nio.file.Paths /** @@ -37,7 +38,7 @@ class ModulesListGenerator { // builds project name from group,artifact and version to prevent overwriting // while building multiple distribution project def artifact = project.artifact - def projectName = "${artifact.getGroupId()}_${artifact.getArtifactId()}_${artifact.getVersion()}".replace(".","-") + def projectName = pathFriendlyProjectName(artifact) log.info "Generating list of modules started by distribution ${projectName}" @@ -48,7 +49,7 @@ class ModulesListGenerator { log.info "Project ${projectName} : Found modules ${activeModules}" //creates folder modules - def outputPath = Paths.get(project.build.outputDirectory, StartupScriptGenerator.MINIMAL_RESOURCES_FOLDER, MODULES_FOLDER) + def outputPath = modulesConfigFolder(project) //creates module folder outputPath.toFile().mkdirs() @@ -66,4 +67,12 @@ class ModulesListGenerator { outputFile.text = activeModules.join(System.lineSeparator) } } + + public static Path modulesConfigFolder(project) { + return Paths.get(project.build.outputDirectory, StartupScriptGenerator.MINIMAL_RESOURCES_FOLDER, MODULES_FOLDER) + } + + public static String pathFriendlyProjectName(artifact) { + return "${artifact.getGroupId()}_${artifact.getArtifactId()}_${artifact.getVersion()}".replace(".", "-") + } } diff --git a/common/impl-parent/pom.xml b/common/impl-parent/pom.xml index 7d35518d7..65fb573c3 100644 --- a/common/impl-parent/pom.xml +++ b/common/impl-parent/pom.xml @@ -33,6 +33,7 @@ 4.1.0 1.2.0 1.0.8 + 3.0.2 @@ -68,4 +69,81 @@ + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 2.10 + + + copy-dependencies + + process-sources + + copy-dependencies + + + ${project.build.directory}/lib + true + true + yang-jmx-generator,test-api + + + + + + + org.codehaus.gmaven + groovy-maven-plugin + + + generate-yang-index + generate-resources + + execute + + + + io.fd.honeycomb.common.scripts.ModuleYangIndexGenerator.generateIndexForPresentModules(project, log) + + + + + + + io.fd.honeycomb.common + common-scripts + ${project.version} + + + + + + + + + org.codehaus.gmaven + groovy-maven-plugin + + + + + + src/main/resources + + **/* + + + + ${project.build.outputDirectory} + + **/yang-modules-binding/yang-modules + + + + diff --git a/common/minimal-distribution-parent/pom.xml b/common/minimal-distribution-parent/pom.xml index f191fda52..617509b96 100644 --- a/common/minimal-distribution-parent/pom.xml +++ b/common/minimal-distribution-parent/pom.xml @@ -111,31 +111,18 @@ true - config/ cert/ modules/ + config/ cert/ modules/ yang-mapping/ - org.apache.maven.plugins maven-dependency-plugin 2.10 - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/lib - true - true - yang-jmx-generator - - + unpack-configuration prepare-package @@ -182,6 +169,18 @@ + + generate-module-to-yang-index + prepare-package + + execute + + + + io.fd.honeycomb.common.scripts.ModuleYangIndexGenerator.pairDistributionModulesWithYangModules(project, log) + + + -- cgit 1.2.3-korg