diff options
Diffstat (limited to 'common')
6 files changed, 380 insertions, 17 deletions
diff --git a/common/common-scripts/asciidoc/Readme.adoc b/common/common-scripts/asciidoc/Readme.adoc new file mode 100644 index 000000000..d8d5f8e3e --- /dev/null +++ b/common/common-scripts/asciidoc/Readme.adoc @@ -0,0 +1,3 @@ += common-scripts + +Overview of common-scripts
\ No newline at end of file diff --git a/common/common-scripts/pom.xml b/common/common-scripts/pom.xml index e41364154..7be29b662 100644 --- a/common/common-scripts/pom.xml +++ b/common/common-scripts/pom.xml @@ -31,6 +31,8 @@ <groovy.version>2.4.7</groovy.version> <groovy.eclipse.compiler.version>2.9.2-01</groovy.eclipse.compiler.version> <groovy.eclipse.batch.version>2.4.3-01</groovy.eclipse.batch.version> + <commons-io.version>2.5</commons-io.version> + <yang-binding.version>0.9.3-Boron-SR3</yang-binding.version> </properties> <build> @@ -82,6 +84,16 @@ <artifactId>groovy-all</artifactId> <version>${groovy.version}</version> </dependency> + <dependency> + <groupId>org.opendaylight.mdsal</groupId> + <artifactId>yang-binding</artifactId> + <version>${yang-binding.version}</version> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + <version>${commons-io.version}</version> + </dependency> </dependencies> </project>
\ No newline at end of file diff --git a/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModuleYangIndexGenerator.groovy b/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModuleYangIndexGenerator.groovy new file mode 100644 index 000000000..21877c51c --- /dev/null +++ b/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModuleYangIndexGenerator.groovy @@ -0,0 +1,262 @@ +/* + * Copyright (c) 2017 Cisco and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fd.honeycomb.common.scripts + +import com.google.common.base.Strings +import com.google.common.io.Files +import org.apache.commons.io.FileUtils +import org.opendaylight.yangtools.yang.binding.YangModelBindingProvider + +import java.nio.charset.StandardCharsets +import java.nio.file.Paths +import java.util.jar.JarFile +import java.util.stream.Collectors + +/** + * Provides logic to generate: + * <li><b>generateIndexForPresentModules()</b> - yang-modules-binding/yang-modules - + * List of Yang modules used by project(classpath + deps)</li> + * <li><b>pairDistributionModulesWithYangModules()</b> - yang-mapping/FULL_PROJECT_NAME-yang-modules-index - + * Index from Guice modules to Yang modules that are used by respective Guice module</li> + * <br> + * These files can be then included in jars and distribution resources to allow + * conditional yang module loading according to list of Guice modules that are started by distribution + * */ +class ModuleYangIndexGenerator { + + private static final String YANG_MODEL_PROVIDER_NAME = YangModelBindingProvider.class.getName() + private static + final YANG_PROVIDERS_PATH = "META-INF/services/" + YANG_MODEL_PROVIDER_NAME + private static final MODULES_DELIMITER = "," + private static final CLASS_EXT = "class" + private static final String[] EXTENSIONS = [CLASS_EXT] + private static final YANG_MODULES_FOLDER = "yang-modules-binding" + private static final YANG_MODULES_FILE_NAME = "yang-modules" + private static final YANG_MAPPING_FOLDER = "yang-mapping" + private static final YANG_MODULES_INDEX_FILE_NAME = "yang-modules-index" + + public static void generateIndexForPresentModules(project, log) { + log.info "Checking module providers for project ${project.getName()}" + // checks module provides from dependencies + // folder with extracted libs + def libsFolder = Paths.get(project.getBuild().getDirectory(), "lib") + if (!libsFolder.toFile().exists()) { + // Plugin for collecting dependencies is executed from parent project, + // therefore it will run also for parent, that does not have any depedencies(just dep management) + // so lib folder wont be created + log.info "Folder ${libsFolder} does not exist - No dependencies to process" + return; + } + + String yangModules = java.nio.file.Files.walk(libsFolder) + .map { path -> path.toFile() } + .filter { file -> file.isFile() } + .filter { file -> file.getName().endsWith(".jar") } + .map { file -> getModuleProviderContentFromApiJar(new JarFile(file), log) } + .filter { content -> !Strings.isNullOrEmpty(content.trim()) } + .collect().join(MODULES_DELIMITER) + log.info "Yang yangModules found : $yangModules" + def outputDir = Paths.get(project.getBuild().getOutputDirectory(), YANG_MODULES_FOLDER).toFile() + outputDir.mkdirs() + def outputFile = Paths.get(outputDir.getPath(), YANG_MODULES_FILE_NAME).toFile() + outputFile.createNewFile() + Files.write(yangModules, outputFile, StandardCharsets.UTF_8) + log.info "Yang yangModules configuration successfully written to ${outputFile.getPath()}" + } + + /** + * Loads module list of current distribution, and attempts + * to pair them with yang module providers from either their classpath or direct/indirect dependencies. + * */ + public static void pairDistributionModulesWithYangModules(project, log) { + def modules = modulesList(project) + if (modules.isEmpty()) { + log.warn "No distribution modules defined, skipping" + return + } + + log.info "Pairing distribution modules ${modules} to yang modules" + def moduleToYangModulesIndex = new HashMap<String, String>() + def outputDir = project.getBuild().getOutputDirectory() + + // TODO - HONEYCOMB-373 - eliminate local matching after distribution modules are moved to separate project + // first ,matches modules against local files, helps to filter local classpath modules, to reduce scope + // of dependency scanning that is more performance heavy + log.info "Pairing against local classpath" + pairAgainsLocalFiles(outputDir, modules, moduleToYangModulesIndex, log) + + // go to dependencies only if some modules are left. this occurs for modules + // started by distribution that are not part of its classpath(basically all plugin modules) + if (!modules.isEmpty()) { + log.info "Pairing against dependencies" + // The rest of the modules is looked up in dependencies + pairAgainsDependencyArtifacts(project, modules, log, moduleToYangModulesIndex) + } + + // for ex.: /target/honeycomb-minimal-resources/yang-mapping + def yangMappingFolder = Paths.get(project.getBuild().getOutputDirectory(), StartupScriptGenerator.MINIMAL_RESOURCES_FOLDER, YANG_MAPPING_FOLDER).toFile() + + if (!yangMappingFolder.exists()) { + yangMappingFolder.mkdir() + } + def outputFileName = "${ModulesListGenerator.pathFriendlyProjectName(project.artifact)}_$YANG_MODULES_INDEX_FILE_NAME" + + def outputFile = Paths.get(yangMappingFolder.getPath(), outputFileName).toFile() + outputFile.createNewFile() + + def indexFileContent = moduleToYangModulesIndex.entrySet() + .stream() + .map { entry -> "GUICE_MODULE:${entry.getKey()}|YANG_MODULES:${entry.getValue()}${System.lineSeparator()}" } + .collect(Collectors.joining()) + + Files.write(indexFileContent, outputFile, StandardCharsets.UTF_8) + if (!modules.isEmpty()) { + log.warn "No yang configuration found for modules ${modules}" + } + log.info "Distribution to yang modules index successfully generated to $outputFile" + + } + + // provides list of modules for distribution, not from property, but already processed list from /modules folder. + // this allows us to skip all validation that is present in modules list generation, and just take final list of modules + private static Set<String> modulesList(project) { + def modulesFolder = ModulesListGenerator.modulesConfigFolder(project).toFile() + Arrays.stream(modulesFolder.listFiles()) + // picks up only file for currently processed distribution + .filter { file -> file.getName().contains(ModulesListGenerator.pathFriendlyProjectName(project.artifact)) } + .map { file -> FileUtils.readLines(file, StandardCharsets.UTF_8) } + .flatMap { lines -> lines.stream() } + .map { line -> line.replace("//", "") } + .map { line -> line.trim() } + .collect(Collectors.toSet()) + } + + private static void pairAgainsDependencyArtifacts(project, modules, log, index) { + // loads jar file + def artifacts = project.getDependencyArtifacts() + log.info "Artifacts used for pairing $artifacts" + artifacts.stream() + .map { artifact -> artifact.getFile() } + .map { file -> new JarFile(file) } + .forEach { jar -> + // first tries to find content of yang module provides file, + // if not found, skip's this jar + def moduleProvidersContent = getModuleProviderContentFromImplJar(jar, log) + if (Strings.isNullOrEmpty(moduleProvidersContent.trim())) { + log.debug "No yang module configuration found in ${jar.getName()}" + return + } + + def entryNames = Collections.list(jar.entries()).stream() + .map { entry -> entry.getName() } + .filter { name -> name.endsWith(CLASS_EXT) } + .map { name -> pathToClassName(name) } + .collect(Collectors.toSet()) + + log.info "Entries $entryNames" + log.info "Modules $modules" + for (String module : modules) { + if (entryNames.contains(module)) { + log.info "Module $module found in artifact ${jar.getName()}" + index.put(module, moduleProvidersContent) + } + } + } + modules.removeAll(index.keySet()); + log.info "Modules left after dependency pairing $modules" + } + + // TODO - HONEYCOMB-373 - eliminate local matching + private static void pairAgainsLocalFiles(outputDir, modules, HashMap<String, String> index, log) { + // Pairs modules that are part of distribution classpath + def yangModulesLocalConfig = Paths.get(outputDir, YANG_MODULES_FOLDER, YANG_MODULES_FILE_NAME).toFile() + if (!yangModulesLocalConfig.exists()) { + log.debug "Local configuration for yang modules does not exist, skiping local matching" + return + } + + log.info "Local file ${yangModulesLocalConfig}" + def localYangModules = fixDelimiters(FileUtils.readFileToString(yangModulesLocalConfig, StandardCharsets.UTF_8)) + + log.info "Output dir $outputDir" + FileUtils.listFiles(Paths.get(outputDir).toFile(), EXTENSIONS, true) + .stream() + .map { file -> file.getPath() } + .map { path -> relativizePath(path, outputDir) } + .forEach { path -> + for (String module : modules) { + if (path.equals(classNameToPath(module))) { + log.info "Module $module found in local classpath" + // mapping by standard class name + index.put(module, localYangModules) + } + } + } + + // remove all matching modules to reduce scope of search + modules.removeAll(index.keySet()); + log.info "Modules left after local classpath pairing $modules" + } + + private static String relativizePath(String path, String outputDir) { + return path.replace(outputDir, "").substring(1).trim(); + } + + private static String pathToClassName(String path) { + return path.replace("/", ".").replace(".class", "").trim() + } + + private static String classNameToPath(String className) { + return className.replace(".", "/").concat(".class").trim() + } + + private static String getModuleProviderContentFromImplJar(JarFile jarFile, log) { + def moduleProviderEntry = jarFile.getJarEntry(YANG_MODULES_FOLDER + "/" + YANG_MODULES_FILE_NAME) + if (moduleProviderEntry == null) { + return ""; + } + // module provider files are in general a couple of lines, so should'nt be a problem + // to read at once + InputStream input = jarFile.getInputStream(moduleProviderEntry) + byte[] data = new byte[(int) moduleProviderEntry.getSize()] + input.read(data) + input.close() + + return fixDelimiters(new String(data, StandardCharsets.UTF_8)); + } + + private static String getModuleProviderContentFromApiJar(JarFile jarFile, log) { + def moduleProviderEntry = jarFile.getJarEntry(YANG_PROVIDERS_PATH) + if (moduleProviderEntry == null) { + return ""; + } + // module provider files are in general a couple of lines, so should'nt be a problem + // to read at once + InputStream input = jarFile.getInputStream(moduleProviderEntry) + byte[] data = new byte[(int) moduleProviderEntry.getSize()] + input.read(data) + input.close() + + return fixDelimiters(new String(data, StandardCharsets.UTF_8)) + } + + private static String fixDelimiters(String data) { + return Arrays.stream(data.split(System.lineSeparator())) + .map { line -> line.trim() } + .collect().join(MODULES_DELIMITER) + } +} diff --git a/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModulesListGenerator.groovy b/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModulesListGenerator.groovy index c7a74d20e..525a77e66 100644 --- a/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModulesListGenerator.groovy +++ b/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModulesListGenerator.groovy @@ -18,6 +18,7 @@ package io.fd.honeycomb.common.scripts import groovy.text.SimpleTemplateEngine +import java.nio.file.Path import java.nio.file.Paths /** @@ -37,7 +38,7 @@ class ModulesListGenerator { // builds project name from group,artifact and version to prevent overwriting // while building multiple distribution project def artifact = project.artifact - def projectName = "${artifact.getGroupId()}_${artifact.getArtifactId()}_${artifact.getVersion()}".replace(".","-") + def projectName = pathFriendlyProjectName(artifact) log.info "Generating list of modules started by distribution ${projectName}" @@ -48,7 +49,7 @@ class ModulesListGenerator { log.info "Project ${projectName} : Found modules ${activeModules}" //creates folder modules - def outputPath = Paths.get(project.build.outputDirectory, StartupScriptGenerator.MINIMAL_RESOURCES_FOLDER, MODULES_FOLDER) + def outputPath = modulesConfigFolder(project) //creates module folder outputPath.toFile().mkdirs() @@ -66,4 +67,12 @@ class ModulesListGenerator { outputFile.text = activeModules.join(System.lineSeparator) } } + + public static Path modulesConfigFolder(project) { + return Paths.get(project.build.outputDirectory, StartupScriptGenerator.MINIMAL_RESOURCES_FOLDER, MODULES_FOLDER) + } + + public static String pathFriendlyProjectName(artifact) { + return "${artifact.getGroupId()}_${artifact.getArtifactId()}_${artifact.getVersion()}".replace(".", "-") + } } diff --git a/common/impl-parent/pom.xml b/common/impl-parent/pom.xml index 7d35518d7..65fb573c3 100644 --- a/common/impl-parent/pom.xml +++ b/common/impl-parent/pom.xml @@ -33,6 +33,7 @@ <guice.version>4.1.0</guice.version> <guice.config.version>1.2.0</guice.config.version> <skinny.logback.version>1.0.8</skinny.logback.version> + <maven-resources-plugin.version>3.0.2</maven-resources-plugin.version> </properties> <dependencyManagement> @@ -68,4 +69,81 @@ </dependency> </dependencies> </dependencyManagement> + + <build> + <pluginManagement> + <!-- Must be done in parent, to unpack jars for all projects that we generate yang module index for --> + <plugins> <!-- Copy all dependencies --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>2.10</version> + <executions> + <execution> + <id>copy-dependencies</id> + <!-- Must be done before generating yang to module index --> + <phase>process-sources</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <outputDirectory>${project.build.directory}/lib</outputDirectory> + <useBaseVersion>true</useBaseVersion> + <useRepositoryLayout>true</useRepositoryLayout> + <excludeArtifactIds>yang-jmx-generator,test-api</excludeArtifactIds> + </configuration> + </execution> + </executions> + </plugin> + <!-- Generate module to yang provider index --> + <plugin> + <groupId>org.codehaus.gmaven</groupId> + <artifactId>groovy-maven-plugin</artifactId> + <executions> + <execution> + <id>generate-yang-index</id> + <phase>generate-resources</phase> + <goals> + <goal>execute</goal> + </goals> + <configuration> + <source> + io.fd.honeycomb.common.scripts.ModuleYangIndexGenerator.generateIndexForPresentModules(project, log) + </source> + </configuration> + </execution> + </executions> + <dependencies> + <dependency> + <groupId>io.fd.honeycomb.common</groupId> + <artifactId>common-scripts</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + </plugin> + </plugins> + </pluginManagement> + + <plugins> + <plugin> + <groupId>org.codehaus.gmaven</groupId> + <artifactId>groovy-maven-plugin</artifactId> + </plugin> + </plugins> + + <resources> + <resource> + <directory>src/main/resources</directory> + <includes> + <include>**/*</include> + </includes> + </resource> + <resource> + <directory>${project.build.outputDirectory}</directory> + <includes> + <include>**/yang-modules-binding/yang-modules</include> + </includes> + </resource> + </resources> + </build> </project> diff --git a/common/minimal-distribution-parent/pom.xml b/common/minimal-distribution-parent/pom.xml index f191fda52..617509b96 100644 --- a/common/minimal-distribution-parent/pom.xml +++ b/common/minimal-distribution-parent/pom.xml @@ -111,31 +111,18 @@ <classpathMavenRepositoryLayout>true</classpathMavenRepositoryLayout> </manifest> <manifestEntries> - <Class-Path>config/ cert/ modules/</Class-Path> + <Class-Path>config/ cert/ modules/ yang-mapping/</Class-Path> </manifestEntries> </archive> </configuration> </plugin> - <!-- Copy all dependencies --> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-dependency-plugin</artifactId> <version>2.10</version> <executions> - <execution> - <id>copy-dependencies</id> - <phase>package</phase> - <goals> - <goal>copy-dependencies</goal> - </goals> - <configuration> - <outputDirectory>${project.build.directory}/lib</outputDirectory> - <useBaseVersion>true</useBaseVersion> - <useRepositoryLayout>true</useRepositoryLayout> - <excludeArtifactIds>yang-jmx-generator</excludeArtifactIds> - </configuration> - </execution> + <!-- Dependencies are copied by parent project --> <execution> <id>unpack-configuration</id> <phase>prepare-package</phase> @@ -182,6 +169,18 @@ </source> </configuration> </execution> + <execution> + <id>generate-module-to-yang-index</id> + <phase>prepare-package</phase> + <goals> + <goal>execute</goal> + </goals> + <configuration> + <source> + io.fd.honeycomb.common.scripts.ModuleYangIndexGenerator.pairDistributionModulesWithYangModules(project, log) + </source> + </configuration> + </execution> </executions> <dependencies> <dependency> |