summaryrefslogtreecommitdiffstats
path: root/common/common-scripts
diff options
context:
space:
mode:
authorJan Srnicek <jsrnicek@cisco.com>2017-06-30 10:08:14 +0200
committerMarek Gradzki <mgradzki@cisco.com>2017-06-30 11:35:35 +0000
commit62dd4d32fd270d3a6b7bb47c972bbcd5dc7b9f43 (patch)
tree029ce8333a79393088a749dbf3575cd8c9c85c03 /common/common-scripts
parentef852789b2c156196a847b211066ae456c2683f5 (diff)
HONEYCOMB-358 - Generate guice to yang modules index
Generates two descriptor files yang-modules-binding/yang-modules - List of Yang modules by project(classpath + deps) yang-mapping/FULL_PROJECT_NAME-yang-modules-index - Index from Guice modules to Yang modules that are used by respective Guice module These files are included in jar files and distribution resources Change-Id: Iafc178219245df9129fb426a5876215c6fd1837e Signed-off-by: Jan Srnicek <jsrnicek@cisco.com>
Diffstat (limited to 'common/common-scripts')
-rw-r--r--common/common-scripts/asciidoc/Readme.adoc3
-rw-r--r--common/common-scripts/pom.xml12
-rw-r--r--common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModuleYangIndexGenerator.groovy262
-rw-r--r--common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModulesListGenerator.groovy13
4 files changed, 288 insertions, 2 deletions
diff --git a/common/common-scripts/asciidoc/Readme.adoc b/common/common-scripts/asciidoc/Readme.adoc
new file mode 100644
index 000000000..d8d5f8e3e
--- /dev/null
+++ b/common/common-scripts/asciidoc/Readme.adoc
@@ -0,0 +1,3 @@
+= common-scripts
+
+Overview of common-scripts \ No newline at end of file
diff --git a/common/common-scripts/pom.xml b/common/common-scripts/pom.xml
index e41364154..7be29b662 100644
--- a/common/common-scripts/pom.xml
+++ b/common/common-scripts/pom.xml
@@ -31,6 +31,8 @@
<groovy.version>2.4.7</groovy.version>
<groovy.eclipse.compiler.version>2.9.2-01</groovy.eclipse.compiler.version>
<groovy.eclipse.batch.version>2.4.3-01</groovy.eclipse.batch.version>
+ <commons-io.version>2.5</commons-io.version>
+ <yang-binding.version>0.9.3-Boron-SR3</yang-binding.version>
</properties>
<build>
@@ -82,6 +84,16 @@
<artifactId>groovy-all</artifactId>
<version>${groovy.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.opendaylight.mdsal</groupId>
+ <artifactId>yang-binding</artifactId>
+ <version>${yang-binding.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>${commons-io.version}</version>
+ </dependency>
</dependencies>
</project> \ No newline at end of file
diff --git a/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModuleYangIndexGenerator.groovy b/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModuleYangIndexGenerator.groovy
new file mode 100644
index 000000000..21877c51c
--- /dev/null
+++ b/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModuleYangIndexGenerator.groovy
@@ -0,0 +1,262 @@
+/*
+ * Copyright (c) 2017 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.honeycomb.common.scripts
+
+import com.google.common.base.Strings
+import com.google.common.io.Files
+import org.apache.commons.io.FileUtils
+import org.opendaylight.yangtools.yang.binding.YangModelBindingProvider
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Paths
+import java.util.jar.JarFile
+import java.util.stream.Collectors
+
+/**
+ * Provides logic to generate:
+ * <li><b>generateIndexForPresentModules()</b> - yang-modules-binding/yang-modules -
+ * List of Yang modules used by project(classpath + deps)</li>
+ * <li><b>pairDistributionModulesWithYangModules()</b> - yang-mapping/FULL_PROJECT_NAME-yang-modules-index -
+ * Index from Guice modules to Yang modules that are used by respective Guice module</li>
+ * <br>
+ * These files can be then included in jars and distribution resources to allow
+ * conditional yang module loading according to list of Guice modules that are started by distribution
+ * */
+class ModuleYangIndexGenerator {
+
+ private static final String YANG_MODEL_PROVIDER_NAME = YangModelBindingProvider.class.getName()
+ private static
+ final YANG_PROVIDERS_PATH = "META-INF/services/" + YANG_MODEL_PROVIDER_NAME
+ private static final MODULES_DELIMITER = ","
+ private static final CLASS_EXT = "class"
+ private static final String[] EXTENSIONS = [CLASS_EXT]
+ private static final YANG_MODULES_FOLDER = "yang-modules-binding"
+ private static final YANG_MODULES_FILE_NAME = "yang-modules"
+ private static final YANG_MAPPING_FOLDER = "yang-mapping"
+ private static final YANG_MODULES_INDEX_FILE_NAME = "yang-modules-index"
+
+ public static void generateIndexForPresentModules(project, log) {
+ log.info "Checking module providers for project ${project.getName()}"
+ // checks module provides from dependencies
+ // folder with extracted libs
+ def libsFolder = Paths.get(project.getBuild().getDirectory(), "lib")
+ if (!libsFolder.toFile().exists()) {
+ // Plugin for collecting dependencies is executed from parent project,
+ // therefore it will run also for parent, that does not have any depedencies(just dep management)
+ // so lib folder wont be created
+ log.info "Folder ${libsFolder} does not exist - No dependencies to process"
+ return;
+ }
+
+ String yangModules = java.nio.file.Files.walk(libsFolder)
+ .map { path -> path.toFile() }
+ .filter { file -> file.isFile() }
+ .filter { file -> file.getName().endsWith(".jar") }
+ .map { file -> getModuleProviderContentFromApiJar(new JarFile(file), log) }
+ .filter { content -> !Strings.isNullOrEmpty(content.trim()) }
+ .collect().join(MODULES_DELIMITER)
+ log.info "Yang yangModules found : $yangModules"
+ def outputDir = Paths.get(project.getBuild().getOutputDirectory(), YANG_MODULES_FOLDER).toFile()
+ outputDir.mkdirs()
+ def outputFile = Paths.get(outputDir.getPath(), YANG_MODULES_FILE_NAME).toFile()
+ outputFile.createNewFile()
+ Files.write(yangModules, outputFile, StandardCharsets.UTF_8)
+ log.info "Yang yangModules configuration successfully written to ${outputFile.getPath()}"
+ }
+
+ /**
+ * Loads module list of current distribution, and attempts
+ * to pair them with yang module providers from either their classpath or direct/indirect dependencies.
+ * */
+ public static void pairDistributionModulesWithYangModules(project, log) {
+ def modules = modulesList(project)
+ if (modules.isEmpty()) {
+ log.warn "No distribution modules defined, skipping"
+ return
+ }
+
+ log.info "Pairing distribution modules ${modules} to yang modules"
+ def moduleToYangModulesIndex = new HashMap<String, String>()
+ def outputDir = project.getBuild().getOutputDirectory()
+
+ // TODO - HONEYCOMB-373 - eliminate local matching after distribution modules are moved to separate project
+ // first ,matches modules against local files, helps to filter local classpath modules, to reduce scope
+ // of dependency scanning that is more performance heavy
+ log.info "Pairing against local classpath"
+ pairAgainsLocalFiles(outputDir, modules, moduleToYangModulesIndex, log)
+
+ // go to dependencies only if some modules are left. this occurs for modules
+ // started by distribution that are not part of its classpath(basically all plugin modules)
+ if (!modules.isEmpty()) {
+ log.info "Pairing against dependencies"
+ // The rest of the modules is looked up in dependencies
+ pairAgainsDependencyArtifacts(project, modules, log, moduleToYangModulesIndex)
+ }
+
+ // for ex.: /target/honeycomb-minimal-resources/yang-mapping
+ def yangMappingFolder = Paths.get(project.getBuild().getOutputDirectory(), StartupScriptGenerator.MINIMAL_RESOURCES_FOLDER, YANG_MAPPING_FOLDER).toFile()
+
+ if (!yangMappingFolder.exists()) {
+ yangMappingFolder.mkdir()
+ }
+ def outputFileName = "${ModulesListGenerator.pathFriendlyProjectName(project.artifact)}_$YANG_MODULES_INDEX_FILE_NAME"
+
+ def outputFile = Paths.get(yangMappingFolder.getPath(), outputFileName).toFile()
+ outputFile.createNewFile()
+
+ def indexFileContent = moduleToYangModulesIndex.entrySet()
+ .stream()
+ .map { entry -> "GUICE_MODULE:${entry.getKey()}|YANG_MODULES:${entry.getValue()}${System.lineSeparator()}" }
+ .collect(Collectors.joining())
+
+ Files.write(indexFileContent, outputFile, StandardCharsets.UTF_8)
+ if (!modules.isEmpty()) {
+ log.warn "No yang configuration found for modules ${modules}"
+ }
+ log.info "Distribution to yang modules index successfully generated to $outputFile"
+
+ }
+
+ // provides list of modules for distribution, not from property, but already processed list from /modules folder.
+ // this allows us to skip all validation that is present in modules list generation, and just take final list of modules
+ private static Set<String> modulesList(project) {
+ def modulesFolder = ModulesListGenerator.modulesConfigFolder(project).toFile()
+ Arrays.stream(modulesFolder.listFiles())
+ // picks up only file for currently processed distribution
+ .filter { file -> file.getName().contains(ModulesListGenerator.pathFriendlyProjectName(project.artifact)) }
+ .map { file -> FileUtils.readLines(file, StandardCharsets.UTF_8) }
+ .flatMap { lines -> lines.stream() }
+ .map { line -> line.replace("//", "") }
+ .map { line -> line.trim() }
+ .collect(Collectors.toSet())
+ }
+
+ private static void pairAgainsDependencyArtifacts(project, modules, log, index) {
+ // loads jar file
+ def artifacts = project.getDependencyArtifacts()
+ log.info "Artifacts used for pairing $artifacts"
+ artifacts.stream()
+ .map { artifact -> artifact.getFile() }
+ .map { file -> new JarFile(file) }
+ .forEach { jar ->
+ // first tries to find content of yang module provides file,
+ // if not found, skip's this jar
+ def moduleProvidersContent = getModuleProviderContentFromImplJar(jar, log)
+ if (Strings.isNullOrEmpty(moduleProvidersContent.trim())) {
+ log.debug "No yang module configuration found in ${jar.getName()}"
+ return
+ }
+
+ def entryNames = Collections.list(jar.entries()).stream()
+ .map { entry -> entry.getName() }
+ .filter { name -> name.endsWith(CLASS_EXT) }
+ .map { name -> pathToClassName(name) }
+ .collect(Collectors.toSet())
+
+ log.info "Entries $entryNames"
+ log.info "Modules $modules"
+ for (String module : modules) {
+ if (entryNames.contains(module)) {
+ log.info "Module $module found in artifact ${jar.getName()}"
+ index.put(module, moduleProvidersContent)
+ }
+ }
+ }
+ modules.removeAll(index.keySet());
+ log.info "Modules left after dependency pairing $modules"
+ }
+
+ // TODO - HONEYCOMB-373 - eliminate local matching
+ private static void pairAgainsLocalFiles(outputDir, modules, HashMap<String, String> index, log) {
+ // Pairs modules that are part of distribution classpath
+ def yangModulesLocalConfig = Paths.get(outputDir, YANG_MODULES_FOLDER, YANG_MODULES_FILE_NAME).toFile()
+ if (!yangModulesLocalConfig.exists()) {
+ log.debug "Local configuration for yang modules does not exist, skiping local matching"
+ return
+ }
+
+ log.info "Local file ${yangModulesLocalConfig}"
+ def localYangModules = fixDelimiters(FileUtils.readFileToString(yangModulesLocalConfig, StandardCharsets.UTF_8))
+
+ log.info "Output dir $outputDir"
+ FileUtils.listFiles(Paths.get(outputDir).toFile(), EXTENSIONS, true)
+ .stream()
+ .map { file -> file.getPath() }
+ .map { path -> relativizePath(path, outputDir) }
+ .forEach { path ->
+ for (String module : modules) {
+ if (path.equals(classNameToPath(module))) {
+ log.info "Module $module found in local classpath"
+ // mapping by standard class name
+ index.put(module, localYangModules)
+ }
+ }
+ }
+
+ // remove all matching modules to reduce scope of search
+ modules.removeAll(index.keySet());
+ log.info "Modules left after local classpath pairing $modules"
+ }
+
+ private static String relativizePath(String path, String outputDir) {
+ return path.replace(outputDir, "").substring(1).trim();
+ }
+
+ private static String pathToClassName(String path) {
+ return path.replace("/", ".").replace(".class", "").trim()
+ }
+
+ private static String classNameToPath(String className) {
+ return className.replace(".", "/").concat(".class").trim()
+ }
+
+ private static String getModuleProviderContentFromImplJar(JarFile jarFile, log) {
+ def moduleProviderEntry = jarFile.getJarEntry(YANG_MODULES_FOLDER + "/" + YANG_MODULES_FILE_NAME)
+ if (moduleProviderEntry == null) {
+ return "";
+ }
+ // module provider files are in general a couple of lines, so should'nt be a problem
+ // to read at once
+ InputStream input = jarFile.getInputStream(moduleProviderEntry)
+ byte[] data = new byte[(int) moduleProviderEntry.getSize()]
+ input.read(data)
+ input.close()
+
+ return fixDelimiters(new String(data, StandardCharsets.UTF_8));
+ }
+
+ private static String getModuleProviderContentFromApiJar(JarFile jarFile, log) {
+ def moduleProviderEntry = jarFile.getJarEntry(YANG_PROVIDERS_PATH)
+ if (moduleProviderEntry == null) {
+ return "";
+ }
+ // module provider files are in general a couple of lines, so should'nt be a problem
+ // to read at once
+ InputStream input = jarFile.getInputStream(moduleProviderEntry)
+ byte[] data = new byte[(int) moduleProviderEntry.getSize()]
+ input.read(data)
+ input.close()
+
+ return fixDelimiters(new String(data, StandardCharsets.UTF_8))
+ }
+
+ private static String fixDelimiters(String data) {
+ return Arrays.stream(data.split(System.lineSeparator()))
+ .map { line -> line.trim() }
+ .collect().join(MODULES_DELIMITER)
+ }
+}
diff --git a/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModulesListGenerator.groovy b/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModulesListGenerator.groovy
index c7a74d20e..525a77e66 100644
--- a/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModulesListGenerator.groovy
+++ b/common/common-scripts/src/main/groovy/io/fd/honeycomb/common/scripts/ModulesListGenerator.groovy
@@ -18,6 +18,7 @@ package io.fd.honeycomb.common.scripts
import groovy.text.SimpleTemplateEngine
+import java.nio.file.Path
import java.nio.file.Paths
/**
@@ -37,7 +38,7 @@ class ModulesListGenerator {
// builds project name from group,artifact and version to prevent overwriting
// while building multiple distribution project
def artifact = project.artifact
- def projectName = "${artifact.getGroupId()}_${artifact.getArtifactId()}_${artifact.getVersion()}".replace(".","-")
+ def projectName = pathFriendlyProjectName(artifact)
log.info "Generating list of modules started by distribution ${projectName}"
@@ -48,7 +49,7 @@ class ModulesListGenerator {
log.info "Project ${projectName} : Found modules ${activeModules}"
//creates folder modules
- def outputPath = Paths.get(project.build.outputDirectory, StartupScriptGenerator.MINIMAL_RESOURCES_FOLDER, MODULES_FOLDER)
+ def outputPath = modulesConfigFolder(project)
//creates module folder
outputPath.toFile().mkdirs()
@@ -66,4 +67,12 @@ class ModulesListGenerator {
outputFile.text = activeModules.join(System.lineSeparator)
}
}
+
+ public static Path modulesConfigFolder(project) {
+ return Paths.get(project.build.outputDirectory, StartupScriptGenerator.MINIMAL_RESOURCES_FOLDER, MODULES_FOLDER)
+ }
+
+ public static String pathFriendlyProjectName(artifact) {
+ return "${artifact.getGroupId()}_${artifact.getArtifactId()}_${artifact.getVersion()}".replace(".", "-")
+ }
}