From a90863760d1ae1a92520ce29841aec600d25a83a Mon Sep 17 00:00:00 2001 From: Jan Srnicek Date: Fri, 30 Jun 2017 15:10:27 +0200 Subject: HONEYCOMB-358 - Conditional module loading Change-Id: Ic9b7182cc77bf2f73cf5edd3ee19f25f53711cda Signed-off-by: Jan Srnicek --- .../java/io/fd/honeycomb/infra/distro/Main.java | 1 - .../infra/distro/activation/ActivationConfig.java | 8 ++ .../distro/activation/ActiveModuleProvider.java | 2 +- .../distro/schema/ModuleInfoBackedCtxProvider.java | 11 +- .../infra/distro/schema/ResourceLoader.java | 135 +++++++++++++++++++++ .../distro/schema/YangBindingProviderModule.java | 57 +-------- .../distro/schema/YangModuleMappingIndex.java | 69 +++++++++++ .../infra/distro/schema/YangModulesProvider.java | 97 +++++++++++++++ 8 files changed, 318 insertions(+), 62 deletions(-) create mode 100644 infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/ResourceLoader.java create mode 100644 infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/YangModuleMappingIndex.java create mode 100644 infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/YangModulesProvider.java (limited to 'infra/minimal-distribution/src/main/java') diff --git a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/Main.java b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/Main.java index 265740bcc..582fcf687 100644 --- a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/Main.java +++ b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/Main.java @@ -22,7 +22,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSet; import com.google.inject.ConfigurationException; import com.google.inject.CreationException; -import com.google.inject.Guice; import com.google.inject.Injector; import com.google.inject.Key; import com.google.inject.Module; diff --git a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/activation/ActivationConfig.java b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/activation/ActivationConfig.java index 8777c32a2..a17f5d632 100644 --- a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/activation/ActivationConfig.java +++ b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/activation/ActivationConfig.java @@ -27,7 +27,15 @@ public class ActivationConfig { @InjectConfig("modules-resource-path") private String modulesResourcePath; + @InjectConfig("yang-modules-index-path") + private String yangModulesIndexPath; + public String getModulesResourcePath() { return Optional.ofNullable(modulesResourcePath).orElse("../modules/"); } + + public String getYangModulesIndexPath() { + return Optional.ofNullable(yangModulesIndexPath).orElse("../yang-mapping/"); + } + } diff --git a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/activation/ActiveModuleProvider.java b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/activation/ActiveModuleProvider.java index 21d108082..d31024ce9 100644 --- a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/activation/ActiveModuleProvider.java +++ b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/activation/ActiveModuleProvider.java @@ -139,7 +139,7 @@ class ActiveModuleProvider implements Provider { private static Class moduleNameToClass(final String name, final ClassLoader classLoader) { try { - LOG.info("Loading module class {}", name); + LOG.debug("Loading module class {}", name); return (Class) classLoader.loadClass(name); } catch (ClassNotFoundException e) { LOG.error("Unable to convert {} to class, make sure you've provided sources to classpath", name); diff --git a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/ModuleInfoBackedCtxProvider.java b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/ModuleInfoBackedCtxProvider.java index f82cdd0b1..e2d0fbfaa 100644 --- a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/ModuleInfoBackedCtxProvider.java +++ b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/ModuleInfoBackedCtxProvider.java @@ -16,11 +16,11 @@ package io.fd.honeycomb.infra.distro.schema; +import static io.fd.honeycomb.infra.distro.schema.YangModulesProvider.YangModules; + import com.google.common.base.MoreObjects; import com.google.inject.Inject; import io.fd.honeycomb.infra.distro.ProviderTrait; -import java.util.HashSet; -import java.util.Set; import java.util.stream.Collectors; import org.opendaylight.yangtools.sal.binding.generator.impl.ModuleInfoBackedContext; import org.opendaylight.yangtools.yang.binding.YangModelBindingProvider; @@ -30,13 +30,14 @@ import org.slf4j.LoggerFactory; public final class ModuleInfoBackedCtxProvider extends ProviderTrait { private static final Logger LOG = LoggerFactory.getLogger(ModuleInfoBackedCtxProvider.class); - @Inject(optional = true) - private Set moduleInfos = new HashSet<>(); + // optional in sense that list of modules inside can be empty if none was found + @Inject + private YangModules moduleInfos; @Override protected ModuleInfoBackedContext create() { ModuleInfoBackedContext create = ModuleInfoBackedContext.create(); - create.addModuleInfos(moduleInfos.stream() + create.addModuleInfos(moduleInfos.getYangBindings().stream() .map(YangModelBindingProvider::getModuleInfo) .collect(Collectors.toList())); LOG.debug("ModuleInfoBackedContext created from {}", moduleInfos); diff --git a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/ResourceLoader.java b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/ResourceLoader.java new file mode 100644 index 000000000..c851dab1b --- /dev/null +++ b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/ResourceLoader.java @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2017 Cisco and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fd.honeycomb.infra.distro.schema; + +import static com.google.common.base.Preconditions.checkNotNull; +import static java.lang.String.format; + +import com.google.common.base.Charsets; +import com.google.common.base.Strings; +import com.google.common.io.Resources; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.Set; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.stream.Collectors; +import org.apache.commons.io.IOUtils; + +interface ResourceLoader { + + default Set loadResourceContentsOnPath(final String path) { + final URL folderUrl = getClass().getClassLoader().getResource(path); + checkNotNull(folderUrl, "Resources %s not found", path); + + if (ResourceLoaderIml.urlToUri(folderUrl).getScheme().equals("jar")) { + return ResourceLoaderIml.readFromJar(path, folderUrl); + } else { + return ResourceLoaderIml.readFromFolder(folderUrl); + } + + } + + final class ResourceLoaderIml { + + private static Set readFromFolder(final URL folderUrl) { + final File folder = new File(folderUrl.getPath()); + final File[] files = checkNotNull(folder.listFiles(), "No files present on path %s", folderUrl); + return Arrays.stream(files) + .map(ResourceLoaderIml::fileToUrl) + .map(ResourceLoaderIml::urlToContentString) + .flatMap(content -> Arrays.stream(content.split(System.lineSeparator()))) + .filter(ResourceLoaderIml::filterNonEmpty) + .collect(Collectors.toSet()); + } + + private static Set readFromJar(final String path, final URL url) { + final String uriString = urlToUri(url).toString(); + final String fileReference = extractJarFilePath(uriString); + try (JarFile jar = new JarFile(new File(fileReference))) { + return Collections.list(jar.entries()) + .stream() + .filter(jarEntry -> jarEntry.getName().contains(path)) + .map(jarEntry -> getJarEntryStream(jar, jarEntry)) + .map(ResourceLoaderIml::readJarEntryStream) + .flatMap(content -> Arrays.stream(content.split(System.lineSeparator()))) + .filter(ResourceLoaderIml::filterNonEmpty) + .collect(Collectors.toSet()); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + private static String extractJarFilePath(final String uriString) { + return uriString.substring(0, uriString.indexOf("!")).replace("jar:file:", ""); + } + + private static boolean filterNonEmpty(final String line) { + return !Strings.isNullOrEmpty(line.trim()); + } + + private static String readJarEntryStream(final InputStream inputStream) { + try { + final String value = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + IOUtils.closeQuietly(inputStream); + return value; + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + private static InputStream getJarEntryStream(final JarFile jar, final JarEntry jarEntry) { + try { + return jar.getInputStream(jarEntry); + } catch (IOException e) { + throw new IllegalStateException(format("Unable to get stream for entry %s | jar %s", jar, jarEntry)); + } + } + + private static URI urlToUri(final URL url) { + try { + return url.toURI(); + } catch (URISyntaxException e) { + throw new IllegalStateException(format("Unable to convert URL %s to URI", url)); + } + } + + private static String urlToContentString(final URL url) { + try { + return Resources.toString(url, Charsets.UTF_8); + } catch (IOException e) { + throw new IllegalArgumentException("Unable to read resource from: " + url, e); + } + } + + private static URL fileToUrl(final File file) { + try { + return file.toURI().toURL(); + } catch (MalformedURLException e) { + throw new IllegalStateException(e); + } + } + } +} diff --git a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/YangBindingProviderModule.java b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/YangBindingProviderModule.java index 8e402808b..d705226bf 100644 --- a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/YangBindingProviderModule.java +++ b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/YangBindingProviderModule.java @@ -16,68 +16,15 @@ package io.fd.honeycomb.infra.distro.schema; -import com.google.common.base.Charsets; -import com.google.common.base.Strings; -import com.google.common.collect.Lists; -import com.google.common.io.Resources; import com.google.inject.AbstractModule; -import com.google.inject.Singleton; -import com.google.inject.multibindings.Multibinder; -import java.io.IOException; -import java.net.URL; -import java.util.Collections; -import java.util.List; -import javax.annotation.Nonnull; -import org.opendaylight.yangtools.yang.binding.YangModelBindingProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Load all YangModelBindingProvider classes from classpath. - *

- * Relying on /META-INF/services/ metadata. - */ public class YangBindingProviderModule extends AbstractModule { private static final Logger LOG = LoggerFactory.getLogger(YangBindingProviderModule.class); - private static final String YANG_BA_PROVIDER_PATH = "META-INF/services/" + YangModelBindingProvider.class.getName(); - protected void configure() { - final Multibinder binder = - Multibinder.newSetBinder(binder(), YangModelBindingProvider.class); - final List resources; - try { - resources = Collections.list(getClass().getClassLoader().getResources(YANG_BA_PROVIDER_PATH)); - } catch (IOException e) { - throw new IllegalStateException("Unable to load binding providers from path: " + YANG_BA_PROVIDER_PATH, e); - } - LOG.debug("ModuleProviders found at {}", resources); - resources.stream() - .map(YangBindingProviderModule::urlToString) - .flatMap(content -> Lists.newArrayList(content.split("\n")).stream()) - .filter(line -> !Strings.isNullOrEmpty(line.trim())) - .distinct() - .map(YangBindingProviderModule::loadClass) - .forEach(providerClass -> { - LOG.debug("ModuleProvider found for {}", providerClass); - binder.addBinding().to((Class) providerClass) - .in(Singleton.class); - }); - } - - private static Class loadClass(@Nonnull final String className) { - try { - return Class.forName(className); - } catch (ClassNotFoundException e) { - throw new IllegalArgumentException("Unable to load class: " + className, e); - } - } - - private static String urlToString(@Nonnull final URL url) { - try { - return Resources.toString(url, Charsets.UTF_8); - } catch (IOException e) { - throw new IllegalArgumentException("Unable to read resource from: " + url, e); - } + LOG.info("Configuring YangBindingProviderModule"); + bind(YangModulesProvider.YangModules.class).toProvider(YangModulesProvider.class).asEagerSingleton(); } } diff --git a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/YangModuleMappingIndex.java b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/YangModuleMappingIndex.java new file mode 100644 index 000000000..a483cfd9c --- /dev/null +++ b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/YangModuleMappingIndex.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fd.honeycomb.infra.distro.schema; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.LinkedListMultimap; +import com.google.common.collect.Multimap; +import java.util.Arrays; +import java.util.Set; +import java.util.stream.Stream; +import javax.annotation.Nonnull; + +/** + * Index from guice module to yang module providers + */ +class YangModuleMappingIndex implements ResourceLoader { + + private static final String G_MODULE_TOKEN = "GUICE_MODULE:"; + private static final String KEY_VALUE_SEPARATOR = "|"; + private static final String Y_MODULE_TOKEN = "YANG_MODULES:"; + private static final String Y_MODULE_SEPARATOR = ","; + + /** + * key - module class name + * value - yang module provider + */ + private final Multimap index; + + YangModuleMappingIndex(final String indexPath) { + this.index = LinkedListMultimap.create(); + loadResourceContentsOnPath(indexPath) + .forEach(line -> { + final String moduleName = parseModuleName(line); + parseYangModules(line).forEach(yModuleProvider -> index.put(moduleName, yModuleProvider)); + }); + } + + Set getByModuleName(@Nonnull final String moduleName) { + return ImmutableSet.copyOf(index.get(moduleName)); + } + + int applicationModulesCount() { + return index.keySet().size(); + } + + private static String parseModuleName(final String rawLine) { + return rawLine.substring(rawLine.indexOf(G_MODULE_TOKEN) + G_MODULE_TOKEN.length(), + rawLine.indexOf(KEY_VALUE_SEPARATOR)); + } + + private static Stream parseYangModules(final String rawLine) { + return Arrays.stream(rawLine.substring(rawLine.indexOf(Y_MODULE_TOKEN) + Y_MODULE_TOKEN.length()) + .split(Y_MODULE_SEPARATOR)); + } +} diff --git a/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/YangModulesProvider.java b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/YangModulesProvider.java new file mode 100644 index 000000000..f6d8ea0c1 --- /dev/null +++ b/infra/minimal-distribution/src/main/java/io/fd/honeycomb/infra/distro/schema/YangModulesProvider.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fd.honeycomb.infra.distro.schema; + + +import static java.lang.String.format; + +import com.google.common.base.Charsets; +import com.google.common.io.Resources; +import com.google.inject.Inject; +import com.google.inject.Provider; +import io.fd.honeycomb.infra.distro.activation.ActivationConfig; +import io.fd.honeycomb.infra.distro.activation.ActiveModules; +import java.io.IOException; +import java.net.URL; +import java.util.Collection; +import java.util.Set; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import org.opendaylight.yangtools.yang.binding.YangModelBindingProvider; + +/** + * Loads active yang modules + * Relying on generate yang-module-index + */ +public class YangModulesProvider implements Provider { + + @Inject + private ActiveModules activeModules; + + @Inject + private ActivationConfig config; + + @Override + public YangModules get() { + // no need to bind this, pretty big map and its needed just here + final YangModuleMappingIndex index = new YangModuleMappingIndex(config.getYangModulesIndexPath()); + + return new YangModules(activeModules.getActiveModulesClasses().stream() + .map(Class::getName) + .map(index::getByModuleName) + .flatMap(Collection::stream) + .map(YangModulesProvider::loadClass) + .map(aClass -> (Class) aClass) + .collect(Collectors.toSet())); + } + + static class YangModules { + private final Set> yangBindings; + + YangModules(final Set> yangBindings) { + this.yangBindings = yangBindings; + } + + Set getYangBindings() { + return yangBindings.stream() + .map(providerClass -> { + try { + return providerClass.newInstance(); + } catch (InstantiationException | IllegalAccessException e) { + throw new IllegalStateException(format("Unable to create instance of %s", providerClass), + e); + } + }).collect(Collectors.toSet()); + } + } + + private static Class loadClass(@Nonnull final String className) { + try { + return Class.forName(className); + } catch (ClassNotFoundException e) { + throw new IllegalArgumentException("Unable to load class: " + className, e); + } + } + + static String urlToString(@Nonnull final URL url) { + try { + return Resources.toString(url, Charsets.UTF_8); + } catch (IOException e) { + throw new IllegalArgumentException("Unable to read resource from: " + url, e); + } + } +} -- cgit 1.2.3-korg