From ffd80be44b795865b42edd60d587a577db54cae3 Mon Sep 17 00:00:00 2001 From: Maros Marsalek Date: Mon, 23 May 2016 15:22:24 +0200 Subject: HONEYCOMB-61: Detect VPP disconnect using keepalives Change-Id: Ic664dbf452504d0fff97e8c766d735d9c5d95c72 Signed-off-by: Maros Marsalek --- v3po/v3po2vpp/pom.xml | 14 ++++ v3po/v3po2vpp/src/main/config/default-config.xml | 10 +++ .../translate/v3po/vppstate/VersionCustomizer.java | 16 +++-- .../rev160406/VppStateHoneycombReaderModule.java | 77 +++++++++++++++++++++- v3po/v3po2vpp/src/main/yang/v3po2vpp.yang | 12 ++++ .../v3po/vppstate/VersionCustomizerTest.java | 10 +-- .../v3po/translate/v3po/vppstate/VppStateTest.java | 11 ++-- 7 files changed, 130 insertions(+), 20 deletions(-) (limited to 'v3po/v3po2vpp') diff --git a/v3po/v3po2vpp/pom.xml b/v3po/v3po2vpp/pom.xml index 113511da1..d02807c9d 100644 --- a/v3po/v3po2vpp/pom.xml +++ b/v3po/v3po2vpp/pom.xml @@ -49,6 +49,20 @@ v3po-api ${project.version} + + + + org.opendaylight.controller + config-util + 0.4.2-Beryllium-SR2 + + + org.opendaylight.controller + threadpool-config-api + 0.4.2-Beryllium-SR2 + + + ${project.groupId} translate-impl diff --git a/v3po/v3po2vpp/src/main/config/default-config.xml b/v3po/v3po2vpp/src/main/config/default-config.xml index be45a291b..19226d4fc 100644 --- a/v3po/v3po2vpp/src/main/config/default-config.xml +++ b/v3po/v3po2vpp/src/main/config/default-config.xml @@ -23,6 +23,9 @@ urn:honeycomb:params:xml:ns:yang:v3po2vpp?module=v3po2vpp&revision=2016-04-06 urn:honeycomb:params:xml:ns:yang:data:api?module=data-api&revision=2016-04-11 urn:honeycomb:params:xml:ns:yang:vpp:util?module=vpp-util&revision=2016-04-06 + + urn:opendaylight:params:xml:ns:yang:controller:threadpool:impl:scheduled?module=threadpool-impl-scheduled&revision=2013-12-01 + @@ -54,6 +57,13 @@ prefix:naming-context bridge-domain-context + + + + prefix:threadpool + + global-netconf-ssh-scheduled-executor + prefix:interfaces-state-honeycomb-reader diff --git a/v3po/v3po2vpp/src/main/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizer.java b/v3po/v3po2vpp/src/main/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizer.java index ad181a701..8122fd245 100644 --- a/v3po/v3po2vpp/src/main/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizer.java +++ b/v3po/v3po2vpp/src/main/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizer.java @@ -21,6 +21,7 @@ import io.fd.honeycomb.v3po.translate.read.ReadFailedException; import io.fd.honeycomb.v3po.translate.spi.read.ChildReaderCustomizer; import io.fd.honeycomb.v3po.translate.v3po.util.FutureJVppCustomizer; import io.fd.honeycomb.v3po.translate.v3po.util.TranslateUtils; +import java.util.concurrent.CompletionStage; import javax.annotation.Nonnull; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.VppStateBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.vpp.state.Version; @@ -36,6 +37,11 @@ public final class VersionCustomizer extends FutureJVppCustomizer implements ChildReaderCustomizer { + /** + * Default timeout for executing version read + */ + private static final int DEFAULT_TIMEOUT_IN_SECONDS = 30; + public VersionCustomizer(@Nonnull final FutureJVpp futureJVpp) { super(futureJVpp); } @@ -54,13 +60,11 @@ public final class VersionCustomizer @Override public void readCurrentAttributes(@Nonnull InstanceIdentifier id, @Nonnull final VersionBuilder builder, @Nonnull final ReadContext context) throws ReadFailedException { + // Execute with timeout + final CompletionStage showVersionFuture = getFutureJVpp().showVersion(new ShowVersion()); + final ShowVersionReply reply = TranslateUtils.getReply(showVersionFuture.toCompletableFuture(), id, + DEFAULT_TIMEOUT_IN_SECONDS); - ShowVersionReply reply; - try { - reply = getFutureJVpp().showVersion(new ShowVersion()).toCompletableFuture().get(); - } catch (Exception e) { - throw new ReadFailedException(id, e); - } builder.setBranch(TranslateUtils.toString(reply.version)); builder.setName(TranslateUtils.toString(reply.program)); builder.setBuildDate(TranslateUtils.toString(reply.buildDate)); diff --git a/v3po/v3po2vpp/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/v3po2vpp/rev160406/VppStateHoneycombReaderModule.java b/v3po/v3po2vpp/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/v3po2vpp/rev160406/VppStateHoneycombReaderModule.java index 9a0165577..32e5dabc0 100644 --- a/v3po/v3po2vpp/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/v3po2vpp/rev160406/VppStateHoneycombReaderModule.java +++ b/v3po/v3po2vpp/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/v3po2vpp/rev160406/VppStateHoneycombReaderModule.java @@ -4,14 +4,26 @@ import io.fd.honeycomb.v3po.translate.impl.read.CompositeChildReader; import io.fd.honeycomb.v3po.translate.impl.read.CompositeListReader; import io.fd.honeycomb.v3po.translate.impl.read.CompositeRootReader; import io.fd.honeycomb.v3po.translate.read.ChildReader; +import io.fd.honeycomb.v3po.translate.util.KeepaliveReaderWrapper; import io.fd.honeycomb.v3po.translate.util.RWUtils; import io.fd.honeycomb.v3po.translate.util.read.CloseableReader; import io.fd.honeycomb.v3po.translate.util.read.ReflexiveChildReaderCustomizer; import io.fd.honeycomb.v3po.translate.util.read.ReflexiveRootReaderCustomizer; +import io.fd.honeycomb.v3po.translate.v3po.util.ReadTimeoutException; import io.fd.honeycomb.v3po.translate.v3po.vppstate.BridgeDomainCustomizer; import io.fd.honeycomb.v3po.translate.v3po.vppstate.VersionCustomizer; +import java.lang.management.ManagementFactory; import java.util.ArrayList; import java.util.List; +import javax.management.Attribute; +import javax.management.InstanceNotFoundException; +import javax.management.ObjectName; +import org.opendaylight.controller.config.api.ConflictingVersionException; +import org.opendaylight.controller.config.api.ValidationException; +import org.opendaylight.controller.config.util.ConfigRegistryJMXClient; +import org.opendaylight.controller.config.util.ConfigTransactionJMXClient; +import org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.vpp.jvpp.cfg.rev160406.VppJvppImplModule; +import org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.vpp.jvpp.cfg.rev160406.VppJvppImplModuleFactory; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.VppState; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.VppStateBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.vpp.state.BridgeDomains; @@ -22,8 +34,13 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.vpp.state.bridge.domains.BridgeDomainKey; import org.opendaylight.yangtools.yang.binding.ChildOf; import org.openvpp.jvpp.future.FutureJVpp; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class VppStateHoneycombReaderModule extends org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.v3po2vpp.rev160406.AbstractVppStateHoneycombReaderModule { + + private static final Logger LOG = LoggerFactory.getLogger(VppStateHoneycombReaderModule.class); + public VppStateHoneycombReaderModule(org.opendaylight.controller.config.api.ModuleIdentifier identifier, org.opendaylight.controller.config.api.DependencyResolver dependencyResolver) { super(identifier, dependencyResolver); } @@ -41,7 +58,13 @@ public class VppStateHoneycombReaderModule extends org.opendaylight.yang.gen.v1. public java.lang.AutoCloseable createInstance() { final FutureJVpp vppApi = getVppJvppDependency(); - final ChildReader versionReader = new CompositeChildReader<>(Version.class, new VersionCustomizer(vppApi)); + ChildReader versionReader = new CompositeChildReader<>(Version.class, new VersionCustomizer(vppApi)); + // Wrap with keepalive reader to detect connection issues + // TODO keepalive reader wrapper relies on VersionReaderCustomizer (to perform timeout on reads) + // Once readers+customizers are asynchronous, pull the timeout to keepalive executor so that keepalive wrapper + // is truly generic + versionReader = new KeepaliveReaderWrapper<>(versionReader, getKeepaliveExecutorDependency().getExecutor(), + ReadTimeoutException.class, 30, () -> reinitializeJVpp(reinitializationCounter)); final CompositeListReader bridgeDomainReader = new CompositeListReader<>(BridgeDomain.class, new BridgeDomainCustomizer(vppApi, @@ -62,4 +85,56 @@ public class VppStateHoneycombReaderModule extends org.opendaylight.yang.gen.v1. RWUtils.emptyAugReaderList(), new ReflexiveRootReaderCustomizer<>(VppStateBuilder.class))); } + + private static long reinitializationCounter; + private static final long reinitializationLimit = 10; + + /** + * In case we detect connection issues with VPP, reinitialize JVpp + */ + private void reinitializeJVpp(final long currentAttempt) { + // FIXME https://jira.fd.io/browse/HONEYCOMB-78 This code correctly re-initializes all the components + // starting with jvpp, but jvpp reconnect fails. Test in a JVpp test and then from C + LOG.info("Reinitializing JVpp, attempt: {}", currentAttempt); + + final long nextAttempt = currentAttempt + 1; + if (nextAttempt - reinitializationCounter > reinitializationLimit) { + LOG.error("Too many JVpp reinitialization attempts. Unable to reinitialize JVpp in {} attempts. Giving up", + reinitializationLimit); + throw new IllegalStateException("Too many JVpp reinitialization attempts. Unable to reinitialize JVpp in " + + reinitializationLimit + " attempts. Giving up"); + } + + final ConfigRegistryJMXClient cfgRegistryClient = + ConfigRegistryJMXClient.createWithoutNotifications(ManagementFactory.getPlatformMBeanServer()); + + final ObjectName objectName = cfgRegistryClient.beginConfig(); + final ConfigTransactionJMXClient txClient = cfgRegistryClient.getConfigTransactionClient(objectName); + + final ObjectName jvppOn; + try { + final String attributeName = VppJvppImplModule.descriptionJmxAttribute.getAttributeName(); + final String factoryName = VppJvppImplModuleFactory.NAME; + jvppOn = txClient.lookupConfigBean(factoryName, "vpp-jvpp"); + + // Change configuration attribute of JVpp to trigger full reinitialization here using config subsystem + // TODO improve this when switching from karaf in planned minimal distribution + txClient.setAttribute(jvppOn, attributeName, new Attribute(attributeName, + Long.toString(nextAttempt))); + + txClient.validateConfig(); + cfgRegistryClient.commitConfig(txClient.getObjectName()); + LOG.info("JVpp reinitialized successfully"); + } catch (InstanceNotFoundException | ValidationException e) { + LOG.error("Unable to reinitialize JVpp. Honeycomb will not work properly from now on.", e); + throw new IllegalStateException("Unable to find jvpp instance in config subsystem. " + + "Unable to reinitialize JVpp", e); + } catch (ConflictingVersionException e) { + LOG.debug("Conflict changes occurred, retrying", e); + // Just retry until there's no conflicting change in progress + reinitializeJVpp(nextAttempt); + } + + reinitializationCounter = nextAttempt; + } } diff --git a/v3po/v3po2vpp/src/main/yang/v3po2vpp.yang b/v3po/v3po2vpp/src/main/yang/v3po2vpp.yang index f706ec60d..5ccac8e94 100644 --- a/v3po/v3po2vpp/src/main/yang/v3po2vpp.yang +++ b/v3po/v3po2vpp/src/main/yang/v3po2vpp.yang @@ -9,6 +9,7 @@ module v3po2vpp { import vpp-cfg-init { prefix init; revision-date "2016-04-07"; } import opendaylight-md-sal-binding { prefix md-sal-binding; revision-date 2013-10-28;} import vpp-util { prefix vpp-u; revision-date 2016-04-06; } + import threadpool {prefix th;} description "This module contains reads and writers for v3po yang model"; @@ -53,6 +54,17 @@ module v3po2vpp { } } } + + container keepalive-executor { + uses config:service-ref { + refine type { + mandatory true; + config:required-identity th:scheduled-threadpool; + } + } + + description "Used to schedule keepalives"; + } } } diff --git a/v3po/v3po2vpp/src/test/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizerTest.java b/v3po/v3po2vpp/src/test/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizerTest.java index 03b923102..0ddef2248 100644 --- a/v3po/v3po2vpp/src/test/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizerTest.java +++ b/v3po/v3po2vpp/src/test/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizerTest.java @@ -24,7 +24,6 @@ import static org.mockito.Mockito.when; import io.fd.honeycomb.v3po.translate.spi.read.ChildReaderCustomizer; import io.fd.honeycomb.v3po.translate.v3po.test.ChildReaderCustomizerTest; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; import org.junit.Test; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.VppStateBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.vpp.state.Version; @@ -54,20 +53,17 @@ public class VersionCustomizerTest extends ChildReaderCustomizerTest replyCS = mock(CompletionStage.class); - final CompletableFuture replyFuture = mock(CompletableFuture.class); - when(replyCS.toCompletableFuture()).thenReturn(replyFuture); + final CompletableFuture replyFuture = new CompletableFuture<>(); final ShowVersionReply reply = new ShowVersionReply(); reply.retval = 0; reply.version = new byte[]{}; reply.program = new byte[]{}; reply.buildDate = new byte[]{}; reply.buildDirectory = new byte[]{}; - when(replyFuture.get()).thenReturn(reply); - when(api.showVersion(any(ShowVersion.class))).thenReturn(replyCS); + replyFuture.complete(reply); + when(api.showVersion(any(ShowVersion.class))).thenReturn(replyFuture); getCustomizer().readCurrentAttributes(InstanceIdentifier.create(Version.class), new VersionBuilder(), ctx); - verify(api).showVersion(any(ShowVersion.class)); } } \ No newline at end of file diff --git a/v3po/v3po2vpp/src/test/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VppStateTest.java b/v3po/v3po2vpp/src/test/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VppStateTest.java index 3d75d09e2..5ac108560 100644 --- a/v3po/v3po2vpp/src/test/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VppStateTest.java +++ b/v3po/v3po2vpp/src/test/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VppStateTest.java @@ -113,17 +113,16 @@ public class VppStateTest { } private void whenShowVersionThenReturn(int retval, Version version) throws ExecutionException, InterruptedException { - final CompletionStage replyCS = mock(CompletionStage.class); - final CompletableFuture replyFuture = mock(CompletableFuture.class); - when(replyCS.toCompletableFuture()).thenReturn(replyFuture); + final CompletableFuture replyFuture = new CompletableFuture<>(); final ShowVersionReply reply = new ShowVersionReply(); reply.retval = 0; // success reply.buildDate = version.getBuildDate().getBytes(); reply.program = version.getName().getBytes(); reply.version = version.getBranch().getBytes(); reply.buildDirectory = version.getBuildDirectory().getBytes(); - when(replyFuture.get()).thenReturn(reply); - when(api.showVersion(any(ShowVersion.class))).thenReturn(replyCS); + + replyFuture.complete(reply); + when(api.showVersion(any(ShowVersion.class))).thenReturn(replyFuture); } private void whenL2FibTableDumpThenReturn(final List entryList) throws ExecutionException, InterruptedException { @@ -305,7 +304,7 @@ public class VppStateTest { @Test(expected = IllegalArgumentException.class) public void testReadBridgeDomainNotExisting() throws Exception { doReturn(Optional.absent()).when(mappingContext).read(getMappingIid("NOT EXISTING", "bd-test-instance")); - + final Optional read = readerRegistry.read(InstanceIdentifier.create(VppState.class).child(BridgeDomains.class).child( BridgeDomain.class, new BridgeDomainKey("NOT EXISTING")), ctx); -- cgit 1.2.3-korg