HONEYCOMB-61: Detect VPP disconnect using keepalives

Change-Id: Ic664dbf452504d0fff97e8c766d735d9c5d95c72
Signed-off-by: Maros Marsalek <mmarsale@cisco.com>

2 files changed, 86 insertions, 7 deletions

diff --git a/v3po/v3po2vpp/src/main/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizer.java b/v3po/v3po2vpp/src/main/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizer.java
index ad181a701..8122fd245 100644
--- a/v3po/v3po2vpp/src/main/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizer.java
+++ b/v3po/v3po2vpp/src/main/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizer.java
@@ -21,6 +21,7 @@ import io.fd.honeycomb.v3po.translate.read.ReadFailedException;
 import io.fd.honeycomb.v3po.translate.spi.read.ChildReaderCustomizer;
 import io.fd.honeycomb.v3po.translate.v3po.util.FutureJVppCustomizer;
 import io.fd.honeycomb.v3po.translate.v3po.util.TranslateUtils;
+import java.util.concurrent.CompletionStage;
 import javax.annotation.Nonnull;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.VppStateBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.vpp.state.Version;
@@ -36,6 +37,11 @@ public final class VersionCustomizer
     extends FutureJVppCustomizer
     implements ChildReaderCustomizer<Version, VersionBuilder> {
 
+    /**
+     * Default timeout for executing version read
+     */
+    private static final int DEFAULT_TIMEOUT_IN_SECONDS = 30;
+
     public VersionCustomizer(@Nonnull final FutureJVpp futureJVpp) {
         super(futureJVpp);
     }
@@ -54,13 +60,11 @@ public final class VersionCustomizer
     @Override
     public void readCurrentAttributes(@Nonnull InstanceIdentifier<Version> id, @Nonnull final VersionBuilder builder,
                                       @Nonnull final ReadContext context) throws ReadFailedException {
+        // Execute with timeout
+        final CompletionStage<ShowVersionReply> showVersionFuture = getFutureJVpp().showVersion(new ShowVersion());
+        final ShowVersionReply reply = TranslateUtils.getReply(showVersionFuture.toCompletableFuture(), id,
+            DEFAULT_TIMEOUT_IN_SECONDS);
 
-        ShowVersionReply reply;
-        try {
-            reply = getFutureJVpp().showVersion(new ShowVersion()).toCompletableFuture().get();
-        } catch (Exception e) {
-            throw new ReadFailedException(id, e);
-        }
         builder.setBranch(TranslateUtils.toString(reply.version));
         builder.setName(TranslateUtils.toString(reply.program));
         builder.setBuildDate(TranslateUtils.toString(reply.buildDate));
diff --git a/v3po/v3po2vpp/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/v3po2vpp/rev160406/VppStateHoneycombReaderModule.java b/v3po/v3po2vpp/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/v3po2vpp/rev160406/VppStateHoneycombReaderModule.java
index 9a0165577..32e5dabc0 100644
--- a/v3po/v3po2vpp/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/v3po2vpp/rev160406/VppStateHoneycombReaderModule.java
+++ b/v3po/v3po2vpp/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/v3po2vpp/rev160406/VppStateHoneycombReaderModule.java
@@ -4,14 +4,26 @@ import io.fd.honeycomb.v3po.translate.impl.read.CompositeChildReader;
 import io.fd.honeycomb.v3po.translate.impl.read.CompositeListReader;
 import io.fd.honeycomb.v3po.translate.impl.read.CompositeRootReader;
 import io.fd.honeycomb.v3po.translate.read.ChildReader;
+import io.fd.honeycomb.v3po.translate.util.KeepaliveReaderWrapper;
 import io.fd.honeycomb.v3po.translate.util.RWUtils;
 import io.fd.honeycomb.v3po.translate.util.read.CloseableReader;
 import io.fd.honeycomb.v3po.translate.util.read.ReflexiveChildReaderCustomizer;
 import io.fd.honeycomb.v3po.translate.util.read.ReflexiveRootReaderCustomizer;
+import io.fd.honeycomb.v3po.translate.v3po.util.ReadTimeoutException;
 import io.fd.honeycomb.v3po.translate.v3po.vppstate.BridgeDomainCustomizer;
 import io.fd.honeycomb.v3po.translate.v3po.vppstate.VersionCustomizer;
+import java.lang.management.ManagementFactory;
 import java.util.ArrayList;
 import java.util.List;
+import javax.management.Attribute;
+import javax.management.InstanceNotFoundException;
+import javax.management.ObjectName;
+import org.opendaylight.controller.config.api.ConflictingVersionException;
+import org.opendaylight.controller.config.api.ValidationException;
+import org.opendaylight.controller.config.util.ConfigRegistryJMXClient;
+import org.opendaylight.controller.config.util.ConfigTransactionJMXClient;
+import org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.vpp.jvpp.cfg.rev160406.VppJvppImplModule;
+import org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.vpp.jvpp.cfg.rev160406.VppJvppImplModuleFactory;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.VppState;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.VppStateBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.vpp.state.BridgeDomains;
@@ -22,8 +34,13 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.vpp.state.bridge.domains.BridgeDomainKey;
 import org.opendaylight.yangtools.yang.binding.ChildOf;
 import org.openvpp.jvpp.future.FutureJVpp;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class VppStateHoneycombReaderModule extends org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.v3po2vpp.rev160406.AbstractVppStateHoneycombReaderModule {
+
+    private static final Logger LOG = LoggerFactory.getLogger(VppStateHoneycombReaderModule.class);
+
     public VppStateHoneycombReaderModule(org.opendaylight.controller.config.api.ModuleIdentifier identifier, org.opendaylight.controller.config.api.DependencyResolver dependencyResolver) {
         super(identifier, dependencyResolver);
     }
@@ -41,7 +58,13 @@ public class VppStateHoneycombReaderModule extends org.opendaylight.yang.gen.v1.
     public java.lang.AutoCloseable createInstance() {
         final FutureJVpp vppApi = getVppJvppDependency();
 
-        final ChildReader<Version> versionReader = new CompositeChildReader<>(Version.class, new VersionCustomizer(vppApi));
+        ChildReader<Version> versionReader = new CompositeChildReader<>(Version.class, new VersionCustomizer(vppApi));
+        // Wrap with keepalive reader to detect connection issues
+        // TODO keepalive reader wrapper relies on VersionReaderCustomizer (to perform timeout on reads)
+        // Once readers+customizers are asynchronous, pull the timeout to keepalive executor so that keepalive wrapper
+        // is truly generic
+        versionReader = new KeepaliveReaderWrapper<>(versionReader, getKeepaliveExecutorDependency().getExecutor(),
+            ReadTimeoutException.class, 30, () -> reinitializeJVpp(reinitializationCounter));
 
         final CompositeListReader<BridgeDomain, BridgeDomainKey, BridgeDomainBuilder> bridgeDomainReader =
             new CompositeListReader<>(BridgeDomain.class, new BridgeDomainCustomizer(vppApi,
@@ -62,4 +85,56 @@ public class VppStateHoneycombReaderModule extends org.opendaylight.yang.gen.v1.
             RWUtils.emptyAugReaderList(),
             new ReflexiveRootReaderCustomizer<>(VppStateBuilder.class)));
     }
+
+    private static long reinitializationCounter;
+    private static final long reinitializationLimit = 10;
+
+    /**
+     * In case we detect connection issues with VPP, reinitialize JVpp
+     */
+    private void reinitializeJVpp(final long currentAttempt) {
+        // FIXME https://jira.fd.io/browse/HONEYCOMB-78 This code correctly re-initializes all the components
+        // starting with jvpp, but jvpp reconnect fails. Test in a JVpp test and then from C
+        LOG.info("Reinitializing JVpp, attempt: {}", currentAttempt);
+
+        final long nextAttempt = currentAttempt + 1;
+        if (nextAttempt - reinitializationCounter > reinitializationLimit) {
+            LOG.error("Too many JVpp reinitialization attempts. Unable to reinitialize JVpp in {} attempts. Giving up",
+                reinitializationLimit);
+            throw new IllegalStateException("Too many JVpp reinitialization attempts. Unable to reinitialize JVpp in "
+                + reinitializationLimit + " attempts. Giving up");
+        }
+
+        final ConfigRegistryJMXClient cfgRegistryClient =
+            ConfigRegistryJMXClient.createWithoutNotifications(ManagementFactory.getPlatformMBeanServer());
+
+        final ObjectName objectName = cfgRegistryClient.beginConfig();
+        final ConfigTransactionJMXClient txClient = cfgRegistryClient.getConfigTransactionClient(objectName);
+
+        final ObjectName jvppOn;
+        try {
+            final String attributeName = VppJvppImplModule.descriptionJmxAttribute.getAttributeName();
+            final String factoryName = VppJvppImplModuleFactory.NAME;
+            jvppOn = txClient.lookupConfigBean(factoryName, "vpp-jvpp");
+
+            // Change configuration attribute of JVpp to trigger full reinitialization here using config subsystem
+            // TODO improve this when switching from karaf in planned minimal distribution
+            txClient.setAttribute(jvppOn, attributeName, new Attribute(attributeName,
+                Long.toString(nextAttempt)));
+
+            txClient.validateConfig();
+            cfgRegistryClient.commitConfig(txClient.getObjectName());
+            LOG.info("JVpp reinitialized successfully");
+        } catch (InstanceNotFoundException | ValidationException e) {
+            LOG.error("Unable to reinitialize JVpp. Honeycomb will not work properly from now on.", e);
+            throw new IllegalStateException("Unable to find jvpp instance in config subsystem. " +
+                "Unable to reinitialize JVpp", e);
+        } catch (ConflictingVersionException e) {
+            LOG.debug("Conflict changes occurred, retrying", e);
+            // Just retry until there's no conflicting change in progress
+            reinitializeJVpp(nextAttempt);
+        }
+
+        reinitializationCounter = nextAttempt;
+    }
 }