HONEYCOMB-61: Detect VPP disconnect using keepalives

Change-Id: Ic664dbf452504d0fff97e8c766d735d9c5d95c72
Signed-off-by: Maros Marsalek <mmarsale@cisco.com>

4 files changed, 108 insertions, 7 deletions

diff --git a/v3po/v3po2vpp/src/main/config/default-config.xml b/v3po/v3po2vpp/src/main/config/default-config.xml
index be45a291b..19226d4fc 100644
--- a/v3po/v3po2vpp/src/main/config/default-config.xml
+++ b/v3po/v3po2vpp/src/main/config/default-config.xml
@@ -23,6 +23,9 @@
         <capability>urn:honeycomb:params:xml:ns:yang:v3po2vpp?module=v3po2vpp&amp;revision=2016-04-06</capability>
         <capability>urn:honeycomb:params:xml:ns:yang:data:api?module=data-api&amp;revision=2016-04-11</capability>
         <capability>urn:honeycomb:params:xml:ns:yang:vpp:util?module=vpp-util&amp;revision=2016-04-06</capability>
+        <capability>
+            urn:opendaylight:params:xml:ns:yang:controller:threadpool:impl:scheduled?module=threadpool-impl-scheduled&amp;revision=2013-12-01
+        </capability>
     </required-capabilities>
     <configuration>
 
@@ -54,6 +57,13 @@
                         <type xmlns:prefix="urn:honeycomb:params:xml:ns:yang:vpp:util">prefix:naming-context</type>
                         <name>bridge-domain-context</name>
                     </bridge-domain-context-vpp-state>
+                    <!-- Reuse netconf's scheduled executor for keepalives -->
+                    <keepalive-executor>
+                        <type xmlns:prefix="urn:opendaylight:params:xml:ns:yang:controller:threadpool">
+                            prefix:threadpool
+                        </type>
+                        <name>global-netconf-ssh-scheduled-executor</name>
+                    </keepalive-executor>
                 </module>
                 <module>
                     <type xmlns:prefix="urn:honeycomb:params:xml:ns:yang:v3po2vpp">prefix:interfaces-state-honeycomb-reader</type>
diff --git a/v3po/v3po2vpp/src/main/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizer.java b/v3po/v3po2vpp/src/main/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizer.java
index ad181a701..8122fd245 100644
--- a/v3po/v3po2vpp/src/main/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizer.java
+++ b/v3po/v3po2vpp/src/main/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizer.java
@@ -21,6 +21,7 @@ import io.fd.honeycomb.v3po.translate.read.ReadFailedException;
 import io.fd.honeycomb.v3po.translate.spi.read.ChildReaderCustomizer;
 import io.fd.honeycomb.v3po.translate.v3po.util.FutureJVppCustomizer;
 import io.fd.honeycomb.v3po.translate.v3po.util.TranslateUtils;
+import java.util.concurrent.CompletionStage;
 import javax.annotation.Nonnull;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.VppStateBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.vpp.state.Version;
@@ -36,6 +37,11 @@ public final class VersionCustomizer
     extends FutureJVppCustomizer
     implements ChildReaderCustomizer<Version, VersionBuilder> {
 
+    /**
+     * Default timeout for executing version read
+     */
+    private static final int DEFAULT_TIMEOUT_IN_SECONDS = 30;
+
     public VersionCustomizer(@Nonnull final FutureJVpp futureJVpp) {
         super(futureJVpp);
     }
@@ -54,13 +60,11 @@ public final class VersionCustomizer
     @Override
     public void readCurrentAttributes(@Nonnull InstanceIdentifier<Version> id, @Nonnull final VersionBuilder builder,
                                       @Nonnull final ReadContext context) throws ReadFailedException {
+        // Execute with timeout
+        final CompletionStage<ShowVersionReply> showVersionFuture = getFutureJVpp().showVersion(new ShowVersion());
+        final ShowVersionReply reply = TranslateUtils.getReply(showVersionFuture.toCompletableFuture(), id,
+            DEFAULT_TIMEOUT_IN_SECONDS);
 
-        ShowVersionReply reply;
-        try {
-            reply = getFutureJVpp().showVersion(new ShowVersion()).toCompletableFuture().get();
-        } catch (Exception e) {
-            throw new ReadFailedException(id, e);
-        }
         builder.setBranch(TranslateUtils.toString(reply.version));
         builder.setName(TranslateUtils.toString(reply.program));
         builder.setBuildDate(TranslateUtils.toString(reply.buildDate));
diff --git a/v3po/v3po2vpp/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/v3po2vpp/rev160406/VppStateHoneycombReaderModule.java b/v3po/v3po2vpp/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/v3po2vpp/rev160406/VppStateHoneycombReaderModule.java
index 9a0165577..32e5dabc0 100644
--- a/v3po/v3po2vpp/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/v3po2vpp/rev160406/VppStateHoneycombReaderModule.java
+++ b/v3po/v3po2vpp/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/v3po2vpp/rev160406/VppStateHoneycombReaderModule.java
@@ -4,14 +4,26 @@ import io.fd.honeycomb.v3po.translate.impl.read.CompositeChildReader;
 import io.fd.honeycomb.v3po.translate.impl.read.CompositeListReader;
 import io.fd.honeycomb.v3po.translate.impl.read.CompositeRootReader;
 import io.fd.honeycomb.v3po.translate.read.ChildReader;
+import io.fd.honeycomb.v3po.translate.util.KeepaliveReaderWrapper;
 import io.fd.honeycomb.v3po.translate.util.RWUtils;
 import io.fd.honeycomb.v3po.translate.util.read.CloseableReader;
 import io.fd.honeycomb.v3po.translate.util.read.ReflexiveChildReaderCustomizer;
 import io.fd.honeycomb.v3po.translate.util.read.ReflexiveRootReaderCustomizer;
+import io.fd.honeycomb.v3po.translate.v3po.util.ReadTimeoutException;
 import io.fd.honeycomb.v3po.translate.v3po.vppstate.BridgeDomainCustomizer;
 import io.fd.honeycomb.v3po.translate.v3po.vppstate.VersionCustomizer;
+import java.lang.management.ManagementFactory;
 import java.util.ArrayList;
 import java.util.List;
+import javax.management.Attribute;
+import javax.management.InstanceNotFoundException;
+import javax.management.ObjectName;
+import org.opendaylight.controller.config.api.ConflictingVersionException;
+import org.opendaylight.controller.config.api.ValidationException;
+import org.opendaylight.controller.config.util.ConfigRegistryJMXClient;
+import org.opendaylight.controller.config.util.ConfigTransactionJMXClient;
+import org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.vpp.jvpp.cfg.rev160406.VppJvppImplModule;
+import org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.vpp.jvpp.cfg.rev160406.VppJvppImplModuleFactory;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.VppState;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.VppStateBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.vpp.state.BridgeDomains;
@@ -22,8 +34,13 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.vpp.state.bridge.domains.BridgeDomainKey;
 import org.opendaylight.yangtools.yang.binding.ChildOf;
 import org.openvpp.jvpp.future.FutureJVpp;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class VppStateHoneycombReaderModule extends org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.v3po2vpp.rev160406.AbstractVppStateHoneycombReaderModule {
+
+    private static final Logger LOG = LoggerFactory.getLogger(VppStateHoneycombReaderModule.class);
+
     public VppStateHoneycombReaderModule(org.opendaylight.controller.config.api.ModuleIdentifier identifier, org.opendaylight.controller.config.api.DependencyResolver dependencyResolver) {
         super(identifier, dependencyResolver);
     }
@@ -41,7 +58,13 @@ public class VppStateHoneycombReaderModule extends org.opendaylight.yang.gen.v1.
     public java.lang.AutoCloseable createInstance() {
         final FutureJVpp vppApi = getVppJvppDependency();
 
-        final ChildReader<Version> versionReader = new CompositeChildReader<>(Version.class, new VersionCustomizer(vppApi));
+        ChildReader<Version> versionReader = new CompositeChildReader<>(Version.class, new VersionCustomizer(vppApi));
+        // Wrap with keepalive reader to detect connection issues
+        // TODO keepalive reader wrapper relies on VersionReaderCustomizer (to perform timeout on reads)
+        // Once readers+customizers are asynchronous, pull the timeout to keepalive executor so that keepalive wrapper
+        // is truly generic
+        versionReader = new KeepaliveReaderWrapper<>(versionReader, getKeepaliveExecutorDependency().getExecutor(),
+            ReadTimeoutException.class, 30, () -> reinitializeJVpp(reinitializationCounter));
 
         final CompositeListReader<BridgeDomain, BridgeDomainKey, BridgeDomainBuilder> bridgeDomainReader =
             new CompositeListReader<>(BridgeDomain.class, new BridgeDomainCustomizer(vppApi,
@@ -62,4 +85,56 @@ public class VppStateHoneycombReaderModule extends org.opendaylight.yang.gen.v1.
             RWUtils.emptyAugReaderList(),
             new ReflexiveRootReaderCustomizer<>(VppStateBuilder.class)));
     }
+
+    private static long reinitializationCounter;
+    private static final long reinitializationLimit = 10;
+
+    /**
+     * In case we detect connection issues with VPP, reinitialize JVpp
+     */
+    private void reinitializeJVpp(final long currentAttempt) {
+        // FIXME https://jira.fd.io/browse/HONEYCOMB-78 This code correctly re-initializes all the components
+        // starting with jvpp, but jvpp reconnect fails. Test in a JVpp test and then from C
+        LOG.info("Reinitializing JVpp, attempt: {}", currentAttempt);
+
+        final long nextAttempt = currentAttempt + 1;
+        if (nextAttempt - reinitializationCounter > reinitializationLimit) {
+            LOG.error("Too many JVpp reinitialization attempts. Unable to reinitialize JVpp in {} attempts. Giving up",
+                reinitializationLimit);
+            throw new IllegalStateException("Too many JVpp reinitialization attempts. Unable to reinitialize JVpp in "
+                + reinitializationLimit + " attempts. Giving up");
+        }
+
+        final ConfigRegistryJMXClient cfgRegistryClient =
+            ConfigRegistryJMXClient.createWithoutNotifications(ManagementFactory.getPlatformMBeanServer());
+
+        final ObjectName objectName = cfgRegistryClient.beginConfig();
+        final ConfigTransactionJMXClient txClient = cfgRegistryClient.getConfigTransactionClient(objectName);
+
+        final ObjectName jvppOn;
+        try {
+            final String attributeName = VppJvppImplModule.descriptionJmxAttribute.getAttributeName();
+            final String factoryName = VppJvppImplModuleFactory.NAME;
+            jvppOn = txClient.lookupConfigBean(factoryName, "vpp-jvpp");
+
+            // Change configuration attribute of JVpp to trigger full reinitialization here using config subsystem
+            // TODO improve this when switching from karaf in planned minimal distribution
+            txClient.setAttribute(jvppOn, attributeName, new Attribute(attributeName,
+                Long.toString(nextAttempt)));
+
+            txClient.validateConfig();
+            cfgRegistryClient.commitConfig(txClient.getObjectName());
+            LOG.info("JVpp reinitialized successfully");
+        } catch (InstanceNotFoundException | ValidationException e) {
+            LOG.error("Unable to reinitialize JVpp. Honeycomb will not work properly from now on.", e);
+            throw new IllegalStateException("Unable to find jvpp instance in config subsystem. " +
+                "Unable to reinitialize JVpp", e);
+        } catch (ConflictingVersionException e) {
+            LOG.debug("Conflict changes occurred, retrying", e);
+            // Just retry until there's no conflicting change in progress
+            reinitializeJVpp(nextAttempt);
+        }
+
+        reinitializationCounter = nextAttempt;
+    }
 }
diff --git a/v3po/v3po2vpp/src/main/yang/v3po2vpp.yang b/v3po/v3po2vpp/src/main/yang/v3po2vpp.yang
index f706ec60d..5ccac8e94 100644
--- a/v3po/v3po2vpp/src/main/yang/v3po2vpp.yang
+++ b/v3po/v3po2vpp/src/main/yang/v3po2vpp.yang
@@ -9,6 +9,7 @@ module v3po2vpp {
     import vpp-cfg-init { prefix init; revision-date "2016-04-07"; }
     import opendaylight-md-sal-binding { prefix md-sal-binding; revision-date 2013-10-28;}
     import vpp-util { prefix vpp-u; revision-date 2016-04-06; }
+    import threadpool {prefix th;}
 
     description
         "This module contains reads and writers for v3po yang model";
@@ -53,6 +54,17 @@ module v3po2vpp {
                     }
                 }
             }
+
+            container keepalive-executor {
+                uses config:service-ref {
+                    refine type {
+                        mandatory true;
+                        config:required-identity th:scheduled-threadpool;
+                    }
+                }
+
+                description "Used to schedule keepalives";
+            }
         }
     }