HONEYCOMB-61: Detect VPP disconnect using keepalives

Change-Id: Ic664dbf452504d0fff97e8c766d735d9c5d95c72
Signed-off-by: Maros Marsalek <mmarsale@cisco.com>

7 files changed, 130 insertions, 20 deletions

diff --git a/v3po/v3po2vpp/pom.xml b/v3po/v3po2vpp/pom.xml
index 113511da1..d02807c9d 100644
--- a/v3po/v3po2vpp/pom.xml
+++ b/v3po/v3po2vpp/pom.xml
@@ -49,6 +49,20 @@
             <artifactId>v3po-api</artifactId>
             <version>${project.version}</version>
         </dependency>
+
+        <!-- Keepalives -->
+        <dependency>
+            <groupId>org.opendaylight.controller</groupId>
+            <artifactId>config-util</artifactId>
+            <version>0.4.2-Beryllium-SR2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.opendaylight.controller</groupId>
+            <artifactId>threadpool-config-api</artifactId>
+            <version>0.4.2-Beryllium-SR2</version>
+        </dependency>
+        <!---->
+
         <dependency>
             <groupId>${project.groupId}</groupId>
             <artifactId>translate-impl</artifactId>
diff --git a/v3po/v3po2vpp/src/main/config/default-config.xml b/v3po/v3po2vpp/src/main/config/default-config.xml
index be45a291b..19226d4fc 100644
--- a/v3po/v3po2vpp/src/main/config/default-config.xml
+++ b/v3po/v3po2vpp/src/main/config/default-config.xml
@@ -23,6 +23,9 @@
         <capability>urn:honeycomb:params:xml:ns:yang:v3po2vpp?module=v3po2vpp&amp;revision=2016-04-06</capability>
         <capability>urn:honeycomb:params:xml:ns:yang:data:api?module=data-api&amp;revision=2016-04-11</capability>
         <capability>urn:honeycomb:params:xml:ns:yang:vpp:util?module=vpp-util&amp;revision=2016-04-06</capability>
+        <capability>
+            urn:opendaylight:params:xml:ns:yang:controller:threadpool:impl:scheduled?module=threadpool-impl-scheduled&amp;revision=2013-12-01
+        </capability>
     </required-capabilities>
     <configuration>
 
@@ -54,6 +57,13 @@
                         <type xmlns:prefix="urn:honeycomb:params:xml:ns:yang:vpp:util">prefix:naming-context</type>
                         <name>bridge-domain-context</name>
                     </bridge-domain-context-vpp-state>
+                    <!-- Reuse netconf's scheduled executor for keepalives -->
+                    <keepalive-executor>
+                        <type xmlns:prefix="urn:opendaylight:params:xml:ns:yang:controller:threadpool">
+                            prefix:threadpool
+                        </type>
+                        <name>global-netconf-ssh-scheduled-executor</name>
+                    </keepalive-executor>
                 </module>
                 <module>
                     <type xmlns:prefix="urn:honeycomb:params:xml:ns:yang:v3po2vpp">prefix:interfaces-state-honeycomb-reader</type>
diff --git a/v3po/v3po2vpp/src/main/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizer.java b/v3po/v3po2vpp/src/main/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizer.java
index ad181a701..8122fd245 100644
--- a/v3po/v3po2vpp/src/main/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizer.java
+++ b/v3po/v3po2vpp/src/main/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizer.java
@@ -21,6 +21,7 @@ import io.fd.honeycomb.v3po.translate.read.ReadFailedException;
 import io.fd.honeycomb.v3po.translate.spi.read.ChildReaderCustomizer;
 import io.fd.honeycomb.v3po.translate.v3po.util.FutureJVppCustomizer;
 import io.fd.honeycomb.v3po.translate.v3po.util.TranslateUtils;
+import java.util.concurrent.CompletionStage;
 import javax.annotation.Nonnull;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.VppStateBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.vpp.state.Version;
@@ -36,6 +37,11 @@ public final class VersionCustomizer
     extends FutureJVppCustomizer
     implements ChildReaderCustomizer<Version, VersionBuilder> {
 
+    /**
+     * Default timeout for executing version read
+     */
+    private static final int DEFAULT_TIMEOUT_IN_SECONDS = 30;
+
     public VersionCustomizer(@Nonnull final FutureJVpp futureJVpp) {
         super(futureJVpp);
     }
@@ -54,13 +60,11 @@ public final class VersionCustomizer
     @Override
     public void readCurrentAttributes(@Nonnull InstanceIdentifier<Version> id, @Nonnull final VersionBuilder builder,
                                       @Nonnull final ReadContext context) throws ReadFailedException {
+        // Execute with timeout
+        final CompletionStage<ShowVersionReply> showVersionFuture = getFutureJVpp().showVersion(new ShowVersion());
+        final ShowVersionReply reply = TranslateUtils.getReply(showVersionFuture.toCompletableFuture(), id,
+            DEFAULT_TIMEOUT_IN_SECONDS);
 
-        ShowVersionReply reply;
-        try {
-            reply = getFutureJVpp().showVersion(new ShowVersion()).toCompletableFuture().get();
-        } catch (Exception e) {
-            throw new ReadFailedException(id, e);
-        }
         builder.setBranch(TranslateUtils.toString(reply.version));
         builder.setName(TranslateUtils.toString(reply.program));
         builder.setBuildDate(TranslateUtils.toString(reply.buildDate));
diff --git a/v3po/v3po2vpp/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/v3po2vpp/rev160406/VppStateHoneycombReaderModule.java b/v3po/v3po2vpp/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/v3po2vpp/rev160406/VppStateHoneycombReaderModule.java
index 9a0165577..32e5dabc0 100644
--- a/v3po/v3po2vpp/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/v3po2vpp/rev160406/VppStateHoneycombReaderModule.java
+++ b/v3po/v3po2vpp/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/v3po2vpp/rev160406/VppStateHoneycombReaderModule.java
@@ -4,14 +4,26 @@ import io.fd.honeycomb.v3po.translate.impl.read.CompositeChildReader;
 import io.fd.honeycomb.v3po.translate.impl.read.CompositeListReader;
 import io.fd.honeycomb.v3po.translate.impl.read.CompositeRootReader;
 import io.fd.honeycomb.v3po.translate.read.ChildReader;
+import io.fd.honeycomb.v3po.translate.util.KeepaliveReaderWrapper;
 import io.fd.honeycomb.v3po.translate.util.RWUtils;
 import io.fd.honeycomb.v3po.translate.util.read.CloseableReader;
 import io.fd.honeycomb.v3po.translate.util.read.ReflexiveChildReaderCustomizer;
 import io.fd.honeycomb.v3po.translate.util.read.ReflexiveRootReaderCustomizer;
+import io.fd.honeycomb.v3po.translate.v3po.util.ReadTimeoutException;
 import io.fd.honeycomb.v3po.translate.v3po.vppstate.BridgeDomainCustomizer;
 import io.fd.honeycomb.v3po.translate.v3po.vppstate.VersionCustomizer;
+import java.lang.management.ManagementFactory;
 import java.util.ArrayList;
 import java.util.List;
+import javax.management.Attribute;
+import javax.management.InstanceNotFoundException;
+import javax.management.ObjectName;
+import org.opendaylight.controller.config.api.ConflictingVersionException;
+import org.opendaylight.controller.config.api.ValidationException;
+import org.opendaylight.controller.config.util.ConfigRegistryJMXClient;
+import org.opendaylight.controller.config.util.ConfigTransactionJMXClient;
+import org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.vpp.jvpp.cfg.rev160406.VppJvppImplModule;
+import org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.vpp.jvpp.cfg.rev160406.VppJvppImplModuleFactory;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.VppState;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.VppStateBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.vpp.state.BridgeDomains;
@@ -22,8 +34,13 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.vpp.state.bridge.domains.BridgeDomainKey;
 import org.opendaylight.yangtools.yang.binding.ChildOf;
 import org.openvpp.jvpp.future.FutureJVpp;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class VppStateHoneycombReaderModule extends org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.v3po2vpp.rev160406.AbstractVppStateHoneycombReaderModule {
+
+    private static final Logger LOG = LoggerFactory.getLogger(VppStateHoneycombReaderModule.class);
+
     public VppStateHoneycombReaderModule(org.opendaylight.controller.config.api.ModuleIdentifier identifier, org.opendaylight.controller.config.api.DependencyResolver dependencyResolver) {
         super(identifier, dependencyResolver);
     }
@@ -41,7 +58,13 @@ public class VppStateHoneycombReaderModule extends org.opendaylight.yang.gen.v1.
     public java.lang.AutoCloseable createInstance() {
         final FutureJVpp vppApi = getVppJvppDependency();
 
-        final ChildReader<Version> versionReader = new CompositeChildReader<>(Version.class, new VersionCustomizer(vppApi));
+        ChildReader<Version> versionReader = new CompositeChildReader<>(Version.class, new VersionCustomizer(vppApi));
+        // Wrap with keepalive reader to detect connection issues
+        // TODO keepalive reader wrapper relies on VersionReaderCustomizer (to perform timeout on reads)
+        // Once readers+customizers are asynchronous, pull the timeout to keepalive executor so that keepalive wrapper
+        // is truly generic
+        versionReader = new KeepaliveReaderWrapper<>(versionReader, getKeepaliveExecutorDependency().getExecutor(),
+            ReadTimeoutException.class, 30, () -> reinitializeJVpp(reinitializationCounter));
 
         final CompositeListReader<BridgeDomain, BridgeDomainKey, BridgeDomainBuilder> bridgeDomainReader =
             new CompositeListReader<>(BridgeDomain.class, new BridgeDomainCustomizer(vppApi,
@@ -62,4 +85,56 @@ public class VppStateHoneycombReaderModule extends org.opendaylight.yang.gen.v1.
             RWUtils.emptyAugReaderList(),
             new ReflexiveRootReaderCustomizer<>(VppStateBuilder.class)));
     }
+
+    private static long reinitializationCounter;
+    private static final long reinitializationLimit = 10;
+
+    /**
+     * In case we detect connection issues with VPP, reinitialize JVpp
+     */
+    private void reinitializeJVpp(final long currentAttempt) {
+        // FIXME https://jira.fd.io/browse/HONEYCOMB-78 This code correctly re-initializes all the components
+        // starting with jvpp, but jvpp reconnect fails. Test in a JVpp test and then from C
+        LOG.info("Reinitializing JVpp, attempt: {}", currentAttempt);
+
+        final long nextAttempt = currentAttempt + 1;
+        if (nextAttempt - reinitializationCounter > reinitializationLimit) {
+            LOG.error("Too many JVpp reinitialization attempts. Unable to reinitialize JVpp in {} attempts. Giving up",
+                reinitializationLimit);
+            throw new IllegalStateException("Too many JVpp reinitialization attempts. Unable to reinitialize JVpp in "
+                + reinitializationLimit + " attempts. Giving up");
+        }
+
+        final ConfigRegistryJMXClient cfgRegistryClient =
+            ConfigRegistryJMXClient.createWithoutNotifications(ManagementFactory.getPlatformMBeanServer());
+
+        final ObjectName objectName = cfgRegistryClient.beginConfig();
+        final ConfigTransactionJMXClient txClient = cfgRegistryClient.getConfigTransactionClient(objectName);
+
+        final ObjectName jvppOn;
+        try {
+            final String attributeName = VppJvppImplModule.descriptionJmxAttribute.getAttributeName();
+            final String factoryName = VppJvppImplModuleFactory.NAME;
+            jvppOn = txClient.lookupConfigBean(factoryName, "vpp-jvpp");
+
+            // Change configuration attribute of JVpp to trigger full reinitialization here using config subsystem
+            // TODO improve this when switching from karaf in planned minimal distribution
+            txClient.setAttribute(jvppOn, attributeName, new Attribute(attributeName,
+                Long.toString(nextAttempt)));
+
+            txClient.validateConfig();
+            cfgRegistryClient.commitConfig(txClient.getObjectName());
+            LOG.info("JVpp reinitialized successfully");
+        } catch (InstanceNotFoundException | ValidationException e) {
+            LOG.error("Unable to reinitialize JVpp. Honeycomb will not work properly from now on.", e);
+            throw new IllegalStateException("Unable to find jvpp instance in config subsystem. " +
+                "Unable to reinitialize JVpp", e);
+        } catch (ConflictingVersionException e) {
+            LOG.debug("Conflict changes occurred, retrying", e);
+            // Just retry until there's no conflicting change in progress
+            reinitializeJVpp(nextAttempt);
+        }
+
+        reinitializationCounter = nextAttempt;
+    }
 }
diff --git a/v3po/v3po2vpp/src/main/yang/v3po2vpp.yang b/v3po/v3po2vpp/src/main/yang/v3po2vpp.yang
index f706ec60d..5ccac8e94 100644
--- a/v3po/v3po2vpp/src/main/yang/v3po2vpp.yang
+++ b/v3po/v3po2vpp/src/main/yang/v3po2vpp.yang
@@ -9,6 +9,7 @@ module v3po2vpp {
     import vpp-cfg-init { prefix init; revision-date "2016-04-07"; }
     import opendaylight-md-sal-binding { prefix md-sal-binding; revision-date 2013-10-28;}
     import vpp-util { prefix vpp-u; revision-date 2016-04-06; }
+    import threadpool {prefix th;}
 
     description
         "This module contains reads and writers for v3po yang model";
@@ -53,6 +54,17 @@ module v3po2vpp {
                     }
                 }
             }
+
+            container keepalive-executor {
+                uses config:service-ref {
+                    refine type {
+                        mandatory true;
+                        config:required-identity th:scheduled-threadpool;
+                    }
+                }
+
+                description "Used to schedule keepalives";
+            }
         }
     }
 
diff --git a/v3po/v3po2vpp/src/test/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizerTest.java b/v3po/v3po2vpp/src/test/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizerTest.java
index 03b923102..0ddef2248 100644
--- a/v3po/v3po2vpp/src/test/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizerTest.java
+++ b/v3po/v3po2vpp/src/test/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizerTest.java
@@ -24,7 +24,6 @@ import static org.mockito.Mockito.when;
 import io.fd.honeycomb.v3po.translate.spi.read.ChildReaderCustomizer;
 import io.fd.honeycomb.v3po.translate.v3po.test.ChildReaderCustomizerTest;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionStage;
 import org.junit.Test;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.VppStateBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.vpp.state.Version;
@@ -54,20 +53,17 @@ public class VersionCustomizerTest extends ChildReaderCustomizerTest<Version, Ve
 
     @Test
     public void testReadCurrentAttributes() throws Exception {
-        final CompletionStage<ShowVersionReply> replyCS = mock(CompletionStage.class);
-        final CompletableFuture<ShowVersionReply> replyFuture = mock(CompletableFuture.class);
-        when(replyCS.toCompletableFuture()).thenReturn(replyFuture);
+        final CompletableFuture<ShowVersionReply> replyFuture = new CompletableFuture<>();
         final ShowVersionReply reply = new ShowVersionReply();
         reply.retval = 0;
         reply.version = new byte[]{};
         reply.program = new byte[]{};
         reply.buildDate = new byte[]{};
         reply.buildDirectory = new byte[]{};
-        when(replyFuture.get()).thenReturn(reply);
-        when(api.showVersion(any(ShowVersion.class))).thenReturn(replyCS);
+        replyFuture.complete(reply);
 
+        when(api.showVersion(any(ShowVersion.class))).thenReturn(replyFuture);
         getCustomizer().readCurrentAttributes(InstanceIdentifier.create(Version.class), new VersionBuilder(), ctx);
-
         verify(api).showVersion(any(ShowVersion.class));
     }
 }
\ No newline at end of file
diff --git a/v3po/v3po2vpp/src/test/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VppStateTest.java b/v3po/v3po2vpp/src/test/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VppStateTest.java
index 3d75d09e2..5ac108560 100644
--- a/v3po/v3po2vpp/src/test/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VppStateTest.java
+++ b/v3po/v3po2vpp/src/test/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VppStateTest.java
@@ -113,17 +113,16 @@ public class VppStateTest {
     }
 
     private void whenShowVersionThenReturn(int retval, Version version) throws ExecutionException, InterruptedException {
-        final CompletionStage<ShowVersionReply> replyCS = mock(CompletionStage.class);
-        final CompletableFuture<ShowVersionReply> replyFuture = mock(CompletableFuture.class);
-        when(replyCS.toCompletableFuture()).thenReturn(replyFuture);
+        final CompletableFuture<ShowVersionReply> replyFuture = new CompletableFuture<>();
         final ShowVersionReply reply = new ShowVersionReply();
         reply.retval = 0; // success
         reply.buildDate = version.getBuildDate().getBytes();
         reply.program = version.getName().getBytes();
         reply.version = version.getBranch().getBytes();
         reply.buildDirectory = version.getBuildDirectory().getBytes();
-        when(replyFuture.get()).thenReturn(reply);
-        when(api.showVersion(any(ShowVersion.class))).thenReturn(replyCS);
+
+        replyFuture.complete(reply);
+        when(api.showVersion(any(ShowVersion.class))).thenReturn(replyFuture);
     }
 
     private void whenL2FibTableDumpThenReturn(final List<L2FibTableEntry> entryList) throws ExecutionException, InterruptedException {
@@ -305,7 +304,7 @@ public class VppStateTest {
     @Test(expected = IllegalArgumentException.class)
     public void testReadBridgeDomainNotExisting() throws Exception {
         doReturn(Optional.absent()).when(mappingContext).read(getMappingIid("NOT EXISTING", "bd-test-instance"));
-        
+
         final Optional<? extends DataObject> read =
             readerRegistry.read(InstanceIdentifier.create(VppState.class).child(BridgeDomains.class).child(
                 BridgeDomain.class, new BridgeDomainKey("NOT EXISTING")), ctx);