HONEYCOMB-78: enable HC restart

Includes:
- restart script for honeycomb
- HONEYCOMBC-78 fix: jVPPRegistry.close() is not invoked in case of
  VPP restart. That prevents subsequent attempts to establish
  connection from failing (we cannot invoke vl_client_disconnect_from_vlib
  before connect_to_vpe on fresh vpp instance).

Change-Id: Icf4d6a6a40605ed34d307a0f1405b8804ba5df53
Signed-off-by: Marek Gradzki <mgradzki@cisco.com>

7 files changed, 133 insertions, 14 deletions

diff --git a/common/minimal-distribution-parent/pom.xml b/common/minimal-distribution-parent/pom.xml
index cdaf92582..373b642c7 100644
--- a/common/minimal-distribution-parent/pom.xml
+++ b/common/minimal-distribution-parent/pom.xml
@@ -25,7 +25,21 @@
 
     <properties>
         <nexusproxy>http://nexus.fd.io/content</nexusproxy>
-        <!--<main.class>Main</main.class>-->
+        <start.script.template>
+#!/bin/sh -
+STATUS=100
+
+while [ $STATUS -eq 100 ]
+do
+  %s
+  STATUS=$?
+  echo "Honeycomb exited with status: $STATUS"
+  if [ $STATUS -eq 100 ]
+  then
+    echo "Restarting..."
+  fi
+done
+        </start.script.template>
         <exec.parameters>-Xms32m -Xmx128m -XX:MetaspaceSize=32m -XX:MaxMetaspaceSize=128m</exec.parameters>
         <exec.parameters.minimal>-client -Xms20m -Xmx32m -XX:MetaspaceSize=5m -XX:MaxMetaspaceSize=32m -XX:MaxMetaspaceExpansion=1m -Xss512k -XX:+UseSerialGC -Djava.compiler=NONE -Xverify:none -noverify</exec.parameters.minimal>
         <debug.parameters>-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005</debug.parameters>
@@ -242,29 +256,30 @@
                                     import java.nio.file.Paths
 
                                     log.info "Generating shell exec script"
+                                    def scriptTemplate = properties.getOrDefault("start.script.template", "")
                                     def args = properties.getOrDefault("exec.parameters", "")
                                     log.debug "Additional shell exec script properties: ${args}"
                                     def javaArgs = "${args} -jar \$(dirname \$0)/${project.artifactId}-${project.version}.jar"
                                     def scriptParent = Paths.get(project.build.outputDirectory, "honeycomb-minimal-resources")
                                     scriptParent.toFile().mkdirs()
-                                    def scriptContent = "#!/bin/sh -\njava " + javaArgs
+                                    def scriptContent = "java " + javaArgs
                                     log.info "Generating shell exec script as ${scriptContent}"
                                     def scriptPath = Paths.get(scriptParent.toString(), "honeycomb")
                                     log.info "Writing shell exec script to ${scriptPath}"
-                                    scriptPath.toFile().text = scriptContent
+                                    scriptPath.toFile().text = String.format(scriptTemplate, scriptContent)
                                     scriptPath.toFile().setExecutable(true)
 
                                     scriptPath = Paths.get(scriptParent.toString(), "honeycomb-start")
                                     log.info "Writing shell exec script to ${scriptPath}"
-                                    scriptPath.toFile().text = scriptContent + " &amp;"
+                                    scriptPath.toFile().text = "\$(dirname \$0)/honeycomb &amp;"
                                     scriptPath.toFile().setExecutable(true)
 
                                     def debug_args = properties.getOrDefault("debug.parameters", "")
-                                    def debugScriptContent = "#!/bin/sh -\njava" + " ${debug_args} " + javaArgs
+                                    def debugScriptContent = "java" + " ${debug_args} " + javaArgs
                                     log.info "Generating shell debug script as ${debugScriptContent}"
                                     scriptPath = Paths.get(scriptParent.toString(), "honeycomb-debug")
                                     log.info "Writing shell debug script to ${scriptPath}"
-                                    scriptPath.toFile().text = debugScriptContent
+                                    scriptPath.toFile().text = String.format(scriptTemplate, debugScriptContent)
                                     scriptPath.toFile().setExecutable(true)
                                 </source>
                             </configuration>
diff --git a/v3po/v3po2vpp/src/main/java/io/fd/honeycomb/translate/v3po/VppStateHoneycombReaderFactory.java b/v3po/v3po2vpp/src/main/java/io/fd/honeycomb/translate/v3po/VppStateHoneycombReaderFactory.java
index 75779d949..1c28fbf4a 100644
--- a/v3po/v3po2vpp/src/main/java/io/fd/honeycomb/translate/v3po/VppStateHoneycombReaderFactory.java
+++ b/v3po/v3po2vpp/src/main/java/io/fd/honeycomb/translate/v3po/VppStateHoneycombReaderFactory.java
@@ -25,6 +25,7 @@ import io.fd.honeycomb.translate.read.registry.ModifiableReaderRegistryBuilder;
 import io.fd.honeycomb.translate.util.read.KeepaliveReaderWrapper;
 import io.fd.honeycomb.translate.v3po.util.NamingContext;
 import io.fd.honeycomb.translate.v3po.util.ReadTimeoutException;
+import io.fd.honeycomb.translate.v3po.util.VppStatusListener;
 import io.fd.honeycomb.translate.v3po.vppstate.BridgeDomainCustomizer;
 import io.fd.honeycomb.translate.v3po.vppstate.L2FibEntryCustomizer;
 import io.fd.honeycomb.translate.v3po.vppstate.VersionCustomizer;
@@ -40,27 +41,26 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.vpp.state.bridge.domains.BridgeDomain;
 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 import org.openvpp.jvpp.core.future.FutureJVppCore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 public final class VppStateHoneycombReaderFactory implements ReaderFactory {
 
-    private static final Logger LOG = LoggerFactory.getLogger(VppStateHoneycombReaderFactory.class);
-
     private final FutureJVppCore jVpp;
     private final NamingContext ifcCtx;
     private final NamingContext bdCtx;
     private final ScheduledExecutorService keepaliveExecutor;
+    private final VppStatusListener vppStatusListener;
 
     @Inject
     public VppStateHoneycombReaderFactory(final FutureJVppCore jVpp,
                                           @Named("interface-context") final NamingContext ifcCtx,
                                           @Named("bridge-domain-context") final NamingContext bdCtx,
-                                          final ScheduledExecutorService keepaliveExecutorDependency) {
+                                          final ScheduledExecutorService keepaliveExecutorDependency,
+                                          final VppStatusListener vppStatusListener) {
         this.jVpp = jVpp;
         this.ifcCtx = ifcCtx;
         this.bdCtx = bdCtx;
         this.keepaliveExecutor = keepaliveExecutorDependency;
+        this.vppStatusListener = vppStatusListener;
     }
 
     @Override
@@ -73,9 +73,7 @@ public final class VppStateHoneycombReaderFactory implements ReaderFactory {
         // Relying on VersionCustomizer to provide a "timing out read"
         registry.add(new KeepaliveReaderWrapper<>(
                 new GenericReader<>(vppStateId.child(Version.class), new VersionCustomizer(jVpp)),
-                keepaliveExecutor, ReadTimeoutException.class, 30,
-                // FIXME HONEYCOMB-78 trigger jvpp reinitialization here
-                () -> LOG.error("Keepalive failed. VPP is probably DOWN!")));
+                keepaliveExecutor, ReadTimeoutException.class, 30, vppStatusListener));
         //  BridgeDomains(Structural)
         final InstanceIdentifier<BridgeDomains> bridgeDomainsId = vppStateId.child(BridgeDomains.class);
         registry.addStructuralReader(bridgeDomainsId, BridgeDomainsBuilder.class);
diff --git a/vpp-common/minimal-distribution/src/main/java/io/fd/honeycomb/vpp/distro/JVppRegistryProvider.java b/vpp-common/minimal-distribution/src/main/java/io/fd/honeycomb/vpp/distro/JVppRegistryProvider.java
index 8605fa0cc..33756f273 100644
--- a/vpp-common/minimal-distribution/src/main/java/io/fd/honeycomb/vpp/distro/JVppRegistryProvider.java
+++ b/vpp-common/minimal-distribution/src/main/java/io/fd/honeycomb/vpp/distro/JVppRegistryProvider.java
@@ -18,6 +18,7 @@ package io.fd.honeycomb.vpp.distro;
 
 import com.google.inject.Inject;
 import io.fd.honeycomb.infra.distro.ProviderTrait;
+import io.fd.honeycomb.translate.v3po.util.VppStatusListener;
 import java.io.IOException;
 import org.openvpp.jvpp.JVppRegistry;
 import org.openvpp.jvpp.JVppRegistryImpl;
@@ -34,6 +35,8 @@ public final class JVppRegistryProvider extends ProviderTrait<JVppRegistry> {
 
     @Inject
     private VppConfigAttributes config;
+    @Inject
+    private VppStatusListener vppStatus;
 
     @Override
     protected JVppRegistryImpl create() {
@@ -47,6 +50,10 @@ public final class JVppRegistryProvider extends ProviderTrait<JVppRegistry> {
                 @Override
                 public void run() {
                     LOG.info("Disconnecting from VPP");
+                    if (vppStatus.isDown()) {
+                        LOG.info("VPP is down. JVppRegistry cleanup is not needed. Exiting");
+                        return;
+                    }
                     try {
                         registry.close();
                         LOG.info("Successfully disconnected from VPP as {}", config.jvppConnectionName);
diff --git a/vpp-common/minimal-distribution/src/main/java/io/fd/honeycomb/vpp/distro/VppCommonModule.java b/vpp-common/minimal-distribution/src/main/java/io/fd/honeycomb/vpp/distro/VppCommonModule.java
index 87352356b..93ec559b5 100644
--- a/vpp-common/minimal-distribution/src/main/java/io/fd/honeycomb/vpp/distro/VppCommonModule.java
+++ b/vpp-common/minimal-distribution/src/main/java/io/fd/honeycomb/vpp/distro/VppCommonModule.java
@@ -20,6 +20,7 @@ import com.google.inject.AbstractModule;
 import com.google.inject.Singleton;
 import com.google.inject.multibindings.Multibinder;
 import io.fd.honeycomb.translate.read.ReaderFactory;
+import io.fd.honeycomb.translate.v3po.util.VppStatusListener;
 import net.jmob.guice.conf.core.ConfigurationModule;
 import org.openvpp.jvpp.JVppRegistry;
 import org.openvpp.jvpp.core.future.FutureJVppCore;
@@ -31,6 +32,7 @@ public final class VppCommonModule extends AbstractModule {
         // Inject non-dependency configuration
         requestInjection(VppConfigAttributes.class);
 
+        bind(VppStatusListener.class).toInstance(new VppStatusListener());
         bind(JVppRegistry.class).toProvider(JVppRegistryProvider.class).in(Singleton.class);
         bind(FutureJVppCore.class).toProvider(JVppCoreProvider.class).in(Singleton.class);
 
diff --git a/vpp-common/vpp-translate-utils/pom.xml b/vpp-common/vpp-translate-utils/pom.xml
index bb5568202..935dea132 100644
--- a/vpp-common/vpp-translate-utils/pom.xml
+++ b/vpp-common/vpp-translate-utils/pom.xml
@@ -30,6 +30,8 @@
 
     <properties>
         <commons.codec.version>1.9</commons.codec.version>
+        <system.rules.version>1.16.0</system.rules.version>
+        <skinny.logback.version>1.0.8</skinny.logback.version>
     </properties>
 
     <dependencies>
@@ -80,5 +82,17 @@
             <artifactId>mockito-all</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.skinny-framework</groupId>
+            <artifactId>skinny-logback</artifactId>
+            <version>${skinny.logback.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.github.stefanbirkner</groupId>
+            <artifactId>system-rules</artifactId>
+            <version>${system.rules.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 </project>
diff --git a/vpp-common/vpp-translate-utils/src/main/java/io/fd/honeycomb/translate/v3po/util/VppStatusListener.java b/vpp-common/vpp-translate-utils/src/main/java/io/fd/honeycomb/translate/v3po/util/VppStatusListener.java
new file mode 100644
index 000000000..7a728280d
--- /dev/null
+++ b/vpp-common/vpp-translate-utils/src/main/java/io/fd/honeycomb/translate/v3po/util/VppStatusListener.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.honeycomb.translate.v3po.util;
+
+import io.fd.honeycomb.translate.util.read.KeepaliveReaderWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Listens to vpp status changes. Restarts honeycomb if vpp is down.
+ */
+public final class VppStatusListener implements KeepaliveReaderWrapper.KeepaliveFailureListener {
+
+    /**
+     * Value picked up by honeycomb start script, tigers honeycomb restart when returned by the java process
+     */
+    public static final int RESTART_ERROR_CODE = 100;
+    private static final Logger LOG = LoggerFactory.getLogger(VppStatusListener.class);
+
+    private volatile boolean down;
+
+    public boolean isDown() {
+        return down;
+    }
+
+    @Override
+    public void onKeepaliveFailure() {
+        LOG.error("Keepalive failed. VPP is probably DOWN! Restarting Honeycomb");
+        this.down = true;
+        System.exit(RESTART_ERROR_CODE);
+    }
+}
diff --git a/vpp-common/vpp-translate-utils/src/test/java/io/fd/honeycomb/translate/v3po/util/VppStatusListenerTest.java b/vpp-common/vpp-translate-utils/src/test/java/io/fd/honeycomb/translate/v3po/util/VppStatusListenerTest.java
new file mode 100644
index 000000000..b74c9c7d0
--- /dev/null
+++ b/vpp-common/vpp-translate-utils/src/test/java/io/fd/honeycomb/translate/v3po/util/VppStatusListenerTest.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.honeycomb.translate.v3po.util;
+
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.contrib.java.lang.system.ExpectedSystemExit;
+
+public class VppStatusListenerTest {
+
+    @Rule
+    public final ExpectedSystemExit exit = ExpectedSystemExit.none();
+
+    @Test
+    public void testOnKeepaliveFailure() throws Exception {
+        final VppStatusListener vppStatus = new VppStatusListener();
+        exit.expectSystemExitWithStatus(VppStatusListener.RESTART_ERROR_CODE);
+        exit.checkAssertionAfterwards(() -> assertTrue(vppStatus.isDown()));
+        vppStatus.onKeepaliveFailure();
+    }
+}
\ No newline at end of file