HONEYCOMB-61: Detect VPP disconnect using keepalives

Change-Id: Ic664dbf452504d0fff97e8c766d735d9c5d95c72
Signed-off-by: Maros Marsalek <mmarsale@cisco.com>

12 files changed, 358 insertions, 29 deletions

diff --git a/v3po/translate-utils/src/main/java/io/fd/honeycomb/v3po/translate/util/KeepaliveReaderWrapper.java b/v3po/translate-utils/src/main/java/io/fd/honeycomb/v3po/translate/util/KeepaliveReaderWrapper.java
new file mode 100644
index 000000000..d59111faa
--- /dev/null
+++ b/v3po/translate-utils/src/main/java/io/fd/honeycomb/v3po/translate/util/KeepaliveReaderWrapper.java
@@ -0,0 +1,165 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.honeycomb.v3po.translate.util;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import io.fd.honeycomb.v3po.translate.MappingContext;
+import io.fd.honeycomb.v3po.translate.ModificationCache;
+import io.fd.honeycomb.v3po.translate.read.ChildReader;
+import io.fd.honeycomb.v3po.translate.read.ReadContext;
+import io.fd.honeycomb.v3po.translate.read.ReadFailedException;
+import java.io.Closeable;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import javax.annotation.Nonnegative;
+import javax.annotation.Nonnull;
+import org.opendaylight.yangtools.concepts.Builder;
+import org.opendaylight.yangtools.yang.binding.DataObject;
+import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Reader wrapper that periodically invokes a read to determine whether reads are still fully functional.
+ * In case a specific error occurs, Keep-alive failure listener gets notified.
+ */
+public final class KeepaliveReaderWrapper<D extends DataObject> implements ChildReader<D>, Runnable, Closeable {
+
+    private static final Logger LOG = LoggerFactory.getLogger(KeepaliveReaderWrapper.class);
+
+    private static final NoopReadContext CTX = new NoopReadContext();
+
+    private final ChildReader<D> delegate;
+    private final Class<? extends Exception> exceptionType;
+    private final KeepaliveFailureListener failureListener;
+    private final ScheduledFuture<?> scheduledFuture;
+
+    /**
+     * Create new Keepalive wrapper
+     *
+     * @param delegate underlying reader performing actual reads
+     * @param executor scheduled executor service to schedule keepalive calls
+     * @param exception type of exception used to differentiate keepalive exception from other exceptions
+     * @param delayInSeconds number of seconds to wait between keepalive calls
+     * @param failureListener listener to be called whenever a keepalive failure is detected
+     */
+    public KeepaliveReaderWrapper(@Nonnull final ChildReader<D> delegate,
+                                  @Nonnull final ScheduledExecutorService executor,
+                                  @Nonnull final Class<? extends Exception> exception,
+                                  @Nonnegative final int delayInSeconds,
+                                  @Nonnull final KeepaliveFailureListener failureListener) {
+        this.delegate = delegate;
+        this.exceptionType = exception;
+        this.failureListener = failureListener;
+        Preconditions.checkArgument(delayInSeconds > 0, "Delay cannot be < 0");
+        LOG.debug("Starting keep-alive execution on top of: {} with delay of: {} seconds", delegate, delayInSeconds);
+        scheduledFuture = executor.scheduleWithFixedDelay(this, delayInSeconds, delayInSeconds, TimeUnit.SECONDS);
+    }
+
+    @Nonnull
+    @Override
+    public Optional<? extends DataObject> read(@Nonnull final InstanceIdentifier<? extends DataObject> id,
+                                               @Nonnull final ReadContext ctx) throws ReadFailedException {
+        return delegate.read(id, ctx);
+    }
+
+    @Override
+    public void read(@Nonnull final InstanceIdentifier<? extends DataObject> id,
+                     @Nonnull final Builder<? extends DataObject> parentBuilder, @Nonnull final ReadContext ctx)
+        throws ReadFailedException {
+        delegate.read(id, parentBuilder, ctx);
+    }
+
+    @Nonnull
+    @Override
+    public InstanceIdentifier<D> getManagedDataObjectType() {
+        return delegate.getManagedDataObjectType();
+    }
+
+    @Override
+    public void run() {
+        LOG.trace("Invoking keepalive");
+        try {
+            final Optional<? extends DataObject> read = read(delegate.getManagedDataObjectType(), CTX);
+            LOG.debug("Keepalive executed successfully with data: {}", read);
+        } catch (Exception e) {
+            if(exceptionType.isAssignableFrom(e.getClass())) {
+                LOG.warn("Keepalive failed. Notifying listener", e);
+                failureListener.onKeepaliveFailure();
+            }
+            LOG.warn("Keepalive failed unexpectedly", e);
+            throw new IllegalArgumentException("Unexpected failure during keep-alive execution", e);
+        }
+    }
+
+    @Override
+    public void close() {
+        // Do not interrupt, it's not our executor
+        scheduledFuture.cancel(false);
+    }
+
+    /**
+     * Listener that gets called whenever keepalive fails as expected
+     */
+    public interface KeepaliveFailureListener {
+
+        void onKeepaliveFailure();
+    }
+
+    private static final class NoopMappingContext implements MappingContext {
+        @Override
+        public <T extends DataObject> Optional<T> read(@Nonnull final InstanceIdentifier<T> currentId) {
+            return Optional.absent();
+        }
+
+        @Override
+        public void delete(final InstanceIdentifier<?> path) {}
+
+        @Override
+        public <T extends DataObject> void merge(final InstanceIdentifier<T> path, final T data) {}
+
+        @Override
+        public <T extends DataObject> void put(final InstanceIdentifier<T> path, final T data) {}
+
+        @Override
+        public void close() {}
+    }
+
+    private static class NoopReadContext implements ReadContext {
+
+        private final ModificationCache modificationCache = new ModificationCache();
+
+        @Nonnull
+        @Override
+        public ModificationCache getModificationCache() {
+            return modificationCache;
+        }
+
+        @Nonnull
+        @Override
+        public MappingContext getMappingContext() {
+            return new NoopMappingContext();
+        }
+
+        @Override
+        public void close() {
+
+        }
+    }
+}
diff --git a/v3po/translate-utils/src/main/java/io/fd/honeycomb/v3po/translate/util/write/DelegatingWriterRegistry.java b/v3po/translate-utils/src/main/java/io/fd/honeycomb/v3po/translate/util/write/DelegatingWriterRegistry.java
index 7eba98a07..8b981af0d 100644
--- a/v3po/translate-utils/src/main/java/io/fd/honeycomb/v3po/translate/util/write/DelegatingWriterRegistry.java
+++ b/v3po/translate-utils/src/main/java/io/fd/honeycomb/v3po/translate/util/write/DelegatingWriterRegistry.java
@@ -50,14 +50,6 @@ public final class DelegatingWriterRegistry implements WriterRegistry {
 
     private static final Logger LOG = LoggerFactory.getLogger(DelegatingWriterRegistry.class);
 
-    private static final Function<InstanceIdentifier<?>, Class<? extends DataObject>> ID_TO_CLASS =
-            new Function<InstanceIdentifier<?>, Class<? extends DataObject>>() {
-                @Override
-                public Class<? extends DataObject> apply(final InstanceIdentifier<?> input) {
-                    return input.getTargetType();
-                }
-            };
-
     private final Map<Class<? extends DataObject>, Writer<? extends DataObject>> rootWriters;
 
     /**
@@ -130,7 +122,8 @@ public final class DelegatingWriterRegistry implements WriterRegistry {
 
     private void checkAllWritersPresent(final @Nonnull Map<InstanceIdentifier<?>, DataObject> nodesBefore) {
         final Set<Class<? extends DataObject>> nodesBeforeClasses =
-            Sets.newHashSet(Collections2.transform(nodesBefore.keySet(), ID_TO_CLASS));
+            Sets.newHashSet(Collections2.transform(nodesBefore.keySet(),
+                (Function<InstanceIdentifier<?>, Class<? extends DataObject>>) InstanceIdentifier::getTargetType));
         checkArgument(rootWriters.keySet().containsAll(nodesBeforeClasses),
                 "Unable to handle all changes. Missing dedicated writers for: %s",
                 Sets.difference(nodesBeforeClasses, rootWriters.keySet()));
diff --git a/v3po/v3po2vpp/pom.xml b/v3po/v3po2vpp/pom.xml
index 113511da1..d02807c9d 100644
--- a/v3po/v3po2vpp/pom.xml
+++ b/v3po/v3po2vpp/pom.xml
@@ -49,6 +49,20 @@
             <artifactId>v3po-api</artifactId>
             <version>${project.version}</version>
         </dependency>
+
+        <!-- Keepalives -->
+        <dependency>
+            <groupId>org.opendaylight.controller</groupId>
+            <artifactId>config-util</artifactId>
+            <version>0.4.2-Beryllium-SR2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.opendaylight.controller</groupId>
+            <artifactId>threadpool-config-api</artifactId>
+            <version>0.4.2-Beryllium-SR2</version>
+        </dependency>
+        <!---->
+
         <dependency>
             <groupId>${project.groupId}</groupId>
             <artifactId>translate-impl</artifactId>
diff --git a/v3po/v3po2vpp/src/main/config/default-config.xml b/v3po/v3po2vpp/src/main/config/default-config.xml
index be45a291b..19226d4fc 100644
--- a/v3po/v3po2vpp/src/main/config/default-config.xml
+++ b/v3po/v3po2vpp/src/main/config/default-config.xml
@@ -23,6 +23,9 @@
         <capability>urn:honeycomb:params:xml:ns:yang:v3po2vpp?module=v3po2vpp&amp;revision=2016-04-06</capability>
         <capability>urn:honeycomb:params:xml:ns:yang:data:api?module=data-api&amp;revision=2016-04-11</capability>
         <capability>urn:honeycomb:params:xml:ns:yang:vpp:util?module=vpp-util&amp;revision=2016-04-06</capability>
+        <capability>
+            urn:opendaylight:params:xml:ns:yang:controller:threadpool:impl:scheduled?module=threadpool-impl-scheduled&amp;revision=2013-12-01
+        </capability>
     </required-capabilities>
     <configuration>
 
@@ -54,6 +57,13 @@
                         <type xmlns:prefix="urn:honeycomb:params:xml:ns:yang:vpp:util">prefix:naming-context</type>
                         <name>bridge-domain-context</name>
                     </bridge-domain-context-vpp-state>
+                    <!-- Reuse netconf's scheduled executor for keepalives -->
+                    <keepalive-executor>
+                        <type xmlns:prefix="urn:opendaylight:params:xml:ns:yang:controller:threadpool">
+                            prefix:threadpool
+                        </type>
+                        <name>global-netconf-ssh-scheduled-executor</name>
+                    </keepalive-executor>
                 </module>
                 <module>
                     <type xmlns:prefix="urn:honeycomb:params:xml:ns:yang:v3po2vpp">prefix:interfaces-state-honeycomb-reader</type>
diff --git a/v3po/v3po2vpp/src/main/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizer.java b/v3po/v3po2vpp/src/main/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizer.java
index ad181a701..8122fd245 100644
--- a/v3po/v3po2vpp/src/main/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizer.java
+++ b/v3po/v3po2vpp/src/main/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizer.java
@@ -21,6 +21,7 @@ import io.fd.honeycomb.v3po.translate.read.ReadFailedException;
 import io.fd.honeycomb.v3po.translate.spi.read.ChildReaderCustomizer;
 import io.fd.honeycomb.v3po.translate.v3po.util.FutureJVppCustomizer;
 import io.fd.honeycomb.v3po.translate.v3po.util.TranslateUtils;
+import java.util.concurrent.CompletionStage;
 import javax.annotation.Nonnull;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.VppStateBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.vpp.state.Version;
@@ -36,6 +37,11 @@ public final class VersionCustomizer
     extends FutureJVppCustomizer
     implements ChildReaderCustomizer<Version, VersionBuilder> {
 
+    /**
+     * Default timeout for executing version read
+     */
+    private static final int DEFAULT_TIMEOUT_IN_SECONDS = 30;
+
     public VersionCustomizer(@Nonnull final FutureJVpp futureJVpp) {
         super(futureJVpp);
     }
@@ -54,13 +60,11 @@ public final class VersionCustomizer
     @Override
     public void readCurrentAttributes(@Nonnull InstanceIdentifier<Version> id, @Nonnull final VersionBuilder builder,
                                       @Nonnull final ReadContext context) throws ReadFailedException {
+        // Execute with timeout
+        final CompletionStage<ShowVersionReply> showVersionFuture = getFutureJVpp().showVersion(new ShowVersion());
+        final ShowVersionReply reply = TranslateUtils.getReply(showVersionFuture.toCompletableFuture(), id,
+            DEFAULT_TIMEOUT_IN_SECONDS);
 
-        ShowVersionReply reply;
-        try {
-            reply = getFutureJVpp().showVersion(new ShowVersion()).toCompletableFuture().get();
-        } catch (Exception e) {
-            throw new ReadFailedException(id, e);
-        }
         builder.setBranch(TranslateUtils.toString(reply.version));
         builder.setName(TranslateUtils.toString(reply.program));
         builder.setBuildDate(TranslateUtils.toString(reply.buildDate));
diff --git a/v3po/v3po2vpp/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/v3po2vpp/rev160406/VppStateHoneycombReaderModule.java b/v3po/v3po2vpp/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/v3po2vpp/rev160406/VppStateHoneycombReaderModule.java
index 9a0165577..32e5dabc0 100644
--- a/v3po/v3po2vpp/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/v3po2vpp/rev160406/VppStateHoneycombReaderModule.java
+++ b/v3po/v3po2vpp/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/v3po2vpp/rev160406/VppStateHoneycombReaderModule.java
@@ -4,14 +4,26 @@ import io.fd.honeycomb.v3po.translate.impl.read.CompositeChildReader;
 import io.fd.honeycomb.v3po.translate.impl.read.CompositeListReader;
 import io.fd.honeycomb.v3po.translate.impl.read.CompositeRootReader;
 import io.fd.honeycomb.v3po.translate.read.ChildReader;
+import io.fd.honeycomb.v3po.translate.util.KeepaliveReaderWrapper;
 import io.fd.honeycomb.v3po.translate.util.RWUtils;
 import io.fd.honeycomb.v3po.translate.util.read.CloseableReader;
 import io.fd.honeycomb.v3po.translate.util.read.ReflexiveChildReaderCustomizer;
 import io.fd.honeycomb.v3po.translate.util.read.ReflexiveRootReaderCustomizer;
+import io.fd.honeycomb.v3po.translate.v3po.util.ReadTimeoutException;
 import io.fd.honeycomb.v3po.translate.v3po.vppstate.BridgeDomainCustomizer;
 import io.fd.honeycomb.v3po.translate.v3po.vppstate.VersionCustomizer;
+import java.lang.management.ManagementFactory;
 import java.util.ArrayList;
 import java.util.List;
+import javax.management.Attribute;
+import javax.management.InstanceNotFoundException;
+import javax.management.ObjectName;
+import org.opendaylight.controller.config.api.ConflictingVersionException;
+import org.opendaylight.controller.config.api.ValidationException;
+import org.opendaylight.controller.config.util.ConfigRegistryJMXClient;
+import org.opendaylight.controller.config.util.ConfigTransactionJMXClient;
+import org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.vpp.jvpp.cfg.rev160406.VppJvppImplModule;
+import org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.vpp.jvpp.cfg.rev160406.VppJvppImplModuleFactory;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.VppState;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.VppStateBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.vpp.state.BridgeDomains;
@@ -22,8 +34,13 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.vpp.state.bridge.domains.BridgeDomainKey;
 import org.opendaylight.yangtools.yang.binding.ChildOf;
 import org.openvpp.jvpp.future.FutureJVpp;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class VppStateHoneycombReaderModule extends org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.v3po2vpp.rev160406.AbstractVppStateHoneycombReaderModule {
+
+    private static final Logger LOG = LoggerFactory.getLogger(VppStateHoneycombReaderModule.class);
+
     public VppStateHoneycombReaderModule(org.opendaylight.controller.config.api.ModuleIdentifier identifier, org.opendaylight.controller.config.api.DependencyResolver dependencyResolver) {
         super(identifier, dependencyResolver);
     }
@@ -41,7 +58,13 @@ public class VppStateHoneycombReaderModule extends org.opendaylight.yang.gen.v1.
     public java.lang.AutoCloseable createInstance() {
         final FutureJVpp vppApi = getVppJvppDependency();
 
-        final ChildReader<Version> versionReader = new CompositeChildReader<>(Version.class, new VersionCustomizer(vppApi));
+        ChildReader<Version> versionReader = new CompositeChildReader<>(Version.class, new VersionCustomizer(vppApi));
+        // Wrap with keepalive reader to detect connection issues
+        // TODO keepalive reader wrapper relies on VersionReaderCustomizer (to perform timeout on reads)
+        // Once readers+customizers are asynchronous, pull the timeout to keepalive executor so that keepalive wrapper
+        // is truly generic
+        versionReader = new KeepaliveReaderWrapper<>(versionReader, getKeepaliveExecutorDependency().getExecutor(),
+            ReadTimeoutException.class, 30, () -> reinitializeJVpp(reinitializationCounter));
 
         final CompositeListReader<BridgeDomain, BridgeDomainKey, BridgeDomainBuilder> bridgeDomainReader =
             new CompositeListReader<>(BridgeDomain.class, new BridgeDomainCustomizer(vppApi,
@@ -62,4 +85,56 @@ public class VppStateHoneycombReaderModule extends org.opendaylight.yang.gen.v1.
             RWUtils.emptyAugReaderList(),
             new ReflexiveRootReaderCustomizer<>(VppStateBuilder.class)));
     }
+
+    private static long reinitializationCounter;
+    private static final long reinitializationLimit = 10;
+
+    /**
+     * In case we detect connection issues with VPP, reinitialize JVpp
+     */
+    private void reinitializeJVpp(final long currentAttempt) {
+        // FIXME https://jira.fd.io/browse/HONEYCOMB-78 This code correctly re-initializes all the components
+        // starting with jvpp, but jvpp reconnect fails. Test in a JVpp test and then from C
+        LOG.info("Reinitializing JVpp, attempt: {}", currentAttempt);
+
+        final long nextAttempt = currentAttempt + 1;
+        if (nextAttempt - reinitializationCounter > reinitializationLimit) {
+            LOG.error("Too many JVpp reinitialization attempts. Unable to reinitialize JVpp in {} attempts. Giving up",
+                reinitializationLimit);
+            throw new IllegalStateException("Too many JVpp reinitialization attempts. Unable to reinitialize JVpp in "
+                + reinitializationLimit + " attempts. Giving up");
+        }
+
+        final ConfigRegistryJMXClient cfgRegistryClient =
+            ConfigRegistryJMXClient.createWithoutNotifications(ManagementFactory.getPlatformMBeanServer());
+
+        final ObjectName objectName = cfgRegistryClient.beginConfig();
+        final ConfigTransactionJMXClient txClient = cfgRegistryClient.getConfigTransactionClient(objectName);
+
+        final ObjectName jvppOn;
+        try {
+            final String attributeName = VppJvppImplModule.descriptionJmxAttribute.getAttributeName();
+            final String factoryName = VppJvppImplModuleFactory.NAME;
+            jvppOn = txClient.lookupConfigBean(factoryName, "vpp-jvpp");
+
+            // Change configuration attribute of JVpp to trigger full reinitialization here using config subsystem
+            // TODO improve this when switching from karaf in planned minimal distribution
+            txClient.setAttribute(jvppOn, attributeName, new Attribute(attributeName,
+                Long.toString(nextAttempt)));
+
+            txClient.validateConfig();
+            cfgRegistryClient.commitConfig(txClient.getObjectName());
+            LOG.info("JVpp reinitialized successfully");
+        } catch (InstanceNotFoundException | ValidationException e) {
+            LOG.error("Unable to reinitialize JVpp. Honeycomb will not work properly from now on.", e);
+            throw new IllegalStateException("Unable to find jvpp instance in config subsystem. " +
+                "Unable to reinitialize JVpp", e);
+        } catch (ConflictingVersionException e) {
+            LOG.debug("Conflict changes occurred, retrying", e);
+            // Just retry until there's no conflicting change in progress
+            reinitializeJVpp(nextAttempt);
+        }
+
+        reinitializationCounter = nextAttempt;
+    }
 }
diff --git a/v3po/v3po2vpp/src/main/yang/v3po2vpp.yang b/v3po/v3po2vpp/src/main/yang/v3po2vpp.yang
index f706ec60d..5ccac8e94 100644
--- a/v3po/v3po2vpp/src/main/yang/v3po2vpp.yang
+++ b/v3po/v3po2vpp/src/main/yang/v3po2vpp.yang
@@ -9,6 +9,7 @@ module v3po2vpp {
     import vpp-cfg-init { prefix init; revision-date "2016-04-07"; }
     import opendaylight-md-sal-binding { prefix md-sal-binding; revision-date 2013-10-28;}
     import vpp-util { prefix vpp-u; revision-date 2016-04-06; }
+    import threadpool {prefix th;}
 
     description
         "This module contains reads and writers for v3po yang model";
@@ -53,6 +54,17 @@ module v3po2vpp {
                     }
                 }
             }
+
+            container keepalive-executor {
+                uses config:service-ref {
+                    refine type {
+                        mandatory true;
+                        config:required-identity th:scheduled-threadpool;
+                    }
+                }
+
+                description "Used to schedule keepalives";
+            }
         }
     }
 
diff --git a/v3po/v3po2vpp/src/test/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizerTest.java b/v3po/v3po2vpp/src/test/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizerTest.java
index 03b923102..0ddef2248 100644
--- a/v3po/v3po2vpp/src/test/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizerTest.java
+++ b/v3po/v3po2vpp/src/test/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VersionCustomizerTest.java
@@ -24,7 +24,6 @@ import static org.mockito.Mockito.when;
 import io.fd.honeycomb.v3po.translate.spi.read.ChildReaderCustomizer;
 import io.fd.honeycomb.v3po.translate.v3po.test.ChildReaderCustomizerTest;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionStage;
 import org.junit.Test;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.VppStateBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev150105.vpp.state.Version;
@@ -54,20 +53,17 @@ public class VersionCustomizerTest extends ChildReaderCustomizerTest<Version, Ve
 
     @Test
     public void testReadCurrentAttributes() throws Exception {
-        final CompletionStage<ShowVersionReply> replyCS = mock(CompletionStage.class);
-        final CompletableFuture<ShowVersionReply> replyFuture = mock(CompletableFuture.class);
-        when(replyCS.toCompletableFuture()).thenReturn(replyFuture);
+        final CompletableFuture<ShowVersionReply> replyFuture = new CompletableFuture<>();
         final ShowVersionReply reply = new ShowVersionReply();
         reply.retval = 0;
         reply.version = new byte[]{};
         reply.program = new byte[]{};
         reply.buildDate = new byte[]{};
         reply.buildDirectory = new byte[]{};
-        when(replyFuture.get()).thenReturn(reply);
-        when(api.showVersion(any(ShowVersion.class))).thenReturn(replyCS);
+        replyFuture.complete(reply);
 
+        when(api.showVersion(any(ShowVersion.class))).thenReturn(replyFuture);
         getCustomizer().readCurrentAttributes(InstanceIdentifier.create(Version.class), new VersionBuilder(), ctx);
-
         verify(api).showVersion(any(ShowVersion.class));
     }
 }
\ No newline at end of file
diff --git a/v3po/v3po2vpp/src/test/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VppStateTest.java b/v3po/v3po2vpp/src/test/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VppStateTest.java
index 3d75d09e2..5ac108560 100644
--- a/v3po/v3po2vpp/src/test/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VppStateTest.java
+++ b/v3po/v3po2vpp/src/test/java/io/fd/honeycomb/v3po/translate/v3po/vppstate/VppStateTest.java
@@ -113,17 +113,16 @@ public class VppStateTest {
     }
 
     private void whenShowVersionThenReturn(int retval, Version version) throws ExecutionException, InterruptedException {
-        final CompletionStage<ShowVersionReply> replyCS = mock(CompletionStage.class);
-        final CompletableFuture<ShowVersionReply> replyFuture = mock(CompletableFuture.class);
-        when(replyCS.toCompletableFuture()).thenReturn(replyFuture);
+        final CompletableFuture<ShowVersionReply> replyFuture = new CompletableFuture<>();
         final ShowVersionReply reply = new ShowVersionReply();
         reply.retval = 0; // success
         reply.buildDate = version.getBuildDate().getBytes();
         reply.program = version.getName().getBytes();
         reply.version = version.getBranch().getBytes();
         reply.buildDirectory = version.getBuildDirectory().getBytes();
-        when(replyFuture.get()).thenReturn(reply);
-        when(api.showVersion(any(ShowVersion.class))).thenReturn(replyCS);
+
+        replyFuture.complete(reply);
+        when(api.showVersion(any(ShowVersion.class))).thenReturn(replyFuture);
     }
 
     private void whenL2FibTableDumpThenReturn(final List<L2FibTableEntry> entryList) throws ExecutionException, InterruptedException {
@@ -305,7 +304,7 @@ public class VppStateTest {
     @Test(expected = IllegalArgumentException.class)
     public void testReadBridgeDomainNotExisting() throws Exception {
         doReturn(Optional.absent()).when(mappingContext).read(getMappingIid("NOT EXISTING", "bd-test-instance"));
-        
+
         final Optional<? extends DataObject> read =
             readerRegistry.read(InstanceIdentifier.create(VppState.class).child(BridgeDomains.class).child(
                 BridgeDomain.class, new BridgeDomainKey("NOT EXISTING")), ctx);
diff --git a/v3po/vpp-jvpp-cfg/src/main/yang/vpp-jvpp-cfg.yang b/v3po/vpp-jvpp-cfg/src/main/yang/vpp-jvpp-cfg.yang
index b91fcba09..01a0bf339 100644
--- a/v3po/vpp-jvpp-cfg/src/main/yang/vpp-jvpp-cfg.yang
+++ b/v3po/vpp-jvpp-cfg/src/main/yang/vpp-jvpp-cfg.yang
@@ -32,6 +32,11 @@ module vpp-jvpp-cfg {
                 type string;
                 default "v3poODL";
             }
+
+            leaf description {
+                type string;
+                description "Artificial leaf just to trigger reinitialization of JVpp from HC";
+            }
         }
     }
 
diff --git a/v3po/vpp-translate-utils/src/main/java/io/fd/honeycomb/v3po/translate/v3po/util/ReadTimeoutException.java b/v3po/vpp-translate-utils/src/main/java/io/fd/honeycomb/v3po/translate/v3po/util/ReadTimeoutException.java
new file mode 100644
index 000000000..654b13825
--- /dev/null
+++ b/v3po/vpp-translate-utils/src/main/java/io/fd/honeycomb/v3po/translate/v3po/util/ReadTimeoutException.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.honeycomb.v3po.translate.v3po.util;
+
+import com.google.common.annotations.Beta;
+import io.fd.honeycomb.v3po.translate.read.ReadFailedException;
+import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
+
+/**
+ * Thrown when read method invocation times out.
+ */
+@Beta
+public class ReadTimeoutException extends ReadFailedException {
+
+    public ReadTimeoutException(final InstanceIdentifier<?> id, final Throwable cause) {
+        super(id, cause);
+    }
+
+}
diff --git a/v3po/vpp-translate-utils/src/main/java/io/fd/honeycomb/v3po/translate/v3po/util/TranslateUtils.java b/v3po/vpp-translate-utils/src/main/java/io/fd/honeycomb/v3po/translate/v3po/util/TranslateUtils.java
index 9f6f1f676..f14b2eb6d 100644
--- a/v3po/vpp-translate-utils/src/main/java/io/fd/honeycomb/v3po/translate/v3po/util/TranslateUtils.java
+++ b/v3po/vpp-translate-utils/src/main/java/io/fd/honeycomb/v3po/translate/v3po/util/TranslateUtils.java
@@ -22,10 +22,14 @@ import com.google.common.base.Splitter;
 import java.util.List;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.function.BiConsumer;
+import javax.annotation.Nonnegative;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.Ipv4AddressNoZone;
+import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 import org.openvpp.jvpp.dto.JVppReply;
 
 public final class TranslateUtils {
@@ -49,6 +53,25 @@ public final class TranslateUtils {
         }
     }
 
+    public static <REP extends JVppReply<?>> REP getReply(@Nonnull Future<REP> future,
+                                                          @Nonnull final InstanceIdentifier<?> replyType,
+                                                          @Nonnegative final int timeoutInSeconds)
+        throws ReadTimeoutException {
+        try {
+            checkArgument(timeoutInSeconds > 0, "Timeout cannot be < 0");
+            return future.get(timeoutInSeconds, TimeUnit.SECONDS);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            throw new IllegalStateException("Interrupted", e);
+        } catch (ExecutionException e) {
+            // Execution exception should not occur, since we are using return codes for errors
+            // TODO fix when using exceptions instead of return codes
+            throw new IllegalArgumentException("Future " + " should not fail with an exception", e);
+        } catch (TimeoutException e) {
+            throw new ReadTimeoutException(replyType, e);
+        }
+    }
+
     public static byte[] ipv4AddressNoZoneToArray(final Ipv4AddressNoZone ipv4Addr) {
         byte[] retval = new byte[4];
         String[] dots = ipv4Addr.getValue().split("\\.");