HONEYCOMB-61: Config persister

Add PersistingDataTree adapter for in memory config data tree
Using JSON NormalizedNode writers from ODL

Change-Id: Ida91fe6aa34aaeaedcd061ba1551afe49bbddbbb
Signed-off-by: Maros Marsalek <mmarsale@cisco.com>

8 files changed, 422 insertions, 1 deletions

diff --git a/v3po/data-impl/src/main/java/io/fd/honeycomb/v3po/data/impl/ConfigDataTree.java b/v3po/data-impl/src/main/java/io/fd/honeycomb/v3po/data/impl/ConfigDataTree.java
index ab4cf6446..1e118ff2a 100644
--- a/v3po/data-impl/src/main/java/io/fd/honeycomb/v3po/data/impl/ConfigDataTree.java
+++ b/v3po/data-impl/src/main/java/io/fd/honeycomb/v3po/data/impl/ConfigDataTree.java
@@ -109,7 +109,6 @@ public final class ConfigDataTree implements ModifiableDataTree {
         final Map<InstanceIdentifier<?>, DataObject> nodesAfter = extractNetconfData(normalizedDataAfter);
         LOG.debug("ConfigDataTree.modify() extracted nodesAfter={}", nodesAfter.keySet());
 
-
         final DOMDataReadOnlyTransaction beforeTx = new ReadOnlyTransaction(EMPTY_OPERATIONAL, takeSnapshot());
         final ConfigSnapshot modificationSnapshot = new ConfigSnapshot(modification);
         final DOMDataReadOnlyTransaction afterTx = new ReadOnlyTransaction(EMPTY_OPERATIONAL, modificationSnapshot);
diff --git a/v3po/data-impl/src/main/java/io/fd/honeycomb/v3po/data/impl/PersistingDataTreeAdapter.java b/v3po/data-impl/src/main/java/io/fd/honeycomb/v3po/data/impl/PersistingDataTreeAdapter.java
new file mode 100644
index 000000000..25d1d8dda
--- /dev/null
+++ b/v3po/data-impl/src/main/java/io/fd/honeycomb/v3po/data/impl/PersistingDataTreeAdapter.java
@@ -0,0 +1,185 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.honeycomb.v3po.data.impl;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Optional;
+import com.google.gson.stream.JsonWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import javax.annotation.Nonnull;
+import org.opendaylight.controller.sal.core.api.model.SchemaService;
+import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
+import org.opendaylight.yangtools.yang.data.api.schema.ContainerNode;
+import org.opendaylight.yangtools.yang.data.api.schema.DataContainerChild;
+import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
+import org.opendaylight.yangtools.yang.data.api.schema.stream.NormalizedNodeStreamWriter;
+import org.opendaylight.yangtools.yang.data.api.schema.stream.NormalizedNodeWriter;
+import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTree;
+import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidate;
+import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeModification;
+import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeSnapshot;
+import org.opendaylight.yangtools.yang.data.api.schema.tree.DataValidationFailedException;
+import org.opendaylight.yangtools.yang.data.codec.gson.JSONCodecFactory;
+import org.opendaylight.yangtools.yang.data.codec.gson.JSONNormalizedNodeStreamWriter;
+import org.opendaylight.yangtools.yang.data.codec.gson.JsonWriterFactory;
+import org.opendaylight.yangtools.yang.model.api.SchemaContext;
+import org.opendaylight.yangtools.yang.model.api.SchemaPath;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Adapter for a DataTree that stores current state of data in backing DataTree on each successful commit.
+ * Uses JSON format.
+ */
+public class PersistingDataTreeAdapter implements org.opendaylight.yangtools.yang.data.api.schema.tree.DataTree, AutoCloseable {
+
+    private static final Logger LOG = LoggerFactory.getLogger(PersistingDataTreeAdapter.class);
+
+    private final DataTree delegateDependency;
+    private SchemaService schemaServiceDependency;
+    private final Path path;
+
+    /**
+     * Create new Persisting DataTree adapter
+     *
+     * @param delegateDependency backing data tree that actually handles all the operations
+     * @param persistPath path to a file (existing or not) to be used as storage for persistence. Full control over
+     *                    a file at peristPath is expected
+     * @param schemaServiceDependency schemaContext provier
+     */
+    public PersistingDataTreeAdapter(@Nonnull final DataTree delegateDependency,
+                                     @Nonnull final SchemaService schemaServiceDependency,
+                                     @Nonnull final Path persistPath) {
+        this.path = testPersistPath(persistPath);
+        this.delegateDependency = delegateDependency;
+        this.schemaServiceDependency = schemaServiceDependency;
+    }
+
+    /**
+     * Test whether file at persistPath is a file and can be created/deleted
+     */
+    private Path testPersistPath(final Path persistPath) {
+        try {
+            checkArgument(!Files.isDirectory(persistPath), "Path %s points to a directory", persistPath);
+            Files.createDirectories(persistPath.getParent());
+            Files.write(persistPath, new byte[]{}, StandardOpenOption.CREATE);
+        } catch (IOException | UnsupportedOperationException e) {
+            LOG.warn("Provided path for persistence: {} is not usable", persistPath, e);
+            throw new IllegalArgumentException("Path " + persistPath + " cannot be used as ");
+        } finally {
+            try {
+                Files.delete(persistPath);
+            } catch (IOException e) {
+                LOG.warn("Unable to delete file at {}", persistPath, e);
+            }
+        }
+
+        return persistPath;
+    }
+
+    @Override
+    public DataTreeSnapshot takeSnapshot() {
+        return delegateDependency.takeSnapshot();
+    }
+
+    @Override
+    public void setSchemaContext(final SchemaContext schemaContext) {
+        delegateDependency.setSchemaContext(schemaContext);
+    }
+
+    @Override
+    public void commit(final DataTreeCandidate dataTreeCandidate) {
+        LOG.trace("Commit detected");
+        delegateDependency.commit(dataTreeCandidate);
+        LOG.debug("Delegate commit successful. Persisting data");
+
+        // FIXME doing full read and full write might not be the fastest way of persisting data here
+        final DataTreeSnapshot dataTreeSnapshot = delegateDependency.takeSnapshot();
+
+        // TODO this can be handled in background by a dedicated thread + a limited blocking queue
+        // TODO enable configurable granularity for persists. Maybe doing write on every modification is too much
+        // and we could do bulk persist
+        persistCurrentData(dataTreeSnapshot.readNode(YangInstanceIdentifier.EMPTY));
+    }
+
+    private void persistCurrentData(final Optional<NormalizedNode<?, ?>> currentRoot) {
+        if(currentRoot.isPresent()) {
+            try {
+                LOG.trace("Persisting current data: {} into: {}", currentRoot.get(), path);
+                // TODO once we are in static environment, do the writer, streamWriter and NNWriter initialization only once
+                final JsonWriter
+                    jsonWriter = createJsonWriter(Files.newOutputStream(path, StandardOpenOption.CREATE), true);
+                final NormalizedNodeStreamWriter streamWriter = JSONNormalizedNodeStreamWriter
+                    .createNestedWriter(JSONCodecFactory.create(schemaServiceDependency.getGlobalContext()), SchemaPath.ROOT, null, jsonWriter);
+                final NormalizedNodeWriter normalizedNodeWriter =
+                    NormalizedNodeWriter.forStreamWriter(streamWriter, true);
+                jsonWriter.beginObject();
+                writeChildren(normalizedNodeWriter,(ContainerNode) currentRoot.get());
+                jsonWriter.endObject();
+                jsonWriter.flush();
+                LOG.trace("Data persisted successfully in {}", path);
+            } catch (IOException e) {
+                throw new IllegalStateException("Unable to persist current data", e);
+            }
+        } else {
+            LOG.debug("Skipping persistence, since there's no data to persist");
+        }
+    }
+
+    private void writeChildren(final NormalizedNodeWriter nnWriter, final ContainerNode data) throws IOException {
+        for(final DataContainerChild<? extends YangInstanceIdentifier.PathArgument, ?> child : data.getValue()) {
+            nnWriter.write(child);
+        }
+    }
+
+    private JsonWriter createJsonWriter(final OutputStream entityStream, boolean prettyPrint) {
+        if (prettyPrint) {
+            return JsonWriterFactory.createJsonWriter(new OutputStreamWriter(entityStream, Charsets.UTF_8), 2);
+        } else {
+            return JsonWriterFactory.createJsonWriter(new OutputStreamWriter(entityStream, Charsets.UTF_8));
+        }
+    }
+
+    @Override
+    public YangInstanceIdentifier getRootPath() {
+        return delegateDependency.getRootPath();
+    }
+
+    @Override
+    public void validate(final DataTreeModification dataTreeModification) throws DataValidationFailedException {
+        delegateDependency.validate(dataTreeModification);
+    }
+
+    @Override
+    public DataTreeCandidate prepare(
+        final DataTreeModification dataTreeModification) {
+        return delegateDependency.prepare(dataTreeModification);
+    }
+
+    @Override
+    public void close() throws Exception {
+        LOG.trace("Closing {} for {}", getClass().getSimpleName(), path);
+        // NOOP
+    }
+}
diff --git a/v3po/data-impl/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/data/impl/rev160411/PersistingDataTreeAdapterModule.java b/v3po/data-impl/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/data/impl/rev160411/PersistingDataTreeAdapterModule.java
new file mode 100644
index 000000000..aa57f4d9b
--- /dev/null
+++ b/v3po/data-impl/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/data/impl/rev160411/PersistingDataTreeAdapterModule.java
@@ -0,0 +1,32 @@
+package org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.data.impl.rev160411;
+
+import java.nio.file.Paths;
+
+public class PersistingDataTreeAdapterModule extends
+    org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.data.impl.rev160411.AbstractPersistingDataTreeAdapterModule {
+    public PersistingDataTreeAdapterModule(org.opendaylight.controller.config.api.ModuleIdentifier identifier,
+                                           org.opendaylight.controller.config.api.DependencyResolver dependencyResolver) {
+        super(identifier, dependencyResolver);
+    }
+
+    public PersistingDataTreeAdapterModule(org.opendaylight.controller.config.api.ModuleIdentifier identifier,
+                                           org.opendaylight.controller.config.api.DependencyResolver dependencyResolver,
+                                           org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.data.impl.rev160411.PersistingDataTreeAdapterModule oldModule,
+                                           java.lang.AutoCloseable oldInstance) {
+        super(identifier, dependencyResolver, oldModule, oldInstance);
+    }
+
+    @Override
+    public void customValidation() {
+        // add custom validation form module attributes here.
+    }
+
+    @Override
+    public java.lang.AutoCloseable createInstance() {
+        return new io.fd.honeycomb.v3po.data.impl.PersistingDataTreeAdapter(
+            getDelegateDependency(),
+            getSchemaServiceDependency(),
+            Paths.get(getPersistFilePath()));
+    }
+
+}
diff --git a/v3po/data-impl/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/data/impl/rev160411/PersistingDataTreeAdapterModuleFactory.java b/v3po/data-impl/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/data/impl/rev160411/PersistingDataTreeAdapterModuleFactory.java
new file mode 100644
index 000000000..0b7546c65
--- /dev/null
+++ b/v3po/data-impl/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/data/impl/rev160411/PersistingDataTreeAdapterModuleFactory.java
@@ -0,0 +1,13 @@
+/*
+* Generated file
+*
+* Generated from: yang module name: data-impl yang module local name: persisting-data-tree-adapter
+* Generated by: org.opendaylight.controller.config.yangjmxgenerator.plugin.JMXGenerator
+* Generated at: Thu May 12 14:07:24 CEST 2016
+*
+* Do not modify this file unless it is present under src/main directory
+*/
+package org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.data.impl.rev160411;
+public class PersistingDataTreeAdapterModuleFactory extends org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.data.impl.rev160411.AbstractPersistingDataTreeAdapterModuleFactory {
+
+}
diff --git a/v3po/data-impl/src/main/yang/data-impl.yang b/v3po/data-impl/src/main/yang/data-impl.yang
index 5275e6556..fa6c6f06d 100644
--- a/v3po/data-impl/src/main/yang/data-impl.yang
+++ b/v3po/data-impl/src/main/yang/data-impl.yang
@@ -44,6 +44,40 @@ module data-impl {
         }
     }
 
+    identity persisting-data-tree-adapter {
+        base config:module-type;
+        config:provided-service data-tree;
+        config:java-name-prefix PersistingDataTreeAdapter;
+    }
+
+    augment "/config:modules/config:module/config:configuration" {
+        case persisting-data-tree-adapter {
+            when "/config:modules/config:module/config:type = 'persisting-data-tree-adapter'";
+
+            container delegate {
+                uses config:service-ref {
+                    refine type {
+                        mandatory true;
+                        config:required-identity data-tree;
+                    }
+                }
+            }
+
+            container schema-service {
+                uses config:service-ref {
+                    refine type {
+                        mandatory true;
+                        config:required-identity dom:schema-service;
+                    }
+                }
+            }
+
+            leaf persist-file-path {
+                type string;
+            }
+        }
+    }
+
     identity honeycomb-config-data-tree {
         base config:module-type;
         config:provided-service dapi:honeycomb-modifiable-data-tree;
diff --git a/v3po/data-impl/src/test/java/io/fd/honeycomb/v3po/data/impl/PersistingDataTreeAdapterTest.java b/v3po/data-impl/src/test/java/io/fd/honeycomb/v3po/data/impl/PersistingDataTreeAdapterTest.java
new file mode 100644
index 000000000..986d7cfb1
--- /dev/null
+++ b/v3po/data-impl/src/test/java/io/fd/honeycomb/v3po/data/impl/PersistingDataTreeAdapterTest.java
@@ -0,0 +1,137 @@
+/*
+ * Copyright (c) 2016 Cisco and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.fd.honeycomb.v3po.data.impl;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import com.google.common.io.ByteStreams;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.opendaylight.controller.sal.core.api.model.SchemaService;
+import org.opendaylight.yangtools.yang.common.QName;
+import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
+import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
+import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTree;
+import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidate;
+import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeSnapshot;
+import org.opendaylight.yangtools.yang.data.impl.schema.Builders;
+import org.opendaylight.yangtools.yang.data.impl.schema.ImmutableNodes;
+import org.opendaylight.yangtools.yang.parser.stmt.reactor.CrossSourceStatementReactor;
+import org.opendaylight.yangtools.yang.parser.stmt.rfc6020.YangInferencePipeline;
+import org.opendaylight.yangtools.yang.parser.stmt.rfc6020.YangStatementSourceImpl;
+import org.opendaylight.yangtools.yang.parser.stmt.rfc6020.effective.EffectiveSchemaContext;
+
+public class PersistingDataTreeAdapterTest {
+
+    public static final String NAMESPACE = "urn:opendaylight:params:xml:ns:yang:test:persistence";
+
+    // The root QNAME can be anything, onyl its children are iterated
+    private static final QName ROOT_QNAME = QName.create("random",  "data");
+    private static final QName TOP_CONTAINER_NAME = QName.create(NAMESPACE, "2015-01-05", "top-container");
+    private static final QName STRING_LEAF_QNAME = QName.create(TOP_CONTAINER_NAME, "string");
+
+    @Mock
+    private DataTree delegatingDataTree;
+    @Mock
+    private SchemaService schemaService;
+    @Mock
+    private DataTreeSnapshot snapshot;
+
+    private Path tmpPersistFile;
+
+    private PersistingDataTreeAdapter persistingDataTreeAdapter;
+
+    @Before
+    public void setUp() throws Exception {
+        MockitoAnnotations.initMocks(this);
+        tmpPersistFile = Files.createTempFile("testing-hc-persistence", "json");
+
+        // Build test yang schemas
+        final CrossSourceStatementReactor.BuildAction buildAction = YangInferencePipeline.RFC6020_REACTOR.newBuild();
+        buildAction.addSource(new YangStatementSourceImpl(getClass().getResourceAsStream("/test-persistence.yang")));
+        final EffectiveSchemaContext effectiveSchemaContext = buildAction.buildEffective();
+        doReturn(effectiveSchemaContext).when(schemaService).getGlobalContext();
+
+        persistingDataTreeAdapter = new PersistingDataTreeAdapter(delegatingDataTree, schemaService, tmpPersistFile);
+    }
+
+    @Test
+    public void testPersist() throws Exception {
+        doReturn(snapshot).when(delegatingDataTree).takeSnapshot();
+
+        NormalizedNode<?, ?> data = getData("testing");
+        doReturn(com.google.common.base.Optional.of(data)).when(snapshot).readNode(YangInstanceIdentifier.EMPTY);
+        persistingDataTreeAdapter.commit(null);
+        assertTrue(Files.exists(tmpPersistFile));
+
+        String persisted = new String(Files.readAllBytes(tmpPersistFile));
+        String expected =
+            new String(ByteStreams.toByteArray(getClass().getResourceAsStream("/expected-persisted-output.txt")));
+
+        assertEquals(expected, persisted);
+
+        data = getData("testing2");
+        doReturn(com.google.common.base.Optional.of(data)).when(snapshot).readNode(YangInstanceIdentifier.EMPTY);
+        persistingDataTreeAdapter.commit(null);
+
+        verify(delegatingDataTree, times(2)).commit(null);
+
+        persisted = new String(Files.readAllBytes(tmpPersistFile));
+        assertEquals(expected.replace("testing", "testing2"), persisted);
+
+        persistingDataTreeAdapter.close();
+
+        // File has to stay even after close
+        assertTrue(Files.exists(tmpPersistFile));
+    }
+
+    @Test
+    public void testNoPersistOnFailure() throws Exception {
+        doThrow(new IllegalStateException("testing errors")).when(delegatingDataTree).commit(any(DataTreeCandidate.class));
+
+        try {
+            persistingDataTreeAdapter.commit(null);
+            fail("Exception expected");
+        } catch (IllegalStateException e) {
+            assertFalse(Files.exists(tmpPersistFile));
+            verify(delegatingDataTree, times(0)).takeSnapshot();
+            verify(delegatingDataTree).commit(any(DataTreeCandidate.class));
+        }
+    }
+
+    private NormalizedNode<?, ?> getData(final String stringValue) {
+        return Builders.containerBuilder()
+                .withNodeIdentifier(new YangInstanceIdentifier.NodeIdentifier(ROOT_QNAME))
+                .withChild(Builders.containerBuilder()
+                    .withNodeIdentifier(new YangInstanceIdentifier.NodeIdentifier(TOP_CONTAINER_NAME))
+                    .withChild(ImmutableNodes.leafNode(STRING_LEAF_QNAME, stringValue))
+                    .build())
+                .build();
+    }
+}
\ No newline at end of file
diff --git a/v3po/data-impl/src/test/resources/expected-persisted-output.txt b/v3po/data-impl/src/test/resources/expected-persisted-output.txt
new file mode 100644
index 000000000..fb21d61e8
--- /dev/null
+++ b/v3po/data-impl/src/test/resources/expected-persisted-output.txt
@@ -0,0 +1,5 @@
+{
+  "test-persistence:top-container": {
+    "string": "testing"
+  }
+}
\ No newline at end of file
diff --git a/v3po/data-impl/src/test/resources/test-persistence.yang b/v3po/data-impl/src/test/resources/test-persistence.yang
new file mode 100644
index 000000000..b7dbbb1bc
--- /dev/null
+++ b/v3po/data-impl/src/test/resources/test-persistence.yang
@@ -0,0 +1,16 @@
+module test-persistence {
+    yang-version 1;
+    namespace "urn:opendaylight:params:xml:ns:yang:test:persistence";
+    prefix "tp";
+
+    revision "2015-01-05" {
+        description "Initial revision";
+    }
+
+    container top-container {
+        leaf string {
+            type string;
+        }
+    }
+
+}