diff options
author | 2016-05-12 16:05:46 +0200 | |
---|---|---|
committer | 2016-05-23 09:23:40 +0000 | |
commit | 0fbc1362777869101c375006d6db423ec868924b (patch) | |
tree | 674fb4e2b1f92db7451b58cec228729da201bf80 /v3po/data-impl/src | |
parent | ff272ca4bb768f9075fc3dcf9a2a2e8ef8be38fa (diff) |
HONEYCOMB-61: Config persister
Add PersistingDataTree adapter for in memory config data tree
Using JSON NormalizedNode writers from ODL
Change-Id: Ida91fe6aa34aaeaedcd061ba1551afe49bbddbbb
Signed-off-by: Maros Marsalek <mmarsale@cisco.com>
Diffstat (limited to 'v3po/data-impl/src')
8 files changed, 422 insertions, 1 deletions
diff --git a/v3po/data-impl/src/main/java/io/fd/honeycomb/v3po/data/impl/ConfigDataTree.java b/v3po/data-impl/src/main/java/io/fd/honeycomb/v3po/data/impl/ConfigDataTree.java index ab4cf6446..1e118ff2a 100644 --- a/v3po/data-impl/src/main/java/io/fd/honeycomb/v3po/data/impl/ConfigDataTree.java +++ b/v3po/data-impl/src/main/java/io/fd/honeycomb/v3po/data/impl/ConfigDataTree.java @@ -109,7 +109,6 @@ public final class ConfigDataTree implements ModifiableDataTree { final Map<InstanceIdentifier<?>, DataObject> nodesAfter = extractNetconfData(normalizedDataAfter); LOG.debug("ConfigDataTree.modify() extracted nodesAfter={}", nodesAfter.keySet()); - final DOMDataReadOnlyTransaction beforeTx = new ReadOnlyTransaction(EMPTY_OPERATIONAL, takeSnapshot()); final ConfigSnapshot modificationSnapshot = new ConfigSnapshot(modification); final DOMDataReadOnlyTransaction afterTx = new ReadOnlyTransaction(EMPTY_OPERATIONAL, modificationSnapshot); diff --git a/v3po/data-impl/src/main/java/io/fd/honeycomb/v3po/data/impl/PersistingDataTreeAdapter.java b/v3po/data-impl/src/main/java/io/fd/honeycomb/v3po/data/impl/PersistingDataTreeAdapter.java new file mode 100644 index 000000000..25d1d8dda --- /dev/null +++ b/v3po/data-impl/src/main/java/io/fd/honeycomb/v3po/data/impl/PersistingDataTreeAdapter.java @@ -0,0 +1,185 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fd.honeycomb.v3po.data.impl; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.common.base.Charsets; +import com.google.common.base.Optional; +import com.google.gson.stream.JsonWriter; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import javax.annotation.Nonnull; +import org.opendaylight.controller.sal.core.api.model.SchemaService; +import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier; +import org.opendaylight.yangtools.yang.data.api.schema.ContainerNode; +import org.opendaylight.yangtools.yang.data.api.schema.DataContainerChild; +import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode; +import org.opendaylight.yangtools.yang.data.api.schema.stream.NormalizedNodeStreamWriter; +import org.opendaylight.yangtools.yang.data.api.schema.stream.NormalizedNodeWriter; +import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTree; +import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidate; +import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeModification; +import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeSnapshot; +import org.opendaylight.yangtools.yang.data.api.schema.tree.DataValidationFailedException; +import org.opendaylight.yangtools.yang.data.codec.gson.JSONCodecFactory; +import org.opendaylight.yangtools.yang.data.codec.gson.JSONNormalizedNodeStreamWriter; +import org.opendaylight.yangtools.yang.data.codec.gson.JsonWriterFactory; +import org.opendaylight.yangtools.yang.model.api.SchemaContext; +import org.opendaylight.yangtools.yang.model.api.SchemaPath; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Adapter for a DataTree that stores current state of data in backing DataTree on each successful commit. + * Uses JSON format. + */ +public class PersistingDataTreeAdapter implements org.opendaylight.yangtools.yang.data.api.schema.tree.DataTree, AutoCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(PersistingDataTreeAdapter.class); + + private final DataTree delegateDependency; + private SchemaService schemaServiceDependency; + private final Path path; + + /** + * Create new Persisting DataTree adapter + * + * @param delegateDependency backing data tree that actually handles all the operations + * @param persistPath path to a file (existing or not) to be used as storage for persistence. Full control over + * a file at peristPath is expected + * @param schemaServiceDependency schemaContext provier + */ + public PersistingDataTreeAdapter(@Nonnull final DataTree delegateDependency, + @Nonnull final SchemaService schemaServiceDependency, + @Nonnull final Path persistPath) { + this.path = testPersistPath(persistPath); + this.delegateDependency = delegateDependency; + this.schemaServiceDependency = schemaServiceDependency; + } + + /** + * Test whether file at persistPath is a file and can be created/deleted + */ + private Path testPersistPath(final Path persistPath) { + try { + checkArgument(!Files.isDirectory(persistPath), "Path %s points to a directory", persistPath); + Files.createDirectories(persistPath.getParent()); + Files.write(persistPath, new byte[]{}, StandardOpenOption.CREATE); + } catch (IOException | UnsupportedOperationException e) { + LOG.warn("Provided path for persistence: {} is not usable", persistPath, e); + throw new IllegalArgumentException("Path " + persistPath + " cannot be used as "); + } finally { + try { + Files.delete(persistPath); + } catch (IOException e) { + LOG.warn("Unable to delete file at {}", persistPath, e); + } + } + + return persistPath; + } + + @Override + public DataTreeSnapshot takeSnapshot() { + return delegateDependency.takeSnapshot(); + } + + @Override + public void setSchemaContext(final SchemaContext schemaContext) { + delegateDependency.setSchemaContext(schemaContext); + } + + @Override + public void commit(final DataTreeCandidate dataTreeCandidate) { + LOG.trace("Commit detected"); + delegateDependency.commit(dataTreeCandidate); + LOG.debug("Delegate commit successful. Persisting data"); + + // FIXME doing full read and full write might not be the fastest way of persisting data here + final DataTreeSnapshot dataTreeSnapshot = delegateDependency.takeSnapshot(); + + // TODO this can be handled in background by a dedicated thread + a limited blocking queue + // TODO enable configurable granularity for persists. Maybe doing write on every modification is too much + // and we could do bulk persist + persistCurrentData(dataTreeSnapshot.readNode(YangInstanceIdentifier.EMPTY)); + } + + private void persistCurrentData(final Optional<NormalizedNode<?, ?>> currentRoot) { + if(currentRoot.isPresent()) { + try { + LOG.trace("Persisting current data: {} into: {}", currentRoot.get(), path); + // TODO once we are in static environment, do the writer, streamWriter and NNWriter initialization only once + final JsonWriter + jsonWriter = createJsonWriter(Files.newOutputStream(path, StandardOpenOption.CREATE), true); + final NormalizedNodeStreamWriter streamWriter = JSONNormalizedNodeStreamWriter + .createNestedWriter(JSONCodecFactory.create(schemaServiceDependency.getGlobalContext()), SchemaPath.ROOT, null, jsonWriter); + final NormalizedNodeWriter normalizedNodeWriter = + NormalizedNodeWriter.forStreamWriter(streamWriter, true); + jsonWriter.beginObject(); + writeChildren(normalizedNodeWriter,(ContainerNode) currentRoot.get()); + jsonWriter.endObject(); + jsonWriter.flush(); + LOG.trace("Data persisted successfully in {}", path); + } catch (IOException e) { + throw new IllegalStateException("Unable to persist current data", e); + } + } else { + LOG.debug("Skipping persistence, since there's no data to persist"); + } + } + + private void writeChildren(final NormalizedNodeWriter nnWriter, final ContainerNode data) throws IOException { + for(final DataContainerChild<? extends YangInstanceIdentifier.PathArgument, ?> child : data.getValue()) { + nnWriter.write(child); + } + } + + private JsonWriter createJsonWriter(final OutputStream entityStream, boolean prettyPrint) { + if (prettyPrint) { + return JsonWriterFactory.createJsonWriter(new OutputStreamWriter(entityStream, Charsets.UTF_8), 2); + } else { + return JsonWriterFactory.createJsonWriter(new OutputStreamWriter(entityStream, Charsets.UTF_8)); + } + } + + @Override + public YangInstanceIdentifier getRootPath() { + return delegateDependency.getRootPath(); + } + + @Override + public void validate(final DataTreeModification dataTreeModification) throws DataValidationFailedException { + delegateDependency.validate(dataTreeModification); + } + + @Override + public DataTreeCandidate prepare( + final DataTreeModification dataTreeModification) { + return delegateDependency.prepare(dataTreeModification); + } + + @Override + public void close() throws Exception { + LOG.trace("Closing {} for {}", getClass().getSimpleName(), path); + // NOOP + } +} diff --git a/v3po/data-impl/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/data/impl/rev160411/PersistingDataTreeAdapterModule.java b/v3po/data-impl/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/data/impl/rev160411/PersistingDataTreeAdapterModule.java new file mode 100644 index 000000000..aa57f4d9b --- /dev/null +++ b/v3po/data-impl/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/data/impl/rev160411/PersistingDataTreeAdapterModule.java @@ -0,0 +1,32 @@ +package org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.data.impl.rev160411; + +import java.nio.file.Paths; + +public class PersistingDataTreeAdapterModule extends + org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.data.impl.rev160411.AbstractPersistingDataTreeAdapterModule { + public PersistingDataTreeAdapterModule(org.opendaylight.controller.config.api.ModuleIdentifier identifier, + org.opendaylight.controller.config.api.DependencyResolver dependencyResolver) { + super(identifier, dependencyResolver); + } + + public PersistingDataTreeAdapterModule(org.opendaylight.controller.config.api.ModuleIdentifier identifier, + org.opendaylight.controller.config.api.DependencyResolver dependencyResolver, + org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.data.impl.rev160411.PersistingDataTreeAdapterModule oldModule, + java.lang.AutoCloseable oldInstance) { + super(identifier, dependencyResolver, oldModule, oldInstance); + } + + @Override + public void customValidation() { + // add custom validation form module attributes here. + } + + @Override + public java.lang.AutoCloseable createInstance() { + return new io.fd.honeycomb.v3po.data.impl.PersistingDataTreeAdapter( + getDelegateDependency(), + getSchemaServiceDependency(), + Paths.get(getPersistFilePath())); + } + +} diff --git a/v3po/data-impl/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/data/impl/rev160411/PersistingDataTreeAdapterModuleFactory.java b/v3po/data-impl/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/data/impl/rev160411/PersistingDataTreeAdapterModuleFactory.java new file mode 100644 index 000000000..0b7546c65 --- /dev/null +++ b/v3po/data-impl/src/main/java/org/opendaylight/yang/gen/v1/urn/honeycomb/params/xml/ns/yang/data/impl/rev160411/PersistingDataTreeAdapterModuleFactory.java @@ -0,0 +1,13 @@ +/* +* Generated file +* +* Generated from: yang module name: data-impl yang module local name: persisting-data-tree-adapter +* Generated by: org.opendaylight.controller.config.yangjmxgenerator.plugin.JMXGenerator +* Generated at: Thu May 12 14:07:24 CEST 2016 +* +* Do not modify this file unless it is present under src/main directory +*/ +package org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.data.impl.rev160411; +public class PersistingDataTreeAdapterModuleFactory extends org.opendaylight.yang.gen.v1.urn.honeycomb.params.xml.ns.yang.data.impl.rev160411.AbstractPersistingDataTreeAdapterModuleFactory { + +} diff --git a/v3po/data-impl/src/main/yang/data-impl.yang b/v3po/data-impl/src/main/yang/data-impl.yang index 5275e6556..fa6c6f06d 100644 --- a/v3po/data-impl/src/main/yang/data-impl.yang +++ b/v3po/data-impl/src/main/yang/data-impl.yang @@ -44,6 +44,40 @@ module data-impl { } } + identity persisting-data-tree-adapter { + base config:module-type; + config:provided-service data-tree; + config:java-name-prefix PersistingDataTreeAdapter; + } + + augment "/config:modules/config:module/config:configuration" { + case persisting-data-tree-adapter { + when "/config:modules/config:module/config:type = 'persisting-data-tree-adapter'"; + + container delegate { + uses config:service-ref { + refine type { + mandatory true; + config:required-identity data-tree; + } + } + } + + container schema-service { + uses config:service-ref { + refine type { + mandatory true; + config:required-identity dom:schema-service; + } + } + } + + leaf persist-file-path { + type string; + } + } + } + identity honeycomb-config-data-tree { base config:module-type; config:provided-service dapi:honeycomb-modifiable-data-tree; diff --git a/v3po/data-impl/src/test/java/io/fd/honeycomb/v3po/data/impl/PersistingDataTreeAdapterTest.java b/v3po/data-impl/src/test/java/io/fd/honeycomb/v3po/data/impl/PersistingDataTreeAdapterTest.java new file mode 100644 index 000000000..986d7cfb1 --- /dev/null +++ b/v3po/data-impl/src/test/java/io/fd/honeycomb/v3po/data/impl/PersistingDataTreeAdapterTest.java @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2016 Cisco and/or its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.fd.honeycomb.v3po.data.impl; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import com.google.common.io.ByteStreams; +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.opendaylight.controller.sal.core.api.model.SchemaService; +import org.opendaylight.yangtools.yang.common.QName; +import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier; +import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode; +import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTree; +import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidate; +import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeSnapshot; +import org.opendaylight.yangtools.yang.data.impl.schema.Builders; +import org.opendaylight.yangtools.yang.data.impl.schema.ImmutableNodes; +import org.opendaylight.yangtools.yang.parser.stmt.reactor.CrossSourceStatementReactor; +import org.opendaylight.yangtools.yang.parser.stmt.rfc6020.YangInferencePipeline; +import org.opendaylight.yangtools.yang.parser.stmt.rfc6020.YangStatementSourceImpl; +import org.opendaylight.yangtools.yang.parser.stmt.rfc6020.effective.EffectiveSchemaContext; + +public class PersistingDataTreeAdapterTest { + + public static final String NAMESPACE = "urn:opendaylight:params:xml:ns:yang:test:persistence"; + + // The root QNAME can be anything, onyl its children are iterated + private static final QName ROOT_QNAME = QName.create("random", "data"); + private static final QName TOP_CONTAINER_NAME = QName.create(NAMESPACE, "2015-01-05", "top-container"); + private static final QName STRING_LEAF_QNAME = QName.create(TOP_CONTAINER_NAME, "string"); + + @Mock + private DataTree delegatingDataTree; + @Mock + private SchemaService schemaService; + @Mock + private DataTreeSnapshot snapshot; + + private Path tmpPersistFile; + + private PersistingDataTreeAdapter persistingDataTreeAdapter; + + @Before + public void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + tmpPersistFile = Files.createTempFile("testing-hc-persistence", "json"); + + // Build test yang schemas + final CrossSourceStatementReactor.BuildAction buildAction = YangInferencePipeline.RFC6020_REACTOR.newBuild(); + buildAction.addSource(new YangStatementSourceImpl(getClass().getResourceAsStream("/test-persistence.yang"))); + final EffectiveSchemaContext effectiveSchemaContext = buildAction.buildEffective(); + doReturn(effectiveSchemaContext).when(schemaService).getGlobalContext(); + + persistingDataTreeAdapter = new PersistingDataTreeAdapter(delegatingDataTree, schemaService, tmpPersistFile); + } + + @Test + public void testPersist() throws Exception { + doReturn(snapshot).when(delegatingDataTree).takeSnapshot(); + + NormalizedNode<?, ?> data = getData("testing"); + doReturn(com.google.common.base.Optional.of(data)).when(snapshot).readNode(YangInstanceIdentifier.EMPTY); + persistingDataTreeAdapter.commit(null); + assertTrue(Files.exists(tmpPersistFile)); + + String persisted = new String(Files.readAllBytes(tmpPersistFile)); + String expected = + new String(ByteStreams.toByteArray(getClass().getResourceAsStream("/expected-persisted-output.txt"))); + + assertEquals(expected, persisted); + + data = getData("testing2"); + doReturn(com.google.common.base.Optional.of(data)).when(snapshot).readNode(YangInstanceIdentifier.EMPTY); + persistingDataTreeAdapter.commit(null); + + verify(delegatingDataTree, times(2)).commit(null); + + persisted = new String(Files.readAllBytes(tmpPersistFile)); + assertEquals(expected.replace("testing", "testing2"), persisted); + + persistingDataTreeAdapter.close(); + + // File has to stay even after close + assertTrue(Files.exists(tmpPersistFile)); + } + + @Test + public void testNoPersistOnFailure() throws Exception { + doThrow(new IllegalStateException("testing errors")).when(delegatingDataTree).commit(any(DataTreeCandidate.class)); + + try { + persistingDataTreeAdapter.commit(null); + fail("Exception expected"); + } catch (IllegalStateException e) { + assertFalse(Files.exists(tmpPersistFile)); + verify(delegatingDataTree, times(0)).takeSnapshot(); + verify(delegatingDataTree).commit(any(DataTreeCandidate.class)); + } + } + + private NormalizedNode<?, ?> getData(final String stringValue) { + return Builders.containerBuilder() + .withNodeIdentifier(new YangInstanceIdentifier.NodeIdentifier(ROOT_QNAME)) + .withChild(Builders.containerBuilder() + .withNodeIdentifier(new YangInstanceIdentifier.NodeIdentifier(TOP_CONTAINER_NAME)) + .withChild(ImmutableNodes.leafNode(STRING_LEAF_QNAME, stringValue)) + .build()) + .build(); + } +}
\ No newline at end of file diff --git a/v3po/data-impl/src/test/resources/expected-persisted-output.txt b/v3po/data-impl/src/test/resources/expected-persisted-output.txt new file mode 100644 index 000000000..fb21d61e8 --- /dev/null +++ b/v3po/data-impl/src/test/resources/expected-persisted-output.txt @@ -0,0 +1,5 @@ +{ + "test-persistence:top-container": { + "string": "testing" + } +}
\ No newline at end of file diff --git a/v3po/data-impl/src/test/resources/test-persistence.yang b/v3po/data-impl/src/test/resources/test-persistence.yang new file mode 100644 index 000000000..b7dbbb1bc --- /dev/null +++ b/v3po/data-impl/src/test/resources/test-persistence.yang @@ -0,0 +1,16 @@ +module test-persistence { + yang-version 1; + namespace "urn:opendaylight:params:xml:ns:yang:test:persistence"; + prefix "tp"; + + revision "2015-01-05" { + description "Initial revision"; + } + + container top-container { + leaf string { + type string; + } + } + +} |