Merge tag 'upstream/18.11-rc2' into 18.11.x

New upstream version 18.11-rc2

Change-Id: I5ef3516f5bef8875ef584d1e68b4893c21eb1f8e
Signed-off-by: Luca Boccassi <luca.boccassi@gmail.com>

101 files changed, 8419 insertions, 1785 deletions

diff --git a/examples/Makefile b/examples/Makefile
index 481720cb..33fe0e58 100644
--- a/examples/Makefile
+++ b/examples/Makefile
@@ -17,6 +17,7 @@ DIRS-$(CONFIG_RTE_LIBRTE_DISTRIBUTOR) += distributor
 DIRS-y += ethtool
 DIRS-y += exception_path
 DIRS-$(CONFIG_RTE_LIBRTE_EFD) += server_node_efd
+DIRS-$(CONFIG_RTE_LIBRTE_CRYPTODEV) += fips_validation
 DIRS-$(CONFIG_RTE_LIBRTE_FLOW_CLASSIFY) += flow_classify
 DIRS-y += flow_filtering
 DIRS-y += helloworld
@@ -65,7 +66,7 @@ ifeq ($(CONFIG_RTE_LIBRTE_HASH),y)
 DIRS-$(CONFIG_RTE_LIBRTE_VHOST) += tep_termination
 endif
 DIRS-$(CONFIG_RTE_LIBRTE_TIMER) += timer
-DIRS-$(CONFIG_RTE_LIBRTE_VHOST) += vhost vhost_scsi
+DIRS-$(CONFIG_RTE_LIBRTE_VHOST) += vhost vhost_scsi vdpa
 ifeq ($(CONFIG_RTE_LIBRTE_CRYPTODEV),y)
 DIRS-$(CONFIG_RTE_LIBRTE_VHOST) += vhost_crypto
 endif
diff --git a/examples/bbdev_app/main.c b/examples/bbdev_app/main.c
index 045a190b..d68c06ae 100644
--- a/examples/bbdev_app/main.c
+++ b/examples/bbdev_app/main.c
@@ -64,7 +64,6 @@ static const struct rte_eth_conf port_conf = {
 		.mq_mode = ETH_MQ_RX_NONE,
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.txmode = {
 		.mq_mode = ETH_MQ_TX_NONE,
diff --git a/examples/bond/main.c b/examples/bond/main.c
index 23d0981a..b282e68b 100644
--- a/examples/bond/main.c
+++ b/examples/bond/main.c
@@ -122,7 +122,6 @@ static struct rte_eth_conf port_conf = {
 		.mq_mode = ETH_MQ_RX_NONE,
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.rx_adv_conf = {
 		.rss_conf = {
diff --git a/examples/cmdline/Makefile b/examples/cmdline/Makefile
index 7893c85b..a617cce1 100644
--- a/examples/cmdline/Makefile
+++ b/examples/cmdline/Makefile
@@ -56,7 +56,6 @@ SRCS-y := main.c commands.c parse_obj_list.c
 
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
-CFLAGS_parse_obj_list.o := -D_GNU_SOURCE
 
 include $(RTE_SDK)/mk/rte.extapp.mk
 
diff --git a/examples/ethtool/ethtool-app/Makefile b/examples/ethtool/ethtool-app/Makefile
index 4cd9efdd..9ecfc0b8 100644
--- a/examples/ethtool/ethtool-app/Makefile
+++ b/examples/ethtool/ethtool-app/Makefile
@@ -16,7 +16,7 @@ APP = ethtool
 # all source are stored in SRCS-y
 SRCS-y := main.c ethapp.c
 
-CFLAGS += -O3 -D_GNU_SOURCE -pthread -I$(SRCDIR)/../lib
+CFLAGS += -O3 -pthread -I$(SRCDIR)/../lib
 CFLAGS += $(WERROR_FLAGS)
 
 LDLIBS += -L$(subst ethtool-app,lib,$(RTE_OUTPUT))/lib
diff --git a/examples/eventdev_pipeline/main.c b/examples/eventdev_pipeline/main.c
index 700bc696..92e08bc0 100644
--- a/examples/eventdev_pipeline/main.c
+++ b/examples/eventdev_pipeline/main.c
@@ -26,20 +26,6 @@ core_in_use(unsigned int lcore_id) {
 		fdata->tx_core[lcore_id] || fdata->worker_core[lcore_id]);
 }
 
-static void
-eth_tx_buffer_retry(struct rte_mbuf **pkts, uint16_t unsent,
-			void *userdata)
-{
-	int port_id = (uintptr_t) userdata;
-	unsigned int _sent = 0;
-
-	do {
-		/* Note: hard-coded TX queue */
-		_sent += rte_eth_tx_burst(port_id, 0, &pkts[_sent],
-					  unsent - _sent);
-	} while (_sent != unsent);
-}
-
 /*
  * Parse the coremask given as argument (hexadecimal string) and fill
  * the global configuration (core role and core count) with the parsed
@@ -263,6 +249,7 @@ parse_app_args(int argc, char **argv)
 static inline int
 port_init(uint8_t port, struct rte_mempool *mbuf_pool)
 {
+	struct rte_eth_rxconf rx_conf;
 	static const struct rte_eth_conf port_conf_default = {
 		.rxmode = {
 			.mq_mode = ETH_MQ_RX_RSS,
@@ -291,6 +278,8 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool)
 	if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
 		port_conf.txmode.offloads |=
 			DEV_TX_OFFLOAD_MBUF_FAST_FREE;
+	rx_conf = dev_info.default_rxconf;
+	rx_conf.offloads = port_conf.rxmode.offloads;
 
 	port_conf.rx_adv_conf.rss_conf.rss_hf &=
 		dev_info.flow_type_rss_offloads;
@@ -311,7 +300,8 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool)
 	/* Allocate and set up 1 RX queue per Ethernet port. */
 	for (q = 0; q < rx_rings; q++) {
 		retval = rte_eth_rx_queue_setup(port, q, rx_ring_size,
-				rte_eth_dev_socket_id(port), NULL, mbuf_pool);
+				rte_eth_dev_socket_id(port), &rx_conf,
+				mbuf_pool);
 		if (retval < 0)
 			return retval;
 	}
@@ -350,7 +340,7 @@ port_init(uint8_t port, struct rte_mempool *mbuf_pool)
 static int
 init_ports(uint16_t num_ports)
 {
-	uint16_t portid, i;
+	uint16_t portid;
 
 	if (!cdata.num_mbuf)
 		cdata.num_mbuf = 16384 * num_ports;
@@ -367,36 +357,26 @@ init_ports(uint16_t num_ports)
 			rte_exit(EXIT_FAILURE, "Cannot init port %"PRIu16 "\n",
 					portid);
 
-	RTE_ETH_FOREACH_DEV(i) {
-		void *userdata = (void *)(uintptr_t) i;
-		fdata->tx_buf[i] =
-			rte_malloc(NULL, RTE_ETH_TX_BUFFER_SIZE(32), 0);
-		if (fdata->tx_buf[i] == NULL)
-			rte_panic("Out of memory\n");
-		rte_eth_tx_buffer_init(fdata->tx_buf[i], 32);
-		rte_eth_tx_buffer_set_err_callback(fdata->tx_buf[i],
-						   eth_tx_buffer_retry,
-						   userdata);
-	}
-
 	return 0;
 }
 
 static void
 do_capability_setup(uint8_t eventdev_id)
 {
+	int ret;
 	uint16_t i;
-	uint8_t mt_unsafe = 0;
+	uint8_t generic_pipeline = 0;
 	uint8_t burst = 0;
 
 	RTE_ETH_FOREACH_DEV(i) {
-		struct rte_eth_dev_info dev_info;
-		memset(&dev_info, 0, sizeof(struct rte_eth_dev_info));
-
-		rte_eth_dev_info_get(i, &dev_info);
-		/* Check if it is safe ask worker to tx. */
-		mt_unsafe |= !(dev_info.tx_offload_capa &
-				DEV_TX_OFFLOAD_MT_LOCKFREE);
+		uint32_t caps = 0;
+
+		ret = rte_event_eth_tx_adapter_caps_get(eventdev_id, i, &caps);
+		if (ret)
+			rte_exit(EXIT_FAILURE,
+				"Invalid capability for Tx adptr port %d\n", i);
+		generic_pipeline |= !(caps &
+				RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT);
 	}
 
 	struct rte_event_dev_info eventdev_info;
@@ -406,21 +386,42 @@ do_capability_setup(uint8_t eventdev_id)
 	burst = eventdev_info.event_dev_cap & RTE_EVENT_DEV_CAP_BURST_MODE ? 1 :
 		0;
 
-	if (mt_unsafe)
+	if (generic_pipeline)
 		set_worker_generic_setup_data(&fdata->cap, burst);
 	else
-		set_worker_tx_setup_data(&fdata->cap, burst);
+		set_worker_tx_enq_setup_data(&fdata->cap, burst);
 }
 
 static void
 signal_handler(int signum)
 {
+	static uint8_t once;
+	uint16_t portid;
+
 	if (fdata->done)
 		rte_exit(1, "Exiting on signal %d\n", signum);
-	if (signum == SIGINT || signum == SIGTERM) {
+	if ((signum == SIGINT || signum == SIGTERM) && !once) {
 		printf("\n\nSignal %d received, preparing to exit...\n",
 				signum);
+		if (cdata.dump_dev)
+			rte_event_dev_dump(0, stdout);
+		once = 1;
 		fdata->done = 1;
+		rte_smp_wmb();
+
+		RTE_ETH_FOREACH_DEV(portid) {
+			rte_event_eth_rx_adapter_stop(portid);
+			rte_event_eth_tx_adapter_stop(portid);
+			rte_eth_dev_stop(portid);
+		}
+
+		rte_eal_mp_wait_lcore();
+
+		RTE_ETH_FOREACH_DEV(portid) {
+			rte_eth_dev_close(portid);
+		}
+
+		rte_event_dev_close(0);
 	}
 	if (signum == SIGTSTP)
 		rte_event_dev_dump(0, stdout);
@@ -499,7 +500,7 @@ main(int argc, char **argv)
 	if (worker_data == NULL)
 		rte_panic("rte_calloc failed\n");
 
-	int dev_id = fdata->cap.evdev_setup(&cons_data, worker_data);
+	int dev_id = fdata->cap.evdev_setup(worker_data);
 	if (dev_id < 0)
 		rte_exit(EXIT_FAILURE, "Error setting up eventdev\n");
 
@@ -524,8 +525,8 @@ main(int argc, char **argv)
 
 		if (fdata->tx_core[lcore_id])
 			printf(
-				"[%s()] lcore %d executing NIC Tx, and using eventdev port %u\n",
-				__func__, lcore_id, cons_data.port_id);
+				"[%s()] lcore %d executing NIC Tx\n",
+				__func__, lcore_id);
 
 		if (fdata->sched_core[lcore_id])
 			printf("[%s()] lcore %d executing scheduler\n",
@@ -555,9 +556,6 @@ main(int argc, char **argv)
 
 	rte_eal_mp_wait_lcore();
 
-	if (cdata.dump_dev)
-		rte_event_dev_dump(dev_id, stdout);
-
 	if (!cdata.quiet && (port_stat(dev_id, worker_data[0].port_id) !=
 			(uint64_t)-ENOTSUP)) {
 		printf("\nPort Workload distribution:\n");
diff --git a/examples/eventdev_pipeline/pipeline_common.h b/examples/eventdev_pipeline/pipeline_common.h
index 9703396f..a6cc912f 100644
--- a/examples/eventdev_pipeline/pipeline_common.h
+++ b/examples/eventdev_pipeline/pipeline_common.h
@@ -16,6 +16,7 @@
 #include <rte_ethdev.h>
 #include <rte_eventdev.h>
 #include <rte_event_eth_rx_adapter.h>
+#include <rte_event_eth_tx_adapter.h>
 #include <rte_service.h>
 #include <rte_service_component.h>
 
@@ -23,38 +24,30 @@
 #define BATCH_SIZE 16
 #define MAX_NUM_CORE 64
 
-struct cons_data {
-	uint8_t dev_id;
-	uint8_t port_id;
-	uint8_t release;
-} __rte_cache_aligned;
-
 struct worker_data {
 	uint8_t dev_id;
 	uint8_t port_id;
 } __rte_cache_aligned;
 
 typedef int (*worker_loop)(void *);
-typedef int (*consumer_loop)(void);
 typedef void (*schedule_loop)(unsigned int);
-typedef int (*eventdev_setup)(struct cons_data *, struct worker_data *);
-typedef void (*rx_adapter_setup)(uint16_t nb_ports);
+typedef int (*eventdev_setup)(struct worker_data *);
+typedef void (*adapter_setup)(uint16_t nb_ports);
 typedef void (*opt_check)(void);
 
 struct setup_data {
 	worker_loop worker;
-	consumer_loop consumer;
 	schedule_loop scheduler;
 	eventdev_setup evdev_setup;
-	rx_adapter_setup adptr_setup;
+	adapter_setup adptr_setup;
 	opt_check check_opt;
 };
 
 struct fastpath_data {
 	volatile int done;
-	uint32_t tx_lock;
 	uint32_t evdev_service_id;
 	uint32_t rxadptr_service_id;
+	uint32_t txadptr_service_id;
 	bool rx_single;
 	bool tx_single;
 	bool sched_single;
@@ -62,7 +55,6 @@ struct fastpath_data {
 	unsigned int tx_core[MAX_NUM_CORE];
 	unsigned int sched_core[MAX_NUM_CORE];
 	unsigned int worker_core[MAX_NUM_CORE];
-	struct rte_eth_dev_tx_buffer *tx_buf[RTE_MAX_ETHPORTS];
 	struct setup_data cap;
 } __rte_cache_aligned;
 
@@ -88,6 +80,8 @@ struct config_data {
 	int16_t next_qid[MAX_NUM_STAGES+2];
 	int16_t qid[MAX_NUM_STAGES];
 	uint8_t rx_adapter_id;
+	uint8_t tx_adapter_id;
+	uint8_t tx_queue_id;
 	uint64_t worker_lcore_mask;
 	uint64_t rx_lcore_mask;
 	uint64_t tx_lcore_mask;
@@ -99,8 +93,6 @@ struct port_link {
 	uint8_t priority;
 };
 
-struct cons_data cons_data;
-
 struct fastpath_data *fdata;
 struct config_data cdata;
 
@@ -142,12 +134,11 @@ schedule_devices(unsigned int lcore_id)
 		}
 	}
 
-	if (fdata->tx_core[lcore_id] && (fdata->tx_single ||
-			 rte_atomic32_cmpset(&(fdata->tx_lock), 0, 1))) {
-		fdata->cap.consumer();
-		rte_atomic32_clear((rte_atomic32_t *)&(fdata->tx_lock));
+	if (fdata->tx_core[lcore_id]) {
+		rte_service_run_iter_on_app_lcore(fdata->txadptr_service_id,
+				!fdata->tx_single);
 	}
 }
 
 void set_worker_generic_setup_data(struct setup_data *caps, bool burst);
-void set_worker_tx_setup_data(struct setup_data *caps, bool burst);
+void set_worker_tx_enq_setup_data(struct setup_data *caps, bool burst);
diff --git a/examples/eventdev_pipeline/pipeline_worker_generic.c b/examples/eventdev_pipeline/pipeline_worker_generic.c
index 2215e9eb..16906494 100644
--- a/examples/eventdev_pipeline/pipeline_worker_generic.c
+++ b/examples/eventdev_pipeline/pipeline_worker_generic.c
@@ -119,153 +119,13 @@ worker_generic_burst(void *arg)
 	return 0;
 }
 
-static __rte_always_inline int
-consumer(void)
-{
-	const uint64_t freq_khz = rte_get_timer_hz() / 1000;
-	struct rte_event packet;
-
-	static uint64_t received;
-	static uint64_t last_pkts;
-	static uint64_t last_time;
-	static uint64_t start_time;
-	int i;
-	uint8_t dev_id = cons_data.dev_id;
-	uint8_t port_id = cons_data.port_id;
-
-	do {
-		uint16_t n = rte_event_dequeue_burst(dev_id, port_id,
-				&packet, 1, 0);
-
-		if (n == 0) {
-			RTE_ETH_FOREACH_DEV(i)
-				rte_eth_tx_buffer_flush(i, 0, fdata->tx_buf[i]);
-			return 0;
-		}
-		if (start_time == 0)
-			last_time = start_time = rte_get_timer_cycles();
-
-		received++;
-		uint8_t outport = packet.mbuf->port;
-
-		exchange_mac(packet.mbuf);
-		rte_eth_tx_buffer(outport, 0, fdata->tx_buf[outport],
-				packet.mbuf);
-
-		if (cons_data.release)
-			rte_event_enqueue_burst(dev_id, port_id,
-								&packet, n);
-
-		/* Print out mpps every 1<22 packets */
-		if (!cdata.quiet && received >= last_pkts + (1<<22)) {
-			const uint64_t now = rte_get_timer_cycles();
-			const uint64_t total_ms = (now - start_time) / freq_khz;
-			const uint64_t delta_ms = (now - last_time) / freq_khz;
-			uint64_t delta_pkts = received - last_pkts;
-
-			printf("# %s RX=%"PRIu64", time %"PRIu64 "ms, "
-					"avg %.3f mpps [current %.3f mpps]\n",
-					__func__,
-					received,
-					total_ms,
-					received / (total_ms * 1000.0),
-					delta_pkts / (delta_ms * 1000.0));
-			last_pkts = received;
-			last_time = now;
-		}
-
-		cdata.num_packets--;
-		if (cdata.num_packets <= 0)
-			fdata->done = 1;
-	/* Be stuck in this loop if single. */
-	} while (!fdata->done && fdata->tx_single);
-
-	return 0;
-}
-
-static __rte_always_inline int
-consumer_burst(void)
-{
-	const uint64_t freq_khz = rte_get_timer_hz() / 1000;
-	struct rte_event packets[BATCH_SIZE];
-
-	static uint64_t received;
-	static uint64_t last_pkts;
-	static uint64_t last_time;
-	static uint64_t start_time;
-	unsigned int i, j;
-	uint8_t dev_id = cons_data.dev_id;
-	uint8_t port_id = cons_data.port_id;
-
-	do {
-		uint16_t n = rte_event_dequeue_burst(dev_id, port_id,
-				packets, RTE_DIM(packets), 0);
-
-		if (n == 0) {
-			RTE_ETH_FOREACH_DEV(j)
-				rte_eth_tx_buffer_flush(j, 0, fdata->tx_buf[j]);
-			return 0;
-		}
-		if (start_time == 0)
-			last_time = start_time = rte_get_timer_cycles();
-
-		received += n;
-		for (i = 0; i < n; i++) {
-			uint8_t outport = packets[i].mbuf->port;
-
-			exchange_mac(packets[i].mbuf);
-			rte_eth_tx_buffer(outport, 0, fdata->tx_buf[outport],
-					packets[i].mbuf);
-
-			packets[i].op = RTE_EVENT_OP_RELEASE;
-		}
-
-		if (cons_data.release) {
-			uint16_t nb_tx;
-
-			nb_tx = rte_event_enqueue_burst(dev_id, port_id,
-								packets, n);
-			while (nb_tx < n)
-				nb_tx += rte_event_enqueue_burst(dev_id,
-						port_id, packets + nb_tx,
-						n - nb_tx);
-		}
-
-		/* Print out mpps every 1<22 packets */
-		if (!cdata.quiet && received >= last_pkts + (1<<22)) {
-			const uint64_t now = rte_get_timer_cycles();
-			const uint64_t total_ms = (now - start_time) / freq_khz;
-			const uint64_t delta_ms = (now - last_time) / freq_khz;
-			uint64_t delta_pkts = received - last_pkts;
-
-			printf("# consumer RX=%"PRIu64", time %"PRIu64 "ms, "
-					"avg %.3f mpps [current %.3f mpps]\n",
-					received,
-					total_ms,
-					received / (total_ms * 1000.0),
-					delta_pkts / (delta_ms * 1000.0));
-			last_pkts = received;
-			last_time = now;
-		}
-
-		cdata.num_packets -= n;
-		if (cdata.num_packets <= 0)
-			fdata->done = 1;
-	/* Be stuck in this loop if single. */
-	} while (!fdata->done && fdata->tx_single);
-
-	return 0;
-}
-
 static int
-setup_eventdev_generic(struct cons_data *cons_data,
-		struct worker_data *worker_data)
+setup_eventdev_generic(struct worker_data *worker_data)
 {
 	const uint8_t dev_id = 0;
 	/* +1 stages is for a SINGLE_LINK TX stage */
 	const uint8_t nb_queues = cdata.num_stages + 1;
-	/* + 1 is one port for consumer */
-	const uint8_t nb_ports = cdata.num_workers + 1;
+	const uint8_t nb_ports = cdata.num_workers;
 	struct rte_event_dev_config config = {
 			.nb_event_queues = nb_queues,
 			.nb_event_ports = nb_ports,
@@ -285,11 +145,6 @@ setup_eventdev_generic(struct cons_data *cons_data,
 			.nb_atomic_flows = 1024,
 		.nb_atomic_order_sequences = 1024,
 	};
-	struct rte_event_port_conf tx_p_conf = {
-			.dequeue_depth = 128,
-			.enqueue_depth = 128,
-			.new_event_threshold = 4096,
-	};
 	struct rte_event_queue_conf tx_q_conf = {
 			.priority = RTE_EVENT_DEV_PRIORITY_HIGHEST,
 			.event_queue_cfg = RTE_EVENT_QUEUE_CFG_SINGLE_LINK,
@@ -297,7 +152,6 @@ setup_eventdev_generic(struct cons_data *cons_data,
 
 	struct port_link worker_queues[MAX_NUM_STAGES];
 	uint8_t disable_implicit_release;
-	struct port_link tx_queue;
 	unsigned int i;
 
 	int ret, ndev = rte_event_dev_count();
@@ -314,7 +168,6 @@ setup_eventdev_generic(struct cons_data *cons_data,
 			RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE);
 
 	wkr_p_conf.disable_implicit_release = disable_implicit_release;
-	tx_p_conf.disable_implicit_release = disable_implicit_release;
 
 	if (dev_info.max_event_port_dequeue_depth <
 			config.nb_event_port_dequeue_depth)
@@ -372,8 +225,7 @@ setup_eventdev_generic(struct cons_data *cons_data,
 		printf("%d: error creating qid %d\n", __LINE__, i);
 		return -1;
 	}
-	tx_queue.queue_id = i;
-	tx_queue.priority = RTE_EVENT_DEV_PRIORITY_HIGHEST;
+	cdata.tx_queue_id = i;
 
 	if (wkr_p_conf.dequeue_depth > config.nb_event_port_dequeue_depth)
 		wkr_p_conf.dequeue_depth = config.nb_event_port_dequeue_depth;
@@ -403,26 +255,6 @@ setup_eventdev_generic(struct cons_data *cons_data,
 		w->port_id = i;
 	}
 
-	if (tx_p_conf.dequeue_depth > config.nb_event_port_dequeue_depth)
-		tx_p_conf.dequeue_depth = config.nb_event_port_dequeue_depth;
-	if (tx_p_conf.enqueue_depth > config.nb_event_port_enqueue_depth)
-		tx_p_conf.enqueue_depth = config.nb_event_port_enqueue_depth;
-
-	/* port for consumer, linked to TX queue */
-	if (rte_event_port_setup(dev_id, i, &tx_p_conf) < 0) {
-		printf("Error setting up port %d\n", i);
-		return -1;
-	}
-	if (rte_event_port_link(dev_id, i, &tx_queue.queue_id,
-				&tx_queue.priority, 1) != 1) {
-		printf("%d: error creating link for port %d\n",
-				__LINE__, i);
-		return -1;
-	}
-	*cons_data = (struct cons_data){.dev_id = dev_id,
-					.port_id = i,
-					.release = disable_implicit_release };
-
 	ret = rte_event_dev_service_id_get(dev_id,
 				&fdata->evdev_service_id);
 	if (ret != -ESRCH && ret != 0) {
@@ -431,76 +263,107 @@ setup_eventdev_generic(struct cons_data *cons_data,
 	}
 	rte_service_runstate_set(fdata->evdev_service_id, 1);
 	rte_service_set_runstate_mapped_check(fdata->evdev_service_id, 0);
-	if (rte_event_dev_start(dev_id) < 0) {
-		printf("Error starting eventdev\n");
-		return -1;
-	}
 
 	return dev_id;
 }
 
 static void
-init_rx_adapter(uint16_t nb_ports)
+init_adapters(uint16_t nb_ports)
 {
 	int i;
 	int ret;
+	uint8_t tx_port_id = 0;
 	uint8_t evdev_id = 0;
 	struct rte_event_dev_info dev_info;
 
 	ret = rte_event_dev_info_get(evdev_id, &dev_info);
 
-	struct rte_event_port_conf rx_p_conf = {
-		.dequeue_depth = 8,
-		.enqueue_depth = 8,
-		.new_event_threshold = 1200,
+	struct rte_event_port_conf adptr_p_conf = {
+		.dequeue_depth = cdata.worker_cq_depth,
+		.enqueue_depth = 64,
+		.new_event_threshold = 4096,
 	};
 
-	if (rx_p_conf.dequeue_depth > dev_info.max_event_port_dequeue_depth)
-		rx_p_conf.dequeue_depth = dev_info.max_event_port_dequeue_depth;
-	if (rx_p_conf.enqueue_depth > dev_info.max_event_port_enqueue_depth)
-		rx_p_conf.enqueue_depth = dev_info.max_event_port_enqueue_depth;
+	if (adptr_p_conf.dequeue_depth > dev_info.max_event_port_dequeue_depth)
+		adptr_p_conf.dequeue_depth =
+			dev_info.max_event_port_dequeue_depth;
+	if (adptr_p_conf.enqueue_depth > dev_info.max_event_port_enqueue_depth)
+		adptr_p_conf.enqueue_depth =
+			dev_info.max_event_port_enqueue_depth;
 
 	/* Create one adapter for all the ethernet ports. */
 	ret = rte_event_eth_rx_adapter_create(cdata.rx_adapter_id, evdev_id,
-			&rx_p_conf);
+			&adptr_p_conf);
 	if (ret)
 		rte_exit(EXIT_FAILURE, "failed to create rx adapter[%d]",
 				cdata.rx_adapter_id);
 
+	ret = rte_event_eth_tx_adapter_create(cdata.tx_adapter_id, evdev_id,
+			&adptr_p_conf);
+	if (ret)
+		rte_exit(EXIT_FAILURE, "failed to create tx adapter[%d]",
+				cdata.tx_adapter_id);
+
 	struct rte_event_eth_rx_adapter_queue_conf queue_conf;
 	memset(&queue_conf, 0, sizeof(queue_conf));
 	queue_conf.ev.sched_type = cdata.queue_type;
 	queue_conf.ev.queue_id = cdata.qid[0];
 
 	for (i = 0; i < nb_ports; i++) {
-		uint32_t cap;
-
-		ret = rte_event_eth_rx_adapter_caps_get(evdev_id, i, &cap);
-		if (ret)
-			rte_exit(EXIT_FAILURE,
-					"failed to get event rx adapter "
-					"capabilities");
-
 		ret = rte_event_eth_rx_adapter_queue_add(cdata.rx_adapter_id, i,
 				-1, &queue_conf);
 		if (ret)
 			rte_exit(EXIT_FAILURE,
 					"Failed to add queues to Rx adapter");
+
+		ret = rte_event_eth_tx_adapter_queue_add(cdata.tx_adapter_id, i,
+				-1);
+		if (ret)
+			rte_exit(EXIT_FAILURE,
+					"Failed to add queues to Tx adapter");
 	}
 
+	ret = rte_event_eth_tx_adapter_event_port_get(cdata.tx_adapter_id,
+			&tx_port_id);
+	if (ret)
+		rte_exit(EXIT_FAILURE,
+				"Failed to get Tx adapter port id");
+	ret = rte_event_port_link(evdev_id, tx_port_id, &cdata.tx_queue_id,
+			NULL, 1);
+	if (ret != 1)
+		rte_exit(EXIT_FAILURE,
+				"Unable to link Tx adapter port to Tx queue");
+
 	ret = rte_event_eth_rx_adapter_service_id_get(cdata.rx_adapter_id,
 				&fdata->rxadptr_service_id);
 	if (ret != -ESRCH && ret != 0) {
 		rte_exit(EXIT_FAILURE,
-			"Error getting the service ID for sw eventdev\n");
+			"Error getting the service ID for Rx adapter\n");
 	}
 	rte_service_runstate_set(fdata->rxadptr_service_id, 1);
 	rte_service_set_runstate_mapped_check(fdata->rxadptr_service_id, 0);
 
+	ret = rte_event_eth_tx_adapter_service_id_get(cdata.tx_adapter_id,
+				&fdata->txadptr_service_id);
+	if (ret != -ESRCH && ret != 0) {
+		rte_exit(EXIT_FAILURE,
+			"Error getting the service ID for Tx adapter\n");
+	}
+	rte_service_runstate_set(fdata->txadptr_service_id, 1);
+	rte_service_set_runstate_mapped_check(fdata->txadptr_service_id, 0);
+
 	ret = rte_event_eth_rx_adapter_start(cdata.rx_adapter_id);
 	if (ret)
 		rte_exit(EXIT_FAILURE, "Rx adapter[%d] start failed",
 				cdata.rx_adapter_id);
+
+	ret = rte_event_eth_tx_adapter_start(cdata.tx_adapter_id);
+	if (ret)
+		rte_exit(EXIT_FAILURE, "Tx adapter[%d] start failed",
+				cdata.tx_adapter_id);
+
+	if (rte_event_dev_start(evdev_id) < 0)
+		rte_exit(EXIT_FAILURE, "Error starting eventdev");
 }
 
 static void
@@ -510,6 +373,7 @@ generic_opt_check(void)
 	int ret;
 	uint32_t cap = 0;
 	uint8_t rx_needed = 0;
+	uint8_t sched_needed = 0;
 	struct rte_event_dev_info eventdev_info;
 
 	memset(&eventdev_info, 0, sizeof(struct rte_event_dev_info));
@@ -519,6 +383,8 @@ generic_opt_check(void)
 				RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES))
 		rte_exit(EXIT_FAILURE,
 				"Event dev doesn't support all type queues\n");
+	sched_needed = !(eventdev_info.event_dev_cap &
+		RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED);
 
 	RTE_ETH_FOREACH_DEV(i) {
 		ret = rte_event_eth_rx_adapter_caps_get(0, i, &cap);
@@ -531,9 +397,8 @@ generic_opt_check(void)
 
 	if (cdata.worker_lcore_mask == 0 ||
 			(rx_needed && cdata.rx_lcore_mask == 0) ||
-			cdata.tx_lcore_mask == 0 || (cdata.sched_lcore_mask == 0
-				&& !(eventdev_info.event_dev_cap &
-					RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED))) {
+			(cdata.tx_lcore_mask == 0) ||
+			(sched_needed && cdata.sched_lcore_mask == 0)) {
 		printf("Core part of pipeline was not assigned any cores. "
 			"This will stall the pipeline, please check core masks "
 			"(use -h for details on setting core masks):\n"
@@ -545,23 +410,24 @@ generic_opt_check(void)
 		rte_exit(-1, "Fix core masks\n");
 	}
 
-	if (eventdev_info.event_dev_cap & RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED)
+	if (!sched_needed)
 		memset(fdata->sched_core, 0,
 				sizeof(unsigned int) * MAX_NUM_CORE);
+	if (!rx_needed)
+		memset(fdata->rx_core, 0,
+				sizeof(unsigned int) * MAX_NUM_CORE);
 }
 
 void
 set_worker_generic_setup_data(struct setup_data *caps, bool burst)
 {
 	if (burst) {
-		caps->consumer = consumer_burst;
 		caps->worker = worker_generic_burst;
 	} else {
-		caps->consumer = consumer;
 		caps->worker = worker_generic;
 	}
 
-	caps->adptr_setup = init_rx_adapter;
+	caps->adptr_setup = init_adapters;
 	caps->scheduler = schedule_devices;
 	caps->evdev_setup = setup_eventdev_generic;
 	caps->check_opt = generic_opt_check;
diff --git a/examples/eventdev_pipeline/pipeline_worker_tx.c b/examples/eventdev_pipeline/pipeline_worker_tx.c
index 3dbde92d..85eb075f 100644
--- a/examples/eventdev_pipeline/pipeline_worker_tx.c
+++ b/examples/eventdev_pipeline/pipeline_worker_tx.c
@@ -36,10 +36,11 @@ worker_event_enqueue_burst(const uint8_t dev, const uint8_t port,
 }
 
 static __rte_always_inline void
-worker_tx_pkt(struct rte_mbuf *mbuf)
+worker_tx_pkt(const uint8_t dev, const uint8_t port, struct rte_event *ev)
 {
-	exchange_mac(mbuf);
-	while (rte_eth_tx_burst(mbuf->port, 0, &mbuf, 1) != 1)
+	exchange_mac(ev->mbuf);
+	rte_event_eth_tx_adapter_txq_set(ev->mbuf, 0);
+	while (!rte_event_eth_tx_adapter_enqueue(dev, port, ev, 1))
 		rte_pause();
 }
 
@@ -64,15 +65,15 @@ worker_do_tx_single(void *arg)
 		received++;
 
 		if (ev.sched_type == RTE_SCHED_TYPE_ATOMIC) {
-			worker_tx_pkt(ev.mbuf);
+			worker_tx_pkt(dev, port, &ev);
 			tx++;
-			continue;
+		} else {
+			work();
+			ev.queue_id++;
+			worker_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC);
+			worker_event_enqueue(dev, port, &ev);
+			fwd++;
 		}
-		work();
-		ev.queue_id++;
-		worker_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC);
-		worker_event_enqueue(dev, port, &ev);
-		fwd++;
 	}
 
 	if (!cdata.quiet)
@@ -100,14 +101,14 @@ worker_do_tx_single_atq(void *arg)
 		received++;
 
 		if (ev.sched_type == RTE_SCHED_TYPE_ATOMIC) {
-			worker_tx_pkt(ev.mbuf);
+			worker_tx_pkt(dev, port, &ev);
 			tx++;
-			continue;
+		} else {
+			work();
+			worker_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC);
+			worker_event_enqueue(dev, port, &ev);
+			fwd++;
 		}
-		work();
-		worker_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC);
-		worker_event_enqueue(dev, port, &ev);
-		fwd++;
 	}
 
 	if (!cdata.quiet)
@@ -141,7 +142,7 @@ worker_do_tx_single_burst(void *arg)
 			rte_prefetch0(ev[i + 1].mbuf);
 			if (ev[i].sched_type == RTE_SCHED_TYPE_ATOMIC) {
 
-				worker_tx_pkt(ev[i].mbuf);
+				worker_tx_pkt(dev, port, &ev[i]);
 				ev[i].op = RTE_EVENT_OP_RELEASE;
 				tx++;
 
@@ -188,7 +189,7 @@ worker_do_tx_single_burst_atq(void *arg)
 			rte_prefetch0(ev[i + 1].mbuf);
 			if (ev[i].sched_type == RTE_SCHED_TYPE_ATOMIC) {
 
-				worker_tx_pkt(ev[i].mbuf);
+				worker_tx_pkt(dev, port, &ev[i]);
 				ev[i].op = RTE_EVENT_OP_RELEASE;
 				tx++;
 			} else
@@ -232,7 +233,7 @@ worker_do_tx(void *arg)
 
 		if (cq_id >= lst_qid) {
 			if (ev.sched_type == RTE_SCHED_TYPE_ATOMIC) {
-				worker_tx_pkt(ev.mbuf);
+				worker_tx_pkt(dev, port, &ev);
 				tx++;
 				continue;
 			}
@@ -280,7 +281,7 @@ worker_do_tx_atq(void *arg)
 
 		if (cq_id == lst_qid) {
 			if (ev.sched_type == RTE_SCHED_TYPE_ATOMIC) {
-				worker_tx_pkt(ev.mbuf);
+				worker_tx_pkt(dev, port, &ev);
 				tx++;
 				continue;
 			}
@@ -330,7 +331,7 @@ worker_do_tx_burst(void *arg)
 
 			if (cq_id >= lst_qid) {
 				if (ev[i].sched_type == RTE_SCHED_TYPE_ATOMIC) {
-					worker_tx_pkt(ev[i].mbuf);
+					worker_tx_pkt(dev, port, &ev[i]);
 					tx++;
 					ev[i].op = RTE_EVENT_OP_RELEASE;
 					continue;
@@ -387,7 +388,7 @@ worker_do_tx_burst_atq(void *arg)
 
 			if (cq_id == lst_qid) {
 				if (ev[i].sched_type == RTE_SCHED_TYPE_ATOMIC) {
-					worker_tx_pkt(ev[i].mbuf);
+					worker_tx_pkt(dev, port, &ev[i]);
 					tx++;
 					ev[i].op = RTE_EVENT_OP_RELEASE;
 					continue;
@@ -413,10 +414,8 @@ worker_do_tx_burst_atq(void *arg)
 }
 
 static int
-setup_eventdev_worker_tx(struct cons_data *cons_data,
-		struct worker_data *worker_data)
+setup_eventdev_worker_tx_enq(struct worker_data *worker_data)
 {
-	RTE_SET_USED(cons_data);
 	uint8_t i;
 	const uint8_t atq = cdata.all_type_queues ? 1 : 0;
 	const uint8_t dev_id = 0;
@@ -575,10 +574,9 @@ setup_eventdev_worker_tx(struct cons_data *cons_data,
 	}
 	rte_service_runstate_set(fdata->evdev_service_id, 1);
 	rte_service_set_runstate_mapped_check(fdata->evdev_service_id, 0);
-	if (rte_event_dev_start(dev_id) < 0) {
-		printf("Error starting eventdev\n");
-		return -1;
-	}
+
+	if (rte_event_dev_start(dev_id) < 0)
+		rte_exit(EXIT_FAILURE, "Error starting eventdev");
 
 	return dev_id;
 }
@@ -602,7 +600,7 @@ service_rx_adapter(void *arg)
 }
 
 static void
-init_rx_adapter(uint16_t nb_ports)
+init_adapters(uint16_t nb_ports)
 {
 	int i;
 	int ret;
@@ -613,17 +611,18 @@ init_rx_adapter(uint16_t nb_ports)
 	ret = rte_event_dev_info_get(evdev_id, &dev_info);
 	adptr_services = rte_zmalloc(NULL, sizeof(struct rx_adptr_services), 0);
 
-	struct rte_event_port_conf rx_p_conf = {
-		.dequeue_depth = 8,
-		.enqueue_depth = 8,
-		.new_event_threshold = 1200,
+	struct rte_event_port_conf adptr_p_conf = {
+		.dequeue_depth = cdata.worker_cq_depth,
+		.enqueue_depth = 64,
+		.new_event_threshold = 4096,
 	};
 
-	if (rx_p_conf.dequeue_depth > dev_info.max_event_port_dequeue_depth)
-		rx_p_conf.dequeue_depth = dev_info.max_event_port_dequeue_depth;
-	if (rx_p_conf.enqueue_depth > dev_info.max_event_port_enqueue_depth)
-		rx_p_conf.enqueue_depth = dev_info.max_event_port_enqueue_depth;
-
+	if (adptr_p_conf.dequeue_depth > dev_info.max_event_port_dequeue_depth)
+		adptr_p_conf.dequeue_depth =
+			dev_info.max_event_port_dequeue_depth;
+	if (adptr_p_conf.enqueue_depth > dev_info.max_event_port_enqueue_depth)
+		adptr_p_conf.enqueue_depth =
+			dev_info.max_event_port_enqueue_depth;
 
 	struct rte_event_eth_rx_adapter_queue_conf queue_conf;
 	memset(&queue_conf, 0, sizeof(queue_conf));
@@ -633,11 +632,11 @@ init_rx_adapter(uint16_t nb_ports)
 		uint32_t cap;
 		uint32_t service_id;
 
-		ret = rte_event_eth_rx_adapter_create(i, evdev_id, &rx_p_conf);
+		ret = rte_event_eth_rx_adapter_create(i, evdev_id,
+				&adptr_p_conf);
 		if (ret)
 			rte_exit(EXIT_FAILURE,
-					"failed to create rx adapter[%d]",
-					cdata.rx_adapter_id);
+					"failed to create rx adapter[%d]", i);
 
 		ret = rte_event_eth_rx_adapter_caps_get(evdev_id, i, &cap);
 		if (ret)
@@ -654,7 +653,6 @@ init_rx_adapter(uint16_t nb_ports)
 			rte_exit(EXIT_FAILURE,
 					"Failed to add queues to Rx adapter");
 
-
 		/* Producer needs to be scheduled. */
 		if (!(cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT)) {
 			ret = rte_event_eth_rx_adapter_service_id_get(i,
@@ -680,9 +678,29 @@ init_rx_adapter(uint16_t nb_ports)
 		ret = rte_event_eth_rx_adapter_start(i);
 		if (ret)
 			rte_exit(EXIT_FAILURE, "Rx adapter[%d] start failed",
-					cdata.rx_adapter_id);
+					i);
 	}
 
+	/* We already know that Tx adapter has INTERNAL port cap*/
+	ret = rte_event_eth_tx_adapter_create(cdata.tx_adapter_id, evdev_id,
+			&adptr_p_conf);
+	if (ret)
+		rte_exit(EXIT_FAILURE, "failed to create tx adapter[%d]",
+				cdata.tx_adapter_id);
+
+	for (i = 0; i < nb_ports; i++) {
+		ret = rte_event_eth_tx_adapter_queue_add(cdata.tx_adapter_id, i,
+				-1);
+		if (ret)
+			rte_exit(EXIT_FAILURE,
+					"Failed to add queues to Tx adapter");
+	}
+
+	ret = rte_event_eth_tx_adapter_start(cdata.tx_adapter_id);
+	if (ret)
+		rte_exit(EXIT_FAILURE, "Tx adapter[%d] start failed",
+				cdata.tx_adapter_id);
+
 	if (adptr_services->nb_rx_adptrs) {
 		struct rte_service_spec service;
 
@@ -695,8 +713,7 @@ init_rx_adapter(uint16_t nb_ports)
 				&fdata->rxadptr_service_id);
 		if (ret)
 			rte_exit(EXIT_FAILURE,
-				"Rx adapter[%d] service register failed",
-				cdata.rx_adapter_id);
+				"Rx adapter service register failed");
 
 		rte_service_runstate_set(fdata->rxadptr_service_id, 1);
 		rte_service_component_runstate_set(fdata->rxadptr_service_id,
@@ -708,23 +725,19 @@ init_rx_adapter(uint16_t nb_ports)
 		rte_free(adptr_services);
 	}
 
-	if (!adptr_services->nb_rx_adptrs && fdata->cap.consumer == NULL &&
-			(dev_info.event_dev_cap &
+	if (!adptr_services->nb_rx_adptrs && (dev_info.event_dev_cap &
 			 RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED))
 		fdata->cap.scheduler = NULL;
-
-	if (dev_info.event_dev_cap & RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED)
-		memset(fdata->sched_core, 0,
-				sizeof(unsigned int) * MAX_NUM_CORE);
 }
 
 static void
-worker_tx_opt_check(void)
+worker_tx_enq_opt_check(void)
 {
 	int i;
 	int ret;
 	uint32_t cap = 0;
 	uint8_t rx_needed = 0;
+	uint8_t sched_needed = 0;
 	struct rte_event_dev_info eventdev_info;
 
 	memset(&eventdev_info, 0, sizeof(struct rte_event_dev_info));
@@ -734,32 +747,38 @@ worker_tx_opt_check(void)
 				RTE_EVENT_DEV_CAP_QUEUE_ALL_TYPES))
 		rte_exit(EXIT_FAILURE,
 				"Event dev doesn't support all type queues\n");
+	sched_needed = !(eventdev_info.event_dev_cap &
+		RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED);
 
 	RTE_ETH_FOREACH_DEV(i) {
 		ret = rte_event_eth_rx_adapter_caps_get(0, i, &cap);
 		if (ret)
 			rte_exit(EXIT_FAILURE,
-					"failed to get event rx adapter "
-					"capabilities");
+				"failed to get event rx adapter capabilities");
 		rx_needed |=
 			!(cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT);
 	}
 
 	if (cdata.worker_lcore_mask == 0 ||
 			(rx_needed && cdata.rx_lcore_mask == 0) ||
-			(cdata.sched_lcore_mask == 0 &&
-			 !(eventdev_info.event_dev_cap &
-				 RTE_EVENT_DEV_CAP_DISTRIBUTED_SCHED))) {
+			(sched_needed && cdata.sched_lcore_mask == 0)) {
 		printf("Core part of pipeline was not assigned any cores. "
 			"This will stall the pipeline, please check core masks "
 			"(use -h for details on setting core masks):\n"
-			"\trx: %"PRIu64"\n\ttx: %"PRIu64"\n\tsched: %"PRIu64
-			"\n\tworkers: %"PRIu64"\n",
-			cdata.rx_lcore_mask, cdata.tx_lcore_mask,
-			cdata.sched_lcore_mask,
-			cdata.worker_lcore_mask);
+			"\trx: %"PRIu64"\n\tsched: %"PRIu64
+			"\n\tworkers: %"PRIu64"\n", cdata.rx_lcore_mask,
+			cdata.sched_lcore_mask, cdata.worker_lcore_mask);
 		rte_exit(-1, "Fix core masks\n");
 	}
+
+	if (!sched_needed)
+		memset(fdata->sched_core, 0,
+				sizeof(unsigned int) * MAX_NUM_CORE);
+	if (!rx_needed)
+		memset(fdata->rx_core, 0,
+				sizeof(unsigned int) * MAX_NUM_CORE);
+
+	memset(fdata->tx_core, 0, sizeof(unsigned int) * MAX_NUM_CORE);
 }
 
 static worker_loop
@@ -821,18 +840,15 @@ get_worker_multi_stage(bool burst)
 }
 
 void
-set_worker_tx_setup_data(struct setup_data *caps, bool burst)
+set_worker_tx_enq_setup_data(struct setup_data *caps, bool burst)
 {
 	if (cdata.num_stages == 1)
 		caps->worker = get_worker_single_stage(burst);
 	else
 		caps->worker = get_worker_multi_stage(burst);
 
-	memset(fdata->tx_core, 0, sizeof(unsigned int) * MAX_NUM_CORE);
-
-	caps->check_opt = worker_tx_opt_check;
-	caps->consumer = NULL;
+	caps->check_opt = worker_tx_enq_opt_check;
 	caps->scheduler = schedule_devices;
-	caps->evdev_setup = setup_eventdev_worker_tx;
-	caps->adptr_setup = init_rx_adapter;
+	caps->evdev_setup = setup_eventdev_worker_tx_enq;
+	caps->adptr_setup = init_adapters;
 }
diff --git a/examples/exception_path/main.c b/examples/exception_path/main.c
index 440422bc..4180a868 100644
--- a/examples/exception_path/main.c
+++ b/examples/exception_path/main.c
@@ -87,9 +87,6 @@
 
 /* Options for configuring ethernet port */
 static struct rte_eth_conf port_conf = {
-	.rxmode = {
-		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
-	},
 	.txmode = {
 		.mq_mode = ETH_MQ_TX_NONE,
 	},
diff --git a/examples/fips_validation/Makefile b/examples/fips_validation/Makefile
new file mode 100644
index 00000000..7b1fe34a
--- /dev/null
+++ b/examples/fips_validation/Makefile
@@ -0,0 +1,75 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+# binary name
+APP = fips_validation
+
+# all source are stored in SRCS-y
+SRCS-y := fips_validation.c
+SRCS-y += fips_validation_aes.c
+SRCS-y += fips_validation_hmac.c
+SRCS-y += fips_validation_tdes.c
+SRCS-y += fips_validation_gcm.c
+SRCS-y += fips_validation_cmac.c
+SRCS-y += fips_validation_ccm.c
+SRCS-y += main.c
+
+# Build using pkg-config variables if possible
+$(shell pkg-config --exists libdpdk)
+ifeq ($(.SHELLSTATUS),0)
+
+all: shared
+.PHONY: shared static
+shared: build/$(APP)-shared
+	ln -sf $(APP)-shared build/$(APP)
+static: build/$(APP)-static
+	ln -sf $(APP)-static build/$(APP)
+
+PC_FILE := $(shell pkg-config --path libdpdk)
+CFLAGS += -O3 $(shell pkg-config --cflags libdpdk)
+LDFLAGS_SHARED = $(shell pkg-config --libs libdpdk)
+LDFLAGS_STATIC = -Wl,-Bstatic $(shell pkg-config --static --libs libdpdk)
+
+build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build
+	$(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED)
+
+build/$(APP)-static: $(SRCS-y) Makefile $(PC_FILE) | build
+	$(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_STATIC)
+
+build:
+	@mkdir -p $@
+
+.PHONY: clean
+clean:
+	rm -f build/$(APP) build/$(APP)-static build/$(APP)-shared
+	rmdir --ignore-fail-on-non-empty build
+
+else
+
+ifeq ($(RTE_SDK),)
+$(error "Please define RTE_SDK environment variable")
+endif
+
+# Default target, can be overridden by command line or environment
+RTE_TARGET ?= x86_64-native-linuxapp-gcc
+
+INC += $(sort $(wildcard *.h))
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+CFLAGS += $(WERROR_FLAGS)
+
+# workaround for a gcc bug with noreturn attribute
+# http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603
+ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y)
+CFLAGS_main.o += -Wno-return-type
+endif
+
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+CFLAGS += -I$(SRCDIR)
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+
+include $(RTE_SDK)/mk/rte.extapp.mk
+
+endif
diff --git a/examples/fips_validation/fips_validation.c b/examples/fips_validation/fips_validation.c
new file mode 100644
index 00000000..a835cc3f
--- /dev/null
+++ b/examples/fips_validation/fips_validation.c
@@ -0,0 +1,595 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <stdio.h>
+#include <string.h>
+
+#include <rte_string_fns.h>
+#include <rte_cryptodev.h>
+#include <rte_malloc.h>
+
+#include "fips_validation.h"
+
+#define skip_white_spaces(pos)			\
+({						\
+	__typeof__(pos) _p = (pos);		\
+	for ( ; isspace(*_p); _p++)		\
+		;				\
+	_p;					\
+})
+
+static int
+get_file_line(void)
+{
+	FILE *fp = info.fp_rd;
+	char *line = info.one_line_text;
+	int ret;
+	uint32_t loc = 0;
+
+	memset(line, 0, MAX_LINE_CHAR);
+	while ((ret = fgetc(fp)) != EOF) {
+		char c = (char)ret;
+
+		if (loc >= MAX_LINE_CHAR - 1)
+			return -ENOMEM;
+		if (c == '\n')
+			break;
+		line[loc++] = c;
+	}
+
+	if (ret == EOF)
+		return -EOF;
+
+	return 0;
+}
+
+int
+fips_test_fetch_one_block(void)
+{
+	size_t size;
+	int ret = 0;
+	uint32_t i;
+
+	for (i = 0; i < info.nb_vec_lines; i++) {
+		free(info.vec[i]);
+		info.vec[i] = NULL;
+	}
+
+	i = 0;
+	do {
+		if (i >= MAX_LINE_PER_VECTOR) {
+			ret = -ENOMEM;
+			goto error_exit;
+		}
+
+		ret = get_file_line();
+		size = strlen(info.one_line_text);
+		if (size == 0)
+			break;
+
+		info.vec[i] = calloc(1, size + 5);
+		if (info.vec[i] == NULL)
+			goto error_exit;
+
+		strlcpy(info.vec[i], info.one_line_text, size + 1);
+		i++;
+	} while (ret == 0);
+
+	info.nb_vec_lines = i;
+
+	return ret;
+
+error_exit:
+	for (i = 0; i < MAX_LINE_PER_VECTOR; i++)
+		if (info.vec[i] != NULL) {
+			free(info.vec[i]);
+			info.vec[i] = NULL;
+		}
+
+	info.nb_vec_lines = 0;
+
+	return -ENOMEM;
+}
+
+static int
+fips_test_parse_header(void)
+{
+	uint32_t i;
+	char *tmp;
+	int ret;
+	time_t t = time(NULL);
+	struct tm *tm_now = localtime(&t);
+
+	ret = fips_test_fetch_one_block();
+	if (ret < 0)
+		return ret;
+
+	for (i = 0; i < info.nb_vec_lines; i++) {
+		if (strstr(info.vec[i], "AESVS")) {
+			info.algo = FIPS_TEST_ALGO_AES;
+			ret = parse_test_aes_init();
+			if (ret < 0)
+				return ret;
+		} else if (strstr(info.vec[i], "GCM")) {
+			info.algo = FIPS_TEST_ALGO_AES_GCM;
+			ret = parse_test_gcm_init();
+			if (ret < 0)
+				return ret;
+		} else if (strstr(info.vec[i], "CMAC")) {
+			info.algo = FIPS_TEST_ALGO_AES_CMAC;
+			ret = parse_test_cmac_init();
+			if (ret < 0)
+				return 0;
+		} else if (strstr(info.vec[i], "CCM")) {
+			info.algo = FIPS_TEST_ALGO_AES_CCM;
+			ret = parse_test_ccm_init();
+			if (ret < 0)
+				return 0;
+		} else if (strstr(info.vec[i], "HMAC")) {
+			info.algo = FIPS_TEST_ALGO_HMAC;
+			ret = parse_test_hmac_init();
+			if (ret < 0)
+				return ret;
+		} else if (strstr(info.vec[i], "TDES")) {
+			info.algo = FIPS_TEST_ALGO_TDES;
+			ret = parse_test_tdes_init();
+			if (ret < 0)
+				return 0;
+		}
+
+		tmp = strstr(info.vec[i], "# Config info for ");
+		if (tmp != NULL) {
+			fprintf(info.fp_wr, "%s%s\n", "# Config info for DPDK Cryptodev ",
+					info.device_name);
+			continue;
+		}
+
+		tmp = strstr(info.vec[i], "#  HMAC information for ");
+		if (tmp != NULL) {
+			fprintf(info.fp_wr, "%s%s\n", "#  HMAC information for "
+				"DPDK Cryptodev ",
+				info.device_name);
+			continue;
+		}
+
+		tmp = strstr(info.vec[i], "# Config Info for : ");
+		if (tmp != NULL) {
+
+			fprintf(info.fp_wr, "%s%s\n", "# Config Info for DPDK Cryptodev : ",
+					info.device_name);
+			continue;
+		}
+
+		tmp = strstr(info.vec[i], "# information for ");
+		if (tmp != NULL) {
+
+			char tmp_output[128] = {0};
+
+			strlcpy(tmp_output, info.vec[i], tmp - info.vec[i] + 1);
+
+			fprintf(info.fp_wr, "%s%s%s\n", tmp_output,
+					"information for DPDK Cryptodev ",
+					info.device_name);
+			continue;
+		}
+
+		tmp = strstr(info.vec[i], " test information for ");
+		if (tmp != NULL) {
+			char tmp_output[128] = {0};
+
+			strlcpy(tmp_output, info.vec[i], tmp - info.vec[i] + 1);
+
+			fprintf(info.fp_wr, "%s%s%s\n", tmp_output,
+					"test information for DPDK Cryptodev ",
+					info.device_name);
+			continue;
+		}
+
+		if (i == info.nb_vec_lines - 1) {
+			/** update the time as current time, write to file */
+			fprintf(info.fp_wr, "%s%s\n", "# Generated on ",
+					asctime(tm_now));
+			continue;
+		}
+
+		/* to this point, no field need to update,
+		 *  only copy to rsp file
+		 */
+		fprintf(info.fp_wr, "%s\n", info.vec[i]);
+	}
+
+	return 0;
+}
+
+static int
+parse_file_type(const char *path)
+{
+	const char *tmp = path + strlen(path) - 3;
+
+	if (strstr(tmp, REQ_FILE_PERFIX))
+		info.file_type = FIPS_TYPE_REQ;
+	else if (strstr(tmp, RSP_FILE_PERFIX))
+		info.file_type = FIPS_TYPE_RSP;
+	else if (strstr(path, FAX_FILE_PERFIX))
+		info.file_type = FIPS_TYPE_FAX;
+	else
+		return -EINVAL;
+
+	return 0;
+}
+
+int
+fips_test_init(const char *req_file_path, const char *rsp_file_path,
+		const char *device_name)
+{
+	if (strcmp(req_file_path, rsp_file_path) == 0) {
+		RTE_LOG(ERR, USER1, "File paths cannot be the same\n");
+		return -EINVAL;
+	}
+
+	fips_test_clear();
+
+	info.algo = FIPS_TEST_ALGO_MAX;
+	if (parse_file_type(req_file_path) < 0) {
+		RTE_LOG(ERR, USER1, "File %s type not supported\n",
+				req_file_path);
+		return -EINVAL;
+	}
+
+	info.fp_rd = fopen(req_file_path, "r");
+	if (!info.fp_rd) {
+		RTE_LOG(ERR, USER1, "Cannot open file %s\n", req_file_path);
+		return -EINVAL;
+	}
+
+	info.fp_wr = fopen(rsp_file_path, "w");
+	if (!info.fp_wr) {
+		RTE_LOG(ERR, USER1, "Cannot open file %s\n", rsp_file_path);
+		return -EINVAL;
+	}
+
+	info.one_line_text = calloc(1, MAX_LINE_CHAR);
+	if (!info.one_line_text) {
+		RTE_LOG(ERR, USER1, "Insufficient memory\n");
+		return -ENOMEM;
+	}
+
+	strlcpy(info.device_name, device_name, sizeof(info.device_name));
+
+	if (fips_test_parse_header() < 0) {
+		RTE_LOG(ERR, USER1, "Failed parsing header\n");
+		return -1;
+	}
+
+	return 0;
+}
+
+void
+fips_test_clear(void)
+{
+	if (info.fp_rd)
+		fclose(info.fp_rd);
+	if (info.fp_wr)
+		fclose(info.fp_wr);
+	if (info.one_line_text)
+		free(info.one_line_text);
+	if (info.nb_vec_lines) {
+		uint32_t i;
+
+		for (i = 0; i < info.nb_vec_lines; i++)
+			free(info.vec[i]);
+	}
+
+	memset(&info, 0, sizeof(info));
+}
+
+int
+fips_test_parse_one_case(void)
+{
+	uint32_t i, j = 0;
+	uint32_t is_interim = 0;
+	int ret;
+
+	if (info.interim_callbacks) {
+		for (i = 0; i < info.nb_vec_lines; i++) {
+			for (j = 0; info.interim_callbacks[j].key != NULL; j++)
+				if (strstr(info.vec[i],
+					info.interim_callbacks[j].key)) {
+					is_interim = 1;
+
+					ret = info.interim_callbacks[j].cb(
+						info.interim_callbacks[j].key,
+						info.vec[i],
+						info.interim_callbacks[j].val);
+					if (ret < 0)
+						return ret;
+				}
+		}
+	}
+
+	if (is_interim) {
+		for (i = 0; i < info.nb_vec_lines; i++)
+			fprintf(info.fp_wr, "%s\n", info.vec[i]);
+		fprintf(info.fp_wr, "\n");
+		return 1;
+	}
+
+	for (i = 0; i < info.nb_vec_lines; i++) {
+		for (j = 0; info.callbacks[j].key != NULL; j++)
+			if (strstr(info.vec[i], info.callbacks[j].key)) {
+				ret = info.callbacks[j].cb(
+					info.callbacks[j].key,
+					info.vec[i], info.callbacks[j].val);
+				if (ret < 0)
+					return ret;
+				break;
+			}
+	}
+
+	return 0;
+}
+
+void
+fips_test_write_one_case(void)
+{
+	uint32_t i;
+
+	for (i = 0; i < info.nb_vec_lines; i++)
+		fprintf(info.fp_wr, "%s\n", info.vec[i]);
+}
+
+static int
+parser_read_uint64_hex(uint64_t *value, const char *p)
+{
+	char *next;
+	uint64_t val;
+
+	p = skip_white_spaces(p);
+
+	val = strtoul(p, &next, 16);
+	if (p == next)
+		return -EINVAL;
+
+	p = skip_white_spaces(next);
+	if (*p != '\0')
+		return -EINVAL;
+
+	*value = val;
+	return 0;
+}
+
+int
+parser_read_uint8_hex(uint8_t *value, const char *p)
+{
+	uint64_t val = 0;
+	int ret = parser_read_uint64_hex(&val, p);
+
+	if (ret < 0)
+		return ret;
+
+	if (val > UINT8_MAX)
+		return -ERANGE;
+
+	*value = val;
+	return 0;
+}
+
+int
+parse_uint8_known_len_hex_str(const char *key, char *src, struct fips_val *val)
+{
+	struct fips_val tmp_val = {0};
+	uint32_t len = val->len;
+	int ret;
+
+	if (len == 0) {
+		if (val->val != NULL) {
+			rte_free(val->val);
+			val->val = NULL;
+		}
+
+		return 0;
+	}
+
+	ret = parse_uint8_hex_str(key, src, &tmp_val);
+	if (ret < 0)
+		return ret;
+
+	if (tmp_val.len == val->len) {
+		val->val = tmp_val.val;
+		return 0;
+	}
+
+	if (tmp_val.len < val->len) {
+		rte_free(tmp_val.val);
+		return -EINVAL;
+	}
+
+	val->val = rte_zmalloc(NULL, val->len, 0);
+	if (!val->val) {
+		rte_free(tmp_val.val);
+		memset(val, 0, sizeof(*val));
+		return -ENOMEM;
+	}
+
+	memcpy(val->val, tmp_val.val, val->len);
+	rte_free(tmp_val.val);
+
+	return 0;
+}
+
+int
+parse_uint8_hex_str(const char *key, char *src, struct fips_val *val)
+{
+	uint32_t len, j;
+
+	src += strlen(key);
+
+	len = strlen(src) / 2;
+
+	if (val->val) {
+		rte_free(val->val);
+		val->val = NULL;
+	}
+
+	val->val = rte_zmalloc(NULL, len, 0);
+	if (!val->val)
+		return -ENOMEM;
+
+	for (j = 0; j < len; j++) {
+		char byte[3] = {src[j * 2], src[j * 2 + 1], '\0'};
+
+		if (parser_read_uint8_hex(&val->val[j], byte) < 0) {
+			rte_free(val->val);
+			memset(val, 0, sizeof(*val));
+			return -EINVAL;
+		}
+	}
+
+	val->len = len;
+
+	return 0;
+}
+
+int
+parser_read_uint32_val(const char *key, char *src, struct fips_val *val)
+{
+	char *data = src + strlen(key);
+	size_t data_len = strlen(data);
+	int ret;
+
+	if (data[data_len - 1] == ']') {
+		char *tmp_data = calloc(1, data_len + 1);
+
+		if (tmp_data == NULL)
+			return -ENOMEM;
+
+		strlcpy(tmp_data, data, data_len);
+
+		ret = parser_read_uint32(&val->len, tmp_data);
+
+		free(tmp_data);
+	} else
+		ret = parser_read_uint32(&val->len, data);
+
+	return ret;
+}
+
+int
+parser_read_uint32_bit_val(const char *key, char *src, struct fips_val *val)
+{
+	int ret;
+
+	ret = parser_read_uint32_val(key, src, val);
+
+	if (ret < 0)
+		return ret;
+
+	val->len /= 8;
+
+	return 0;
+}
+
+int
+writeback_hex_str(const char *key, char *dst, struct fips_val *val)
+{
+	char *str = dst;
+	uint32_t len;
+
+	str += strlen(key);
+
+	for (len = 0; len < val->len; len++)
+		snprintf(str + len * 2, 255, "%02x", val->val[len]);
+
+	return 0;
+}
+
+static int
+parser_read_uint64(uint64_t *value, const char *p)
+{
+	char *next;
+	uint64_t val;
+
+	p = skip_white_spaces(p);
+	if (!isdigit(*p))
+		return -EINVAL;
+
+	val = strtoul(p, &next, 10);
+	if (p == next)
+		return -EINVAL;
+
+	p = next;
+	switch (*p) {
+	case 'T':
+		val *= 1024ULL;
+		/* fall through */
+	case 'G':
+		val *= 1024ULL;
+		/* fall through */
+	case 'M':
+		val *= 1024ULL;
+		/* fall through */
+	case 'k':
+	case 'K':
+		val *= 1024ULL;
+		p++;
+		break;
+	}
+
+	p = skip_white_spaces(p);
+	if (*p != '\0')
+		return -EINVAL;
+
+	*value = val;
+	return 0;
+}
+
+int
+parser_read_uint32(uint32_t *value, char *p)
+{
+	uint64_t val = 0;
+	int ret = parser_read_uint64(&val, p);
+
+	if (ret < 0)
+		return ret;
+
+	if (val > UINT32_MAX)
+		return -EINVAL;
+
+	*value = val;
+	return 0;
+}
+
+void
+parse_write_hex_str(struct fips_val *src)
+{
+	writeback_hex_str("", info.one_line_text, src);
+
+	fprintf(info.fp_wr, "%s\n", info.one_line_text);
+}
+
+int
+update_info_vec(uint32_t count)
+{
+	const struct fips_test_callback *cb;
+	uint32_t i, j;
+
+	if (!info.writeback_callbacks)
+		return -1;
+
+	cb = &info.writeback_callbacks[0];
+
+	snprintf(info.vec[0], strlen(info.vec[0]) + 4, "%s%u", cb->key, count);
+
+	for (i = 1; i < info.nb_vec_lines; i++) {
+		for (j = 1; info.writeback_callbacks[j].key != NULL; j++) {
+			cb = &info.writeback_callbacks[j];
+			if (strstr(info.vec[i], cb->key)) {
+				cb->cb(cb->key, info.vec[i], cb->val);
+				break;
+			}
+		}
+	}
+
+	return 0;
+}
diff --git a/examples/fips_validation/fips_validation.h b/examples/fips_validation/fips_validation.h
new file mode 100644
index 00000000..3e291bc3
--- /dev/null
+++ b/examples/fips_validation/fips_validation.h
@@ -0,0 +1,233 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#ifndef _FIPS_VALIDATION_H_
+#define _FIPS_VALIDATION_H_
+
+#define FIPS_PARSE_ERR(fmt, args)					\
+	RTE_LOG(ERR, USER1, "FIPS parse error" ## fmt ## "\n", ## args)
+
+#define ERR_MSG_SIZE		128
+#define MAX_CASE_LINE		15
+#define MAX_LINE_CHAR		204800 /*< max number of characters per line */
+#define MAX_NB_TESTS		10240
+#define MAX_BUF_SIZE		2048
+#define MAX_STRING_SIZE		64
+
+#define POSITIVE_TEST		0
+#define NEGATIVE_TEST		-1
+
+#define REQ_FILE_PERFIX		"req"
+#define RSP_FILE_PERFIX		"rsp"
+#define FAX_FILE_PERFIX		"fax"
+
+enum fips_test_algorithms {
+		FIPS_TEST_ALGO_AES = 0,
+		FIPS_TEST_ALGO_AES_GCM,
+		FIPS_TEST_ALGO_AES_CMAC,
+		FIPS_TEST_ALGO_AES_CCM,
+		FIPS_TEST_ALGO_HMAC,
+		FIPS_TEST_ALGO_TDES,
+		FIPS_TEST_ALGO_MAX
+};
+
+enum file_types {
+	FIPS_TYPE_REQ = 1,
+	FIPS_TYPE_FAX,
+	FIPS_TYPE_RSP
+};
+
+enum fips_test_op {
+	FIPS_TEST_ENC_AUTH_GEN = 1,
+	FIPS_TEST_DEC_AUTH_VERIF,
+};
+
+#define MAX_LINE_PER_VECTOR            16
+
+struct fips_val {
+	uint8_t *val;
+	uint32_t len;
+};
+
+struct fips_test_vector {
+	union {
+		struct {
+			struct fips_val key;
+			struct fips_val digest;
+			struct fips_val auth_aad;
+			struct fips_val aad;
+		} cipher_auth;
+		struct {
+			struct fips_val key;
+			struct fips_val digest;
+			struct fips_val aad;
+		} aead;
+	};
+
+	struct fips_val pt;
+	struct fips_val ct;
+	struct fips_val iv;
+
+	enum rte_crypto_op_status status;
+};
+
+typedef int (*post_prcess_t)(struct fips_val *val);
+
+typedef int (*parse_callback_t)(const char *key, char *text,
+		struct fips_val *val);
+
+struct fips_test_callback {
+	const char *key;
+	parse_callback_t cb;
+	struct fips_val *val;
+};
+
+enum fips_aesavs_test_types {
+	AESAVS_TYPE_GFXBOX = 1,
+	AESAVS_TYPE_KEYSBOX,
+	AESAVS_TYPE_VARKEY,
+	AESAVS_TYPE_VARTXT,
+	AESAVS_TYPE_MMT,
+	AESAVS_TYPE_MCT,
+};
+
+enum fips_tdes_test_types {
+	TDES_INVERSE_PERMUTATION = 0,
+	TDES_PERMUTATION,
+	TDES_SUBSTITUTION_TABLE,
+	TDES_VARIABLE_KEY,
+	TDES_VARIABLE_TEXT,
+	TDES_KAT,
+	TDES_MCT, /* Monte Carlo (Modes) Test */
+	TDES_MMT /* Multi block Message Test */
+};
+
+enum fips_ccm_test_types {
+	CCM_VADT	= 1, /* Variable Associated Data Test */
+	CCM_VPT,		 /* Variable Payload Test */
+	CCM_VNT,		 /* Variable Nonce Test */
+	CCM_VTT,		 /* Variable Tag Test */
+	CCM_DVPT,	 /*  Decryption-Verification Process Test */
+};
+
+struct aesavs_interim_data {
+	enum fips_aesavs_test_types test_type;
+	uint32_t cipher_algo;
+	uint32_t key_len;
+};
+
+struct hmac_interim_data {
+	enum rte_crypto_auth_algorithm algo;
+};
+
+struct tdes_interim_data {
+	enum fips_tdes_test_types test_type;
+	uint32_t nb_keys;
+};
+
+struct ccm_interim_data {
+	enum fips_ccm_test_types test_type;
+	uint32_t aad_len;
+	uint32_t pt_len;
+	uint32_t digest_len;
+	uint32_t key_len;
+	uint32_t iv_len;
+};
+
+struct fips_test_interim_info {
+	FILE *fp_rd;
+	FILE *fp_wr;
+	enum file_types file_type;
+	enum fips_test_algorithms algo;
+	char *one_line_text;
+	char *vec[MAX_LINE_PER_VECTOR];
+	uint32_t nb_vec_lines;
+	char device_name[MAX_STRING_SIZE];
+
+	union {
+		struct aesavs_interim_data aes_data;
+		struct hmac_interim_data hmac_data;
+		struct tdes_interim_data tdes_data;
+		struct ccm_interim_data ccm_data;
+
+	} interim_info;
+
+	enum fips_test_op op;
+
+	const struct fips_test_callback *callbacks;
+	const struct fips_test_callback *interim_callbacks;
+	const struct fips_test_callback *writeback_callbacks;
+
+	post_prcess_t parse_writeback;
+	post_prcess_t kat_check;
+};
+
+extern struct fips_test_vector vec;
+extern struct fips_test_interim_info info;
+
+int
+fips_test_init(const char *req_file_path, const char *rsp_file_path,
+		const char *device_name);
+
+void
+fips_test_clear(void);
+
+int
+fips_test_fetch_one_block(void);
+
+int
+fips_test_parse_one_case(void);
+
+void
+fips_test_write_one_case(void);
+
+int
+parse_test_aes_init(void);
+
+int
+parse_test_tdes_init(void);
+
+int
+parse_test_hmac_init(void);
+
+int
+parse_test_gcm_init(void);
+
+int
+parse_test_cmac_init(void);
+
+int
+parse_test_ccm_init(void);
+
+int
+parser_read_uint8_hex(uint8_t *value, const char *p);
+
+int
+parse_uint8_hex_str(const char *key, char *src, struct fips_val *val);
+
+int
+parse_uint8_known_len_hex_str(const char *key, char *src, struct fips_val *val);
+
+int
+parser_read_uint32_val(const char *key, char *src, struct fips_val *val);
+
+int
+parser_read_uint32_bit_val(const char *key, char *src, struct fips_val *val);
+
+int
+parser_read_uint32(uint32_t *value, char *p);
+
+int
+parser_read_uint32_val(const char *key, char *src, struct fips_val *val);
+
+int
+writeback_hex_str(const char *key, char *dst, struct fips_val *val);
+
+void
+parse_write_hex_str(struct fips_val *src);
+
+int
+update_info_vec(uint32_t count);
+
+#endif
diff --git a/examples/fips_validation/fips_validation_aes.c b/examples/fips_validation/fips_validation_aes.c
new file mode 100644
index 00000000..8cbc158e
--- /dev/null
+++ b/examples/fips_validation/fips_validation_aes.c
@@ -0,0 +1,188 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <string.h>
+#include <time.h>
+#include <stdio.h>
+
+#include <rte_cryptodev.h>
+
+#include "fips_validation.h"
+
+#define MODE_STR	"AESVS"
+#define ALGO_STR	"test data for "
+#define OP_STR		"State"
+#define KEY_SIZE_STR	"Key Length : "
+
+
+#define COUNT_STR	"COUNT = "
+#define KEY_STR		"KEY = "
+#define IV_STR		"IV = "
+#define PT_STR		"PLAINTEXT = "
+#define CT_STR		"CIPHERTEXT = "
+
+#define OP_ENC_STR	"ENCRYPT"
+#define OP_DEC_STR	"DECRYPT"
+
+struct {
+	uint32_t type;
+	const char *desc;
+} aes_test_types[] = {
+		{AESAVS_TYPE_GFXBOX, "GFSbox"},
+		{AESAVS_TYPE_KEYSBOX, "KeySbox"},
+		{AESAVS_TYPE_VARKEY, "VarKey"},
+		{AESAVS_TYPE_VARTXT, "VarTxt"},
+		{TDES_VARIABLE_TEXT, "VARIABLE PLAINTEXT/CIPHERTEXT"},
+		{TDES_VARIABLE_TEXT, "KAT"},
+		{AESAVS_TYPE_MMT, "MMT"},
+		{AESAVS_TYPE_MCT, "MCT"},
+};
+
+struct aes_test_algo {
+	const char *name;
+	enum rte_crypto_cipher_algorithm algo;
+} const algo_con[] = {
+		{"CBC", RTE_CRYPTO_CIPHER_AES_CBC},
+};
+
+static int
+parse_interim_enc_dec(const char *key,
+		__attribute__((__unused__)) char *text,
+		__attribute__((__unused__)) struct fips_val *val)
+{
+	if (strcmp(key, OP_ENC_STR) == 0)
+		info.op = FIPS_TEST_ENC_AUTH_GEN;
+	else if (strcmp(key, OP_DEC_STR) == 0)
+		info.op = FIPS_TEST_DEC_AUTH_VERIF;
+	else
+		return -1;
+
+	return 0;
+}
+
+struct fips_test_callback aes_tests_interim[] = {
+		{OP_ENC_STR, parse_interim_enc_dec, NULL},
+		{OP_DEC_STR, parse_interim_enc_dec, NULL},
+		{NULL, NULL, NULL} /**< end pointer */
+};
+
+struct fips_test_callback aes_tests_vectors[] = {
+		{KEY_STR, parse_uint8_hex_str, &vec.cipher_auth.key},
+		{IV_STR, parse_uint8_hex_str, &vec.iv},
+		{PT_STR, parse_uint8_hex_str, &vec.pt},
+		{CT_STR, parse_uint8_hex_str, &vec.ct},
+		{NULL, NULL, NULL} /**< end pointer */
+};
+
+struct fips_test_callback aes_tests_interim_vectors[] = {
+		{OP_ENC_STR, parse_interim_enc_dec, NULL},
+		{OP_DEC_STR, parse_interim_enc_dec, NULL},
+		{NULL, NULL, NULL} /**< end pointer */
+};
+
+struct fips_test_callback aes_writeback_callbacks[] = {
+		/** First element is used to pass COUNT string */
+		{COUNT_STR, NULL, NULL},
+		{IV_STR, writeback_hex_str, &vec.iv},
+		{KEY_STR, writeback_hex_str, &vec.cipher_auth.key},
+		{PT_STR, writeback_hex_str, &vec.pt},
+		{CT_STR, writeback_hex_str, &vec.ct},
+		{NULL, NULL, NULL} /**< end pointer */
+};
+
+static int
+parse_test_aes_writeback(struct fips_val *val)
+{
+	if (info.op == FIPS_TEST_ENC_AUTH_GEN)
+		fprintf(info.fp_wr, "%s", CT_STR);
+	else
+		fprintf(info.fp_wr, "%s", PT_STR);
+
+	parse_write_hex_str(val);
+
+	return 0;
+}
+
+static int
+rsp_test_aes_check(struct fips_val *val)
+{
+	struct fips_val *data;
+
+	if (info.op == FIPS_TEST_ENC_AUTH_GEN)
+		data = &vec.ct;
+	else
+		data = &vec.pt;
+
+	if (memcmp(val->val, data->val, val->len) == 0)
+		fprintf(info.fp_wr, "Success\n");
+	else
+		fprintf(info.fp_wr, "Failed\n");
+
+	return 0;
+}
+
+int
+parse_test_aes_init(void)
+{
+	char *tmp;
+	uint32_t i, j;
+
+	for (i = 0; i < info.nb_vec_lines; i++) {
+		char *line = info.vec[i];
+
+		tmp = strstr(line, MODE_STR);
+		if (tmp) {
+			for (j = 0; j < RTE_DIM(aes_test_types); j++)
+				if (strstr(line, aes_test_types[j].desc)) {
+					info.interim_info.aes_data.test_type =
+							aes_test_types[j].type;
+					break;
+				}
+
+			if (j >= RTE_DIM(aes_test_types))
+				return -EINVAL;
+
+			tmp = strstr(line, ALGO_STR);
+			if (!tmp)
+				return -EINVAL;
+
+			tmp += strlen(ALGO_STR);
+			for (j = 0; j < RTE_DIM(algo_con); j++)
+				if (strcmp(algo_con[j].name, tmp) == 0) {
+					info.interim_info.aes_data.cipher_algo =
+						(uint32_t)algo_con[j].algo;
+					break;
+				}
+			if (j >= RTE_DIM(algo_con))
+				return -EINVAL;
+
+			continue;
+		}
+
+		tmp = strstr(line, OP_STR);
+		if (tmp)
+			continue;
+
+		tmp = strstr(line, KEY_SIZE_STR);
+		if (tmp) {
+			tmp += strlen(KEY_SIZE_STR);
+			if (parser_read_uint32
+					(&info.interim_info.aes_data.key_len,
+							tmp) < 0)
+				return -EINVAL;
+
+			info.interim_info.aes_data.key_len /= 8;
+
+			continue;
+		}
+	}
+
+	info.parse_writeback = parse_test_aes_writeback;
+	info.callbacks = aes_tests_vectors;
+	info.interim_callbacks = aes_tests_interim_vectors;
+	info.writeback_callbacks = aes_writeback_callbacks;
+	info.kat_check = rsp_test_aes_check;
+
+	return 0;
+}
diff --git a/examples/fips_validation/fips_validation_ccm.c b/examples/fips_validation/fips_validation_ccm.c
new file mode 100644
index 00000000..632999c1
--- /dev/null
+++ b/examples/fips_validation/fips_validation_ccm.c
@@ -0,0 +1,272 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <stdio.h>
+#include <string.h>
+
+#include <rte_string_fns.h>
+#include <rte_cryptodev.h>
+#include <rte_malloc.h>
+
+#include "fips_validation.h"
+
+#define DVPT_STR	"CCM-DVPT"
+#define VADT_STR	"CCM-VADT"
+#define VPT_STR		"CCM-VPT"
+#define VNT_STR		"CCM-VNT"
+#define VTT_STR		"CCM-VTT"
+
+#define PARAM_PREFIX	"["
+#define ALEN_PREFIX	"Alen = "
+#define PLEN_PREFIX	"Plen = "
+#define IVLEN_PREFIX	"Nlen = "
+#define DIGESTL_PREFIX	"Tlen = "
+
+#define COUNT_STR	"Count = "
+#define KEY_STR		"Key = "
+#define IV_STR		"Nonce = "
+#define PT_STR		"Payload = "
+#define CT_STR		"CT = "
+#define AAD_STR		"Adata = "
+#define POS_NEG_STR	"Result = "
+
+#define POS_KEYWORD	"Pass"
+#define NEG_KEYWORD	"Fail"
+
+static int
+parser_dvpt_interim(const char *key, char *src, struct fips_val *val)
+{
+	char *tmp, c, value[10];
+	char num_pattern[] = "0123456789";
+	int i = 0;
+
+	memset(value, 0, 10);
+
+	tmp = strstr(src, key);
+	if (!tmp)
+		return -1;
+
+	tmp += strlen(key);
+
+	c = tmp[0];
+
+	while (strchr(num_pattern, c) && i < 10) {
+		value[i++] = c;
+		c = tmp[i];
+	}
+
+	return parser_read_uint32_val("", value, val);
+}
+
+static int
+parse_dvpt_ct_hex_str(const char *key, char *src, struct fips_val *val)
+{
+	int ret;
+
+	val->len = vec.pt.len;
+
+	ret = parse_uint8_known_len_hex_str(key, src, val);
+	if (ret < 0)
+		return ret;
+
+	src += strlen(key) + val->len * 2;
+
+	ret = parse_uint8_known_len_hex_str("", src, &vec.aead.digest);
+	if (ret < 0) {
+		rte_free(val->val);
+		memset(val, 0, sizeof(*val));
+		return ret;
+	}
+
+	return 0;
+}
+
+static int
+parse_uint8_ccm_aad_str(const char *key, char *src, struct fips_val *val)
+{
+	uint32_t len = val->len, j;
+
+	src += strlen(key);
+
+	/* CCM aad requires 18 bytes padding before the real content */
+	val->val = rte_zmalloc(NULL, len + 18, 0);
+	if (!val->val)
+		return -1;
+
+	for (j = 0; j < len; j++) {
+		char byte[3] = {src[j * 2], src[j * 2 + 1], '\0'};
+
+		if (parser_read_uint8_hex(&val->val[j + 18], byte) < 0) {
+			rte_free(val->val);
+			memset(val, 0, sizeof(*val));
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+struct fips_test_callback ccm_vnt_vec[] = {
+		{IV_STR, parse_uint8_known_len_hex_str, &vec.iv},
+		{AAD_STR, parse_uint8_ccm_aad_str, &vec.aead.aad},
+		{PT_STR, parse_uint8_known_len_hex_str, &vec.pt},
+		{NULL, NULL, NULL} /**< end pointer */
+};
+
+struct fips_test_callback ccm_vnt_interim_vec[] = {
+		{ALEN_PREFIX, parser_read_uint32_val, &vec.aead.aad},
+		{PLEN_PREFIX, parser_read_uint32_val, &vec.pt},
+		{DIGESTL_PREFIX, parser_read_uint32_val, &vec.aead.digest},
+		{IVLEN_PREFIX, parser_read_uint32_val, &vec.iv},
+		{KEY_STR, parse_uint8_hex_str, &vec.aead.key},
+		{NULL, NULL, NULL} /**< end pointer */
+};
+
+struct fips_test_callback ccm_vtt_vec[] = {
+		{AAD_STR, parse_uint8_ccm_aad_str, &vec.aead.aad},
+		{PT_STR, parse_uint8_known_len_hex_str, &vec.pt},
+		{NULL, NULL, NULL} /**< end pointer */
+};
+
+struct fips_test_callback ccm_vtt_interim_vec[] = {
+		{ALEN_PREFIX, parser_read_uint32_val, &vec.aead.aad},
+		{PLEN_PREFIX, parser_read_uint32_val, &vec.pt},
+		{IVLEN_PREFIX, parser_read_uint32_val, &vec.iv},
+		{DIGESTL_PREFIX, parser_read_uint32_val, &vec.aead.digest},
+		{KEY_STR, parse_uint8_hex_str, &vec.aead.key},
+		{IV_STR, parse_uint8_known_len_hex_str, &vec.iv},
+		{NULL, NULL, NULL} /**< end pointer */
+};
+
+struct fips_test_callback ccm_vadt_vec[] = {
+		{AAD_STR, parse_uint8_ccm_aad_str, &vec.aead.aad},
+		{PT_STR, parse_uint8_known_len_hex_str, &vec.pt},
+		{NULL, NULL, NULL} /**< end pointer */
+};
+
+struct fips_test_callback ccm_vadt_interim_vec[] = {
+		{PLEN_PREFIX, parser_read_uint32_val, &vec.pt},
+		{IVLEN_PREFIX, parser_read_uint32_val, &vec.iv},
+		{ALEN_PREFIX, parser_read_uint32_val, &vec.aead.aad},
+		{DIGESTL_PREFIX, parser_read_uint32_val, &vec.aead.digest},
+		{KEY_STR, parse_uint8_hex_str, &vec.aead.key},
+		{IV_STR, parse_uint8_known_len_hex_str, &vec.iv},
+		{NULL, NULL, NULL} /**< end pointer */
+};
+
+struct fips_test_callback ccm_vpt_vec[] = {
+		{AAD_STR, parse_uint8_ccm_aad_str, &vec.aead.aad},
+		{PT_STR, parse_uint8_known_len_hex_str, &vec.pt},
+		{NULL, NULL, NULL} /**< end pointer */
+};
+
+struct fips_test_callback ccm_vpt_interim_vec[] = {
+		{ALEN_PREFIX, parser_read_uint32_val, &vec.aead.aad},
+		{IVLEN_PREFIX, parser_read_uint32_val, &vec.iv},
+		{DIGESTL_PREFIX, parser_read_uint32_val, &vec.aead.digest},
+		{PLEN_PREFIX, parser_read_uint32_val, &vec.pt},
+		{KEY_STR, parse_uint8_hex_str, &vec.aead.key},
+		{IV_STR, parse_uint8_known_len_hex_str, &vec.iv},
+		{NULL, NULL, NULL} /**< end pointer */
+};
+
+struct fips_test_callback ccm_dvpt_vec[] = {
+		{IV_STR, parse_uint8_known_len_hex_str, &vec.iv},
+		{AAD_STR, parse_uint8_ccm_aad_str, &vec.aead.aad},
+		{CT_STR, parse_dvpt_ct_hex_str, &vec.ct},
+		{NULL, NULL, NULL} /**< end pointer */
+};
+
+struct fips_test_callback ccm_dvpt_interim_vec[] = {
+		{ALEN_PREFIX, parser_dvpt_interim, &vec.aead.aad},
+		{PLEN_PREFIX, parser_dvpt_interim, &vec.pt},
+		{IVLEN_PREFIX, parser_dvpt_interim, &vec.iv},
+		{DIGESTL_PREFIX, parser_dvpt_interim, &vec.aead.digest},
+		{KEY_STR, parse_uint8_hex_str, &vec.aead.key},
+		{NULL, NULL, NULL} /**< end pointer */
+};
+
+struct ccm_test_types {
+	const char *str;
+	uint32_t type;
+	const struct fips_test_callback *cb;
+	const struct fips_test_callback *cb_interim;
+	enum fips_test_op op;
+} ctt[] = {
+		{DVPT_STR, CCM_DVPT, ccm_dvpt_vec, ccm_dvpt_interim_vec,
+			FIPS_TEST_DEC_AUTH_VERIF},
+		{VPT_STR, CCM_VPT, ccm_vpt_vec, ccm_vpt_interim_vec,
+			FIPS_TEST_ENC_AUTH_GEN},
+		{VADT_STR, CCM_VADT, ccm_vadt_vec, ccm_vadt_interim_vec,
+			FIPS_TEST_ENC_AUTH_GEN},
+		{VNT_STR, CCM_VNT, ccm_vnt_vec, ccm_vnt_interim_vec,
+			FIPS_TEST_ENC_AUTH_GEN},
+		{VTT_STR, CCM_VTT, ccm_vtt_vec, ccm_vtt_interim_vec,
+			FIPS_TEST_ENC_AUTH_GEN},
+};
+
+static int
+parse_test_ccm_writeback(struct fips_val *val)
+{
+	struct fips_val tmp_val;
+
+	switch (info.interim_info.ccm_data.test_type) {
+	case CCM_DVPT:
+		fprintf(info.fp_wr, "%s", POS_NEG_STR);
+		if (vec.status == RTE_CRYPTO_OP_STATUS_SUCCESS) {
+			fprintf(info.fp_wr, "%s\n", POS_KEYWORD);
+			fprintf(info.fp_wr, "%s", PT_STR);
+
+			tmp_val.val = val->val;
+			tmp_val.len = vec.pt.len;
+
+			if (tmp_val.len == 0)
+				fprintf(info.fp_wr, "00\n");
+			else
+				parse_write_hex_str(&tmp_val);
+		} else
+			fprintf(info.fp_wr, "%s\n", NEG_KEYWORD);
+
+		break;
+
+	case CCM_VADT:
+	case CCM_VNT:
+	case CCM_VPT:
+	case CCM_VTT:
+		fprintf(info.fp_wr, "%s", CT_STR);
+
+		parse_write_hex_str(val);
+
+		break;
+
+	}
+
+	return 0;
+}
+
+int
+parse_test_ccm_init(void)
+{
+
+	uint32_t i;
+
+	for (i = 0; i < info.nb_vec_lines; i++) {
+		char *line = info.vec[i];
+		uint32_t j;
+
+		for (j = 0; j < RTE_DIM(ctt); j++)
+			if (strstr(line, ctt[j].str)) {
+				info.interim_info.ccm_data.test_type =
+						ctt[j].type;
+				info.callbacks = ctt[j].cb;
+				info.interim_callbacks = ctt[j].cb_interim;
+				info.op = ctt[j].op;
+				break;
+		}
+	}
+
+	info.parse_writeback = parse_test_ccm_writeback;
+
+	return 0;
+}
diff --git a/examples/fips_validation/fips_validation_cmac.c b/examples/fips_validation/fips_validation_cmac.c
new file mode 100644
index 00000000..54c951ef
--- /dev/null
+++ b/examples/fips_validation/fips_validation_cmac.c
@@ -0,0 +1,116 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <string.h>
+#include <time.h>
+#include <stdio.h>
+#include <rte_string_fns.h>
+
+#include <rte_cryptodev.h>
+
+#include "fips_validation.h"
+
+#define NEW_LINE_STR	"#"
+#define OP_STR		"CMAC"
+
+#define ALGO_STR	"Alg = "
+#define MODE_STR	"Mode = "
+
+#define COUNT_STR	"Count = "
+#define KLEN_STR	"Klen = "
+#define PTLEN_STR	"Mlen = "
+#define TAGLEN_STR	"Tlen = "
+#define KEY_STR		"Key = "
+#define PT_STR		"Msg = "
+#define TAG_STR		"Mac = "
+
+#define GEN_STR		"Generate"
+#define VERIF_STR	"Verify"
+
+#define POS_NEG_STR	"Result = "
+#define PASS_STR	"P"
+#define FAIL_STR	"F"
+
+struct hash_algo_conversion {
+	const char *str;
+	enum fips_test_algorithms algo;
+} cmac_algo[] = {
+		{"AES", FIPS_TEST_ALGO_AES_CMAC},
+};
+
+static int
+parse_test_cmac_writeback(struct fips_val *val)
+{
+	if (info.op == FIPS_TEST_ENC_AUTH_GEN) {
+		struct fips_val tmp_val = {val->val + vec.pt.len,
+				vec.cipher_auth.digest.len};
+
+		fprintf(info.fp_wr, "%s", TAG_STR);
+		parse_write_hex_str(&tmp_val);
+	} else {
+		fprintf(info.fp_wr, "%s", POS_NEG_STR);
+
+		if (vec.status == RTE_CRYPTO_OP_STATUS_SUCCESS)
+			fprintf(info.fp_wr, "%s\n", PASS_STR);
+		else if (vec.status == RTE_CRYPTO_OP_STATUS_AUTH_FAILED)
+			fprintf(info.fp_wr, "%s\n", FAIL_STR);
+		else
+			fprintf(info.fp_wr, "Error\n");
+	}
+
+	return 0;
+}
+
+struct fips_test_callback cmac_tests_vectors[] = {
+		{KLEN_STR, parser_read_uint32_val, &vec.cipher_auth.key},
+		{PTLEN_STR, parser_read_uint32_val, &vec.pt},
+		{TAGLEN_STR, parser_read_uint32_val, &vec.cipher_auth.digest},
+		{KEY_STR, parse_uint8_hex_str, &vec.cipher_auth.key},
+		{PT_STR, parse_uint8_known_len_hex_str, &vec.pt},
+		{TAG_STR, parse_uint8_known_len_hex_str,
+				&vec.cipher_auth.digest},
+		{NULL, NULL, NULL} /**< end pointer */
+};
+
+int
+parse_test_cmac_init(void)
+{
+	char *tmp;
+	uint32_t i, j;
+
+	for (i = 0; i < info.nb_vec_lines; i++) {
+		char *line = info.vec[i];
+
+		tmp = strstr(line, ALGO_STR);
+		if (!tmp)
+			continue;
+
+		for (j = 0; j < RTE_DIM(cmac_algo); j++) {
+			if (!strstr(line, cmac_algo[j].str))
+				continue;
+
+			info.algo = cmac_algo[j].algo;
+			break;
+		}
+
+		if (j == RTE_DIM(cmac_algo))
+			return -EINVAL;
+
+		tmp = strstr(line, MODE_STR);
+		if (!tmp)
+			return -1;
+
+		if (strstr(tmp, GEN_STR))
+			info.op = FIPS_TEST_ENC_AUTH_GEN;
+		else if (strstr(tmp, VERIF_STR))
+			info.op = FIPS_TEST_DEC_AUTH_VERIF;
+		else
+			return -EINVAL;
+	}
+
+	info.parse_writeback = parse_test_cmac_writeback;
+	info.callbacks = cmac_tests_vectors;
+
+	return 0;
+}
diff --git a/examples/fips_validation/fips_validation_gcm.c b/examples/fips_validation/fips_validation_gcm.c
new file mode 100644
index 00000000..0509b101
--- /dev/null
+++ b/examples/fips_validation/fips_validation_gcm.c
@@ -0,0 +1,125 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <string.h>
+#include <time.h>
+#include <stdio.h>
+
+#include <rte_cryptodev.h>
+
+#include "fips_validation.h"
+
+#define NEW_LINE_STR	"#"
+#define OP_STR		"GCM "
+
+#define PARAM_PREFIX	"["
+#define KEYLEN_STR	"Keylen = "
+#define IVLEN_STR	"IVlen = "
+#define PTLEN_STR	"PTlen = "
+#define AADLEN_STR	"AADlen = "
+#define TAGLEN_STR	"Taglen = "
+
+#define COUNT_STR	"Count = "
+#define KEY_STR		"Key = "
+#define IV_STR		"IV = "
+#define PT_STR		"PT = "
+#define CT_STR		"CT = "
+#define TAG_STR		"Tag = "
+#define AAD_STR		"AAD = "
+
+#define OP_ENC_STR	"Encrypt"
+#define OP_DEC_STR	"Decrypt"
+
+#define NEG_TEST_STR	"FAIL"
+
+struct fips_test_callback gcm_dec_vectors[] = {
+		{KEY_STR, parse_uint8_known_len_hex_str, &vec.cipher_auth.key},
+		{IV_STR, parse_uint8_known_len_hex_str, &vec.iv},
+		{CT_STR, parse_uint8_known_len_hex_str, &vec.ct},
+		{AAD_STR, parse_uint8_known_len_hex_str, &vec.cipher_auth.aad},
+		{TAG_STR, parse_uint8_known_len_hex_str,
+				&vec.cipher_auth.digest},
+		{NULL, NULL, NULL} /**< end pointer */
+};
+struct fips_test_callback gcm_interim_vectors[] = {
+		{KEYLEN_STR, parser_read_uint32_bit_val, &vec.cipher_auth.key},
+		{IVLEN_STR, parser_read_uint32_bit_val, &vec.iv},
+		{PTLEN_STR, parser_read_uint32_bit_val, &vec.pt},
+		{AADLEN_STR, parser_read_uint32_bit_val, &vec.cipher_auth.aad},
+		{TAGLEN_STR, parser_read_uint32_bit_val,
+				&vec.cipher_auth.digest},
+		{NULL, NULL, NULL} /**< end pointer */
+};
+
+struct fips_test_callback gcm_enc_vectors[] = {
+		{KEY_STR, parse_uint8_known_len_hex_str, &vec.cipher_auth.key},
+		{IV_STR, parse_uint8_known_len_hex_str, &vec.iv},
+		{PT_STR, parse_uint8_known_len_hex_str, &vec.pt},
+		{AAD_STR, parse_uint8_known_len_hex_str, &vec.cipher_auth.aad},
+		{NULL, NULL, NULL} /**< end pointer */
+};
+
+static int
+parse_test_gcm_writeback(struct fips_val *val)
+{
+	struct fips_val tmp_val;
+
+	if (info.op == FIPS_TEST_ENC_AUTH_GEN) {
+		fprintf(info.fp_wr, "%s", CT_STR);
+
+		tmp_val.val = val->val;
+		tmp_val.len = vec.pt.len;
+
+		parse_write_hex_str(&tmp_val);
+
+		fprintf(info.fp_wr, "%s", TAG_STR);
+
+		tmp_val.val = val->val + vec.pt.len;
+		tmp_val.len = val->len - vec.pt.len;
+
+		parse_write_hex_str(&tmp_val);
+	} else {
+		if (vec.status == RTE_CRYPTO_OP_STATUS_SUCCESS) {
+			fprintf(info.fp_wr, "%s", PT_STR);
+
+			tmp_val.val = val->val;
+			tmp_val.len = vec.pt.len;
+
+			parse_write_hex_str(&tmp_val);
+		} else
+			fprintf(info.fp_wr, "%s\n", NEG_TEST_STR);
+	}
+
+	return 0;
+}
+
+int
+parse_test_gcm_init(void)
+{
+	char *tmp;
+	uint32_t i;
+
+
+	for (i = 0; i < info.nb_vec_lines; i++) {
+		char *line = info.vec[i];
+
+
+		tmp = strstr(line, OP_STR);
+		if (tmp) {
+			if (strstr(line, OP_ENC_STR)) {
+				info.op = FIPS_TEST_ENC_AUTH_GEN;
+				info.callbacks = gcm_enc_vectors;
+			} else if (strstr(line, OP_DEC_STR)) {
+				info.op = FIPS_TEST_DEC_AUTH_VERIF;
+				info.callbacks = gcm_dec_vectors;
+			} else
+				return -EINVAL;
+		}
+	}
+
+	info.interim_callbacks = gcm_interim_vectors;
+	info.parse_writeback = parse_test_gcm_writeback;
+
+	return 0;
+}
diff --git a/examples/fips_validation/fips_validation_hmac.c b/examples/fips_validation/fips_validation_hmac.c
new file mode 100644
index 00000000..97ac7186
--- /dev/null
+++ b/examples/fips_validation/fips_validation_hmac.c
@@ -0,0 +1,105 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <string.h>
+#include <time.h>
+#include <stdio.h>
+
+#include <rte_cryptodev.h>
+
+#include "fips_validation.h"
+
+#define ALGO_PREFIX	"[L="
+#define KEYLEN_STR	"Klen = "
+#define TAGLEN_STR	"Tlen = "
+
+#define COUNT_STR	"Count = "
+#define KEY_STR		"Key = "
+#define PT_STR		"Msg = "
+#define TAG_STR		"Mac = "
+
+struct hash_size_conversion {
+	const char *str;
+	enum rte_crypto_auth_algorithm algo;
+} hsc[] = {
+		{"20", RTE_CRYPTO_AUTH_SHA1_HMAC},
+		{"28", RTE_CRYPTO_AUTH_SHA224_HMAC},
+		{"32", RTE_CRYPTO_AUTH_SHA256_HMAC},
+		{"48", RTE_CRYPTO_AUTH_SHA384_HMAC},
+		{"64", RTE_CRYPTO_AUTH_SHA512_HMAC},
+};
+
+static int
+parse_interim_algo(__attribute__((__unused__)) const char *key,
+		char *text,
+		__attribute__((__unused__)) struct fips_val *val)
+{
+
+	uint32_t i;
+
+	for (i = 0; i < RTE_DIM(hsc); i++) {
+		if (strstr(text, hsc[i].str)) {
+			info.interim_info.hmac_data.algo = hsc[i].algo;
+			break;
+		}
+	}
+
+	if (i == RTE_DIM(hsc))
+		return -1;
+
+	return 0;
+}
+
+struct fips_test_callback hmac_tests_vectors[] = {
+		{KEYLEN_STR, parser_read_uint32_val, &vec.cipher_auth.key},
+		{TAGLEN_STR, parser_read_uint32_val, &vec.cipher_auth.digest},
+		{KEY_STR, parse_uint8_hex_str, &vec.cipher_auth.key},
+		{PT_STR, parse_uint8_hex_str, &vec.pt},
+		{TAG_STR, parse_uint8_hex_str, &vec.cipher_auth.digest},
+		{NULL, NULL, NULL} /**< end pointer */
+};
+
+struct fips_test_callback hmac_tests_interim_vectors[] = {
+		{ALGO_PREFIX, parse_interim_algo, NULL},
+		{NULL, NULL, NULL} /**< end pointer */
+};
+
+static int
+parse_test_hmac_writeback(struct fips_val *val)
+{
+	struct fips_val val_local;
+
+	fprintf(info.fp_wr, "%s", TAG_STR);
+
+	val_local.val = val->val + vec.pt.len;
+	val_local.len = vec.cipher_auth.digest.len;
+
+	parse_write_hex_str(&val_local);
+	return 0;
+}
+
+static int
+rsp_test_hmac_check(struct fips_val *val)
+{
+	if (memcmp(val->val + vec.pt.len, vec.cipher_auth.digest.val,
+			vec.cipher_auth.digest.len) == 0)
+		fprintf(info.fp_wr, "Success\n");
+	else
+		fprintf(info.fp_wr, "Failed\n");
+
+	return 0;
+}
+
+int
+parse_test_hmac_init(void)
+{
+	info.op = FIPS_TEST_ENC_AUTH_GEN;
+	info.parse_writeback = parse_test_hmac_writeback;
+	info.callbacks = hmac_tests_vectors;
+	info.interim_callbacks = hmac_tests_interim_vectors;
+	info.writeback_callbacks = NULL;
+	info.kat_check = rsp_test_hmac_check;
+
+	return 0;
+}
diff --git a/examples/fips_validation/fips_validation_tdes.c b/examples/fips_validation/fips_validation_tdes.c
new file mode 100644
index 00000000..5064ff3b
--- /dev/null
+++ b/examples/fips_validation/fips_validation_tdes.c
@@ -0,0 +1,264 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <string.h>
+#include <stdio.h>
+
+#include <rte_malloc.h>
+#include <rte_cryptodev.h>
+
+#include "fips_validation.h"
+
+#define NEW_LINE_STR	"#"
+#define TEST_TYPE_KEY	" for CBC"
+#define TEST_CBCI_KEY	" for CBCI"
+
+#define ENC_STR		"[ENCRYPT]"
+#define DEC_STR		"[DECRYPT]"
+
+#define COUNT_STR	"COUNT = "
+#define KEY1_STR	"KEY1 = "
+#define KEY2_STR	"KEY2 = "
+#define KEY3_STR	"KEY3 = "
+
+#define KEYS_STR	"KEYs = "
+#define IV_STR		"IV = "
+#define PT_STR		"PLAINTEXT = "
+#define CT_STR		"CIPHERTEXT = "
+#define NK_STR		"NumKeys = "
+
+#define SET_STR		" = "
+
+#define PLAIN_TEXT	0
+#define CIPHER_TEXT	1
+#define KEY_TEXT	2
+#define IV_TEXT		3
+
+#define DEVICE_STR	"# Config Info for : "
+
+struct {
+	uint32_t type;
+	const char *desc;
+} test_types[] = {
+		{TDES_INVERSE_PERMUTATION, "INVERSE PERMUTATION"},
+		{TDES_PERMUTATION, "PERMUTATION OPERATION"},
+		{TDES_SUBSTITUTION_TABLE, "SUBSTITUTION TABLE"},
+		{TDES_VARIABLE_KEY, "VARIABLE KEY"},
+		{TDES_VARIABLE_TEXT, "VARIABLE PLAINTEXT/CIPHERTEXT"},
+		{TDES_VARIABLE_TEXT, "KAT"},
+		{TDES_MCT, "Monte Carlo (Modes) Test"},
+		{TDES_MMT, "Multi block Message Test"},
+};
+
+static int
+writeback_tdes_hex_str(const char *key, char *dst, struct fips_val *val);
+
+static int
+parse_tdes_uint8_hex_str(const char *key, char *src, struct fips_val *val);
+
+static int
+parse_tdes_interim(const char *key,
+		__attribute__((__unused__)) char *text,
+		struct fips_val *val);
+
+struct fips_test_callback tdes_tests_vectors[] = {
+		{KEYS_STR, parse_tdes_uint8_hex_str, &vec.cipher_auth.key},
+		{KEY1_STR, parse_tdes_uint8_hex_str, &vec.cipher_auth.key},
+		{KEY2_STR, parse_tdes_uint8_hex_str, &vec.cipher_auth.key},
+		{KEY3_STR, parse_tdes_uint8_hex_str, &vec.cipher_auth.key},
+		{IV_STR, parse_uint8_hex_str, &vec.iv},
+		{PT_STR, parse_uint8_hex_str, &vec.pt},
+		{CT_STR, parse_uint8_hex_str, &vec.ct},
+		{NULL, NULL, NULL} /**< end pointer */
+};
+
+struct fips_test_callback tdes_tests_interim_vectors[] = {
+		{ENC_STR, parse_tdes_interim, NULL},
+		{DEC_STR, parse_tdes_interim, NULL},
+		{NULL, NULL, NULL} /**< end pointer */
+};
+
+struct fips_test_callback tdes_writeback_callbacks[] = {
+		/** First element is used to pass COUNT string */
+		{COUNT_STR, NULL, NULL},
+		{IV_STR, writeback_hex_str, &vec.iv},
+		{KEY1_STR, writeback_tdes_hex_str, &vec.cipher_auth.key},
+		{KEY2_STR, writeback_tdes_hex_str, &vec.cipher_auth.key},
+		{KEY3_STR, writeback_tdes_hex_str, &vec.cipher_auth.key},
+		{KEYS_STR, writeback_tdes_hex_str, &vec.cipher_auth.key},
+		{PT_STR, writeback_hex_str, &vec.pt},
+		{CT_STR, writeback_hex_str, &vec.ct},
+		{NULL, NULL, NULL} /**< end pointer */
+};
+
+static int
+parse_tdes_interim(const char *key,
+		__attribute__((__unused__)) char *text,
+		__attribute__((__unused__)) struct fips_val *val)
+{
+	if (strstr(key, ENC_STR))
+		info.op = FIPS_TEST_ENC_AUTH_GEN;
+	else if (strstr(key, DEC_STR))
+		info.op = FIPS_TEST_DEC_AUTH_VERIF;
+	else if (strstr(NK_STR, "NumKeys = 1"))
+		info.interim_info.tdes_data.nb_keys = 1;
+	else if (strstr(NK_STR, "NumKeys = 2"))
+		info.interim_info.tdes_data.nb_keys = 2;
+	else if (strstr(NK_STR, "NumKeys = 3"))
+		info.interim_info.tdes_data.nb_keys = 3;
+	else
+		return -EINVAL;
+
+	return 0;
+}
+
+static int
+parse_tdes_uint8_hex_str(const char *key, char *src, struct fips_val *val)
+{
+	uint8_t tmp_key[24] = {0};
+	uint32_t len, i;
+
+	src += strlen(key);
+
+	len = strlen(src) / 2;
+
+	if (val->val) {
+		memcpy(tmp_key, val->val, val->len);
+		rte_free(val->val);
+	}
+
+	val->val = rte_zmalloc(NULL, 24, 0);
+	if (!val->val)
+		return -1;
+
+	memcpy(val->val, tmp_key, 24);
+
+	if (strstr(key, KEYS_STR)) {
+		for (i = 0; i < len; i++) {
+			char byte[3] = {src[i * 2], src[i * 2 + 1], '\0'};
+
+			if (parser_read_uint8_hex(&val->val[i], byte) < 0)
+				goto error_exit;
+		}
+
+		memcpy(val->val + 8, val->val, 8);
+		memcpy(val->val + 16, val->val, 8);
+
+	} else if (strstr(key, KEY1_STR)) {
+		for (i = 0; i < len; i++) {
+			char byte[3] = {src[i * 2], src[i * 2 + 1], '\0'};
+
+			if (parser_read_uint8_hex(&val->val[i], byte) < 0)
+				goto error_exit;
+		}
+
+		if (info.interim_info.tdes_data.nb_keys == 2)
+			memcpy(val->val + 16, val->val, 8);
+
+	} else if (strstr(key, KEY2_STR)) {
+		for (i = 0; i < len; i++) {
+			char byte[3] = {src[i * 2], src[i * 2 + 1], '\0'};
+
+			if (parser_read_uint8_hex(&val->val[i + 8], byte) < 0)
+				goto error_exit;
+		}
+
+	} else if (strstr(key, KEY3_STR)) {
+		for (i = 0; i < len; i++) {
+			char byte[3] = {src[i * 2], src[i * 2 + 1], '\0'};
+
+			if (parser_read_uint8_hex(&val->val[i + 16], byte) < 0)
+				goto error_exit;
+		}
+	} else
+		return -EINVAL;
+
+	val->len = 24;
+
+	return 0;
+
+error_exit:
+	rte_free(val->val);
+	memset(val, 0, sizeof(*val));
+	return -EINVAL;
+}
+
+static int
+parse_test_tdes_writeback(struct fips_val *val)
+{
+
+	if (info.op == FIPS_TEST_ENC_AUTH_GEN)
+		fprintf(info.fp_wr, "%s", CT_STR);
+	else
+		fprintf(info.fp_wr, "%s", PT_STR);
+
+	parse_write_hex_str(val);
+
+	return 0;
+
+}
+
+static int
+writeback_tdes_hex_str(const char *key, char *dst, struct fips_val *val)
+{
+	struct fips_val tmp_val;
+
+	tmp_val.len = 8;
+
+	if (strstr(key, KEY1_STR))
+		tmp_val.val = val->val;
+	else if (strstr(key, KEY2_STR))
+		tmp_val.val = val->val + 8;
+	else if (strstr(key, KEY3_STR))
+		tmp_val.val = val->val + 16;
+
+	return writeback_hex_str(key, dst, &tmp_val);
+}
+
+static int
+rsp_test_tdes_check(struct fips_val *val)
+{
+	struct fips_val *data;
+
+	if (info.op == FIPS_TEST_ENC_AUTH_GEN)
+		data = &vec.ct;
+	else
+		data = &vec.pt;
+
+	if (memcmp(val->val, data->val, val->len) == 0)
+		fprintf(info.fp_wr, "Success\n");
+	else
+		fprintf(info.fp_wr, "Failed\n");
+
+	return 0;
+}
+
+int
+parse_test_tdes_init(void)
+{
+	uint32_t i;
+
+	for (i = 0; i < info.nb_vec_lines; i++) {
+		char *line = info.vec[i];
+		uint32_t j;
+
+		if (strstr(line, TEST_CBCI_KEY))
+			return -EPERM;
+
+		for (j = 0; j < RTE_DIM(test_types); j++)
+			if (strstr(line, test_types[j].desc)) {
+				info.interim_info.tdes_data.test_type =
+						test_types[j].type;
+				break;
+			}
+	}
+
+	info.parse_writeback = parse_test_tdes_writeback;
+	info.callbacks = tdes_tests_vectors;
+	info.interim_callbacks = tdes_tests_interim_vectors;
+	info.writeback_callbacks = tdes_writeback_callbacks;
+	info.kat_check = rsp_test_tdes_check;
+
+	return 0;
+}
diff --git a/examples/fips_validation/main.c b/examples/fips_validation/main.c
new file mode 100644
index 00000000..85f54cbf
--- /dev/null
+++ b/examples/fips_validation/main.c
@@ -0,0 +1,1225 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <sys/stat.h>
+#include <getopt.h>
+#include <dirent.h>
+
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+#include <rte_string_fns.h>
+
+#include "fips_validation.h"
+
+#define REQ_FILE_PATH_KEYWORD	"req-file"
+#define RSP_FILE_PATH_KEYWORD	"rsp-file"
+#define FOLDER_KEYWORD		"path-is-folder"
+#define CRYPTODEV_KEYWORD	"cryptodev"
+#define CRYPTODEV_ID_KEYWORD	"cryptodev-id"
+
+struct fips_test_vector vec;
+struct fips_test_interim_info info;
+
+struct cryptodev_fips_validate_env {
+	const char *req_path;
+	const char *rsp_path;
+	uint32_t is_path_folder;
+	uint32_t dev_id;
+	struct rte_mempool *mpool;
+	struct rte_mempool *op_pool;
+	struct rte_mbuf *mbuf;
+	struct rte_crypto_op *op;
+	struct rte_cryptodev_sym_session *sess;
+} env;
+
+static int
+cryptodev_fips_validate_app_int(void)
+{
+	struct rte_cryptodev_config conf = {rte_socket_id(), 1};
+	struct rte_cryptodev_qp_conf qp_conf = {128};
+	int ret;
+
+	ret = rte_cryptodev_configure(env.dev_id, &conf);
+	if (ret < 0)
+		return ret;
+
+	env.mpool = rte_pktmbuf_pool_create("FIPS_MEMPOOL", 128, 0, 0,
+			UINT16_MAX, rte_socket_id());
+	if (!env.mpool)
+		return ret;
+
+	ret = rte_cryptodev_queue_pair_setup(env.dev_id, 0, &qp_conf,
+			rte_socket_id(), env.mpool);
+	if (ret < 0)
+		return ret;
+
+	ret = -ENOMEM;
+
+	env.op_pool = rte_crypto_op_pool_create(
+			"FIPS_OP_POOL",
+			RTE_CRYPTO_OP_TYPE_SYMMETRIC,
+			1, 0,
+			16,
+			rte_socket_id());
+	if (!env.op_pool)
+		goto error_exit;
+
+	env.mbuf = rte_pktmbuf_alloc(env.mpool);
+	if (!env.mbuf)
+		goto error_exit;
+
+	env.op = rte_crypto_op_alloc(env.op_pool, RTE_CRYPTO_OP_TYPE_SYMMETRIC);
+	if (!env.op)
+		goto error_exit;
+
+	return 0;
+
+error_exit:
+	rte_mempool_free(env.mpool);
+	if (env.op_pool)
+		rte_mempool_free(env.op_pool);
+
+	return ret;
+}
+
+static void
+cryptodev_fips_validate_app_uninit(void)
+{
+	rte_pktmbuf_free(env.mbuf);
+	rte_crypto_op_free(env.op);
+	rte_cryptodev_sym_session_clear(env.dev_id, env.sess);
+	rte_cryptodev_sym_session_free(env.sess);
+	rte_mempool_free(env.mpool);
+	rte_mempool_free(env.op_pool);
+}
+
+static int
+fips_test_one_file(void);
+
+static int
+parse_cryptodev_arg(char *arg)
+{
+	int id = rte_cryptodev_get_dev_id(arg);
+
+	if (id < 0) {
+		RTE_LOG(ERR, USER1, "Error %i: invalid cryptodev name %s\n",
+				id, arg);
+		return id;
+	}
+
+	env.dev_id = (uint32_t)id;
+
+	return 0;
+}
+
+static int
+parse_cryptodev_id_arg(char *arg)
+{
+	uint32_t cryptodev_id;
+
+	if (parser_read_uint32(&cryptodev_id, arg) < 0) {
+		RTE_LOG(ERR, USER1, "Error %i: invalid cryptodev id %s\n",
+				-EINVAL, arg);
+		return -1;
+	}
+
+
+	if (!rte_cryptodev_pmd_is_valid_dev(cryptodev_id)) {
+		RTE_LOG(ERR, USER1, "Error %i: invalid cryptodev id %s\n",
+				cryptodev_id, arg);
+		return -1;
+	}
+
+	env.dev_id = (uint32_t)cryptodev_id;
+
+	return 0;
+}
+
+static void
+cryptodev_fips_validate_usage(const char *prgname)
+{
+	printf("%s [EAL options] --\n"
+		"  --%s: REQUEST-FILE-PATH\n"
+		"  --%s: RESPONSE-FILE-PATH\n"
+		"  --%s: indicating both paths are folders\n"
+		"  --%s: CRYPTODEV-NAME\n"
+		"  --%s: CRYPTODEV-ID-NAME\n",
+		prgname, REQ_FILE_PATH_KEYWORD, RSP_FILE_PATH_KEYWORD,
+		FOLDER_KEYWORD, CRYPTODEV_KEYWORD, CRYPTODEV_ID_KEYWORD);
+}
+
+static int
+cryptodev_fips_validate_parse_args(int argc, char **argv)
+{
+	int opt, ret;
+	char *prgname = argv[0];
+	char **argvopt;
+	int option_index;
+	struct option lgopts[] = {
+			{REQ_FILE_PATH_KEYWORD, required_argument, 0, 0},
+			{RSP_FILE_PATH_KEYWORD, required_argument, 0, 0},
+			{FOLDER_KEYWORD, no_argument, 0, 0},
+			{CRYPTODEV_KEYWORD, required_argument, 0, 0},
+			{CRYPTODEV_ID_KEYWORD, required_argument, 0, 0},
+			{NULL, 0, 0, 0}
+	};
+
+	argvopt = argv;
+
+	while ((opt = getopt_long(argc, argvopt, "s:",
+				  lgopts, &option_index)) != EOF) {
+
+		switch (opt) {
+		case 0:
+			if (strcmp(lgopts[option_index].name,
+					REQ_FILE_PATH_KEYWORD) == 0)
+				env.req_path = optarg;
+			else if (strcmp(lgopts[option_index].name,
+					RSP_FILE_PATH_KEYWORD) == 0)
+				env.rsp_path = optarg;
+			else if (strcmp(lgopts[option_index].name,
+					FOLDER_KEYWORD) == 0)
+				env.is_path_folder = 1;
+			else if (strcmp(lgopts[option_index].name,
+					CRYPTODEV_KEYWORD) == 0) {
+				ret = parse_cryptodev_arg(optarg);
+				if (ret < 0) {
+					cryptodev_fips_validate_usage(prgname);
+					return -EINVAL;
+				}
+			} else if (strcmp(lgopts[option_index].name,
+					CRYPTODEV_ID_KEYWORD) == 0) {
+				ret = parse_cryptodev_id_arg(optarg);
+				if (ret < 0) {
+					cryptodev_fips_validate_usage(prgname);
+					return -EINVAL;
+				}
+			} else {
+				cryptodev_fips_validate_usage(prgname);
+				return -EINVAL;
+			}
+			break;
+		default:
+			return -1;
+		}
+	}
+
+	if (env.req_path == NULL || env.rsp_path == NULL ||
+			env.dev_id == UINT32_MAX) {
+		cryptodev_fips_validate_usage(prgname);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int
+main(int argc, char *argv[])
+{
+	int ret;
+
+	ret = rte_eal_init(argc, argv);
+	if (ret < 0) {
+		RTE_LOG(ERR, USER1, "Error %i: Failed init\n", ret);
+		return -1;
+	}
+
+	argc -= ret;
+	argv += ret;
+
+	ret = cryptodev_fips_validate_parse_args(argc, argv);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE, "Failed to parse arguments!\n");
+
+	ret = cryptodev_fips_validate_app_int();
+	if (ret < 0) {
+		RTE_LOG(ERR, USER1, "Error %i: Failed init\n", ret);
+		return -1;
+	}
+
+	if (!env.is_path_folder) {
+		printf("Processing file %s... ", env.req_path);
+
+		ret = fips_test_init(env.req_path, env.rsp_path,
+			rte_cryptodev_name_get(env.dev_id));
+		if (ret < 0) {
+			RTE_LOG(ERR, USER1, "Error %i: Failed test %s\n",
+					ret, env.req_path);
+			goto exit;
+		}
+
+
+		ret = fips_test_one_file();
+		if (ret < 0) {
+			RTE_LOG(ERR, USER1, "Error %i: Failed test %s\n",
+					ret, env.req_path);
+			goto exit;
+		}
+
+		printf("Done\n");
+
+	} else {
+		struct dirent *dir;
+		DIR *d_req, *d_rsp;
+		char req_path[1024];
+		char rsp_path[1024];
+
+		d_req = opendir(env.req_path);
+		if (!d_req) {
+			RTE_LOG(ERR, USER1, "Error %i: Path %s not exist\n",
+					-EINVAL, env.req_path);
+			goto exit;
+		}
+
+		d_rsp = opendir(env.rsp_path);
+		if (!d_rsp) {
+			ret = mkdir(env.rsp_path, 0700);
+			if (ret == 0)
+				d_rsp = opendir(env.rsp_path);
+			else {
+				RTE_LOG(ERR, USER1, "Error %i: Invalid %s\n",
+						-EINVAL, env.rsp_path);
+				goto exit;
+			}
+		}
+		closedir(d_rsp);
+
+		while ((dir = readdir(d_req)) != NULL) {
+			if (strstr(dir->d_name, "req") == NULL)
+				continue;
+
+			snprintf(req_path, 1023, "%s/%s", env.req_path,
+					dir->d_name);
+			snprintf(rsp_path, 1023, "%s/%s", env.rsp_path,
+					dir->d_name);
+			strlcpy(strstr(rsp_path, "req"), "rsp", 4);
+
+			printf("Processing file %s... ", req_path);
+
+			ret = fips_test_init(req_path, rsp_path,
+			rte_cryptodev_name_get(env.dev_id));
+			if (ret < 0) {
+				RTE_LOG(ERR, USER1, "Error %i: Failed test %s\n",
+						ret, req_path);
+				break;
+			}
+
+			ret = fips_test_one_file();
+			if (ret < 0) {
+				RTE_LOG(ERR, USER1, "Error %i: Failed test %s\n",
+						ret, req_path);
+				break;
+			}
+
+			printf("Done\n");
+		}
+
+		closedir(d_req);
+	}
+
+
+exit:
+	fips_test_clear();
+	cryptodev_fips_validate_app_uninit();
+
+	return ret;
+
+}
+
+#define IV_OFF (sizeof(struct rte_crypto_op) + sizeof(struct rte_crypto_sym_op))
+#define CRYPTODEV_FIPS_MAX_RETRIES	16
+
+typedef int (*fips_test_one_case_t)(void);
+typedef int (*fips_prepare_op_t)(void);
+typedef int (*fips_prepare_xform_t)(struct rte_crypto_sym_xform *);
+
+struct fips_test_ops {
+	fips_prepare_xform_t prepare_xform;
+	fips_prepare_op_t prepare_op;
+	fips_test_one_case_t test;
+} test_ops;
+
+static int
+prepare_cipher_op(void)
+{
+	struct rte_crypto_sym_op *sym = env.op->sym;
+	uint8_t *iv = rte_crypto_op_ctod_offset(env.op, uint8_t *, IV_OFF);
+
+	__rte_crypto_op_reset(env.op, RTE_CRYPTO_OP_TYPE_SYMMETRIC);
+	rte_pktmbuf_reset(env.mbuf);
+
+	sym->m_src = env.mbuf;
+	sym->cipher.data.offset = 0;
+
+	memcpy(iv, vec.iv.val, vec.iv.len);
+
+	if (info.op == FIPS_TEST_ENC_AUTH_GEN) {
+		uint8_t *pt;
+
+		if (vec.pt.len > RTE_MBUF_MAX_NB_SEGS) {
+			RTE_LOG(ERR, USER1, "PT len %u\n", vec.pt.len);
+			return -EPERM;
+		}
+
+		pt = (uint8_t *)rte_pktmbuf_append(env.mbuf, vec.pt.len);
+
+		if (!pt) {
+			RTE_LOG(ERR, USER1, "Error %i: MBUF too small\n",
+					-ENOMEM);
+			return -ENOMEM;
+		}
+
+		memcpy(pt, vec.pt.val, vec.pt.len);
+		sym->cipher.data.length = vec.pt.len;
+
+	} else {
+		uint8_t *ct;
+
+		if (vec.ct.len > RTE_MBUF_MAX_NB_SEGS) {
+			RTE_LOG(ERR, USER1, "CT len %u\n", vec.ct.len);
+			return -EPERM;
+		}
+
+		ct = (uint8_t *)rte_pktmbuf_append(env.mbuf, vec.ct.len);
+
+		if (!ct) {
+			RTE_LOG(ERR, USER1, "Error %i: MBUF too small\n",
+					-ENOMEM);
+			return -ENOMEM;
+		}
+
+		memcpy(ct, vec.ct.val, vec.ct.len);
+		sym->cipher.data.length = vec.ct.len;
+	}
+
+	rte_crypto_op_attach_sym_session(env.op, env.sess);
+
+	return 0;
+}
+
+static int
+prepare_auth_op(void)
+{
+	struct rte_crypto_sym_op *sym = env.op->sym;
+
+	__rte_crypto_op_reset(env.op, RTE_CRYPTO_OP_TYPE_SYMMETRIC);
+	rte_pktmbuf_reset(env.mbuf);
+
+	sym->m_src = env.mbuf;
+	sym->auth.data.offset = 0;
+
+	if (info.op == FIPS_TEST_ENC_AUTH_GEN) {
+		uint8_t *pt;
+
+		if (vec.pt.len > RTE_MBUF_MAX_NB_SEGS) {
+			RTE_LOG(ERR, USER1, "PT len %u\n", vec.pt.len);
+			return -EPERM;
+		}
+
+		pt = (uint8_t *)rte_pktmbuf_append(env.mbuf, vec.pt.len +
+				vec.cipher_auth.digest.len);
+
+		if (!pt) {
+			RTE_LOG(ERR, USER1, "Error %i: MBUF too small\n",
+					-ENOMEM);
+			return -ENOMEM;
+		}
+
+		memcpy(pt, vec.pt.val, vec.pt.len);
+		sym->auth.data.length = vec.pt.len;
+		sym->auth.digest.data = pt + vec.pt.len;
+		sym->auth.digest.phys_addr = rte_pktmbuf_mtophys_offset(
+				env.mbuf, vec.pt.len);
+
+	} else {
+		uint8_t *ct;
+
+		if (vec.ct.len > RTE_MBUF_MAX_NB_SEGS) {
+			RTE_LOG(ERR, USER1, "CT len %u\n", vec.ct.len);
+			return -EPERM;
+		}
+
+		ct = (uint8_t *)rte_pktmbuf_append(env.mbuf,
+				vec.ct.len + vec.cipher_auth.digest.len);
+
+		if (!ct) {
+			RTE_LOG(ERR, USER1, "Error %i: MBUF too small\n",
+					-ENOMEM);
+			return -ENOMEM;
+		}
+
+		memcpy(ct, vec.ct.val, vec.ct.len);
+		sym->auth.data.length = vec.ct.len;
+		sym->auth.digest.data = vec.cipher_auth.digest.val;
+		sym->auth.digest.phys_addr = rte_malloc_virt2iova(
+				sym->auth.digest.data);
+	}
+
+	rte_crypto_op_attach_sym_session(env.op, env.sess);
+
+	return 0;
+}
+
+static int
+prepare_aead_op(void)
+{
+	struct rte_crypto_sym_op *sym = env.op->sym;
+	uint8_t *iv = rte_crypto_op_ctod_offset(env.op, uint8_t *, IV_OFF);
+
+	__rte_crypto_op_reset(env.op, RTE_CRYPTO_OP_TYPE_SYMMETRIC);
+	rte_pktmbuf_reset(env.mbuf);
+
+	if (info.algo == FIPS_TEST_ALGO_AES_CCM)
+		memcpy(iv + 1, vec.iv.val, vec.iv.len);
+	else
+		memcpy(iv, vec.iv.val, vec.iv.len);
+
+	sym->m_src = env.mbuf;
+	sym->aead.data.offset = 0;
+	sym->aead.aad.data = vec.aead.aad.val;
+	sym->aead.aad.phys_addr = rte_malloc_virt2iova(sym->aead.aad.data);
+
+	if (info.op == FIPS_TEST_ENC_AUTH_GEN) {
+		uint8_t *pt;
+
+		if (vec.pt.len > RTE_MBUF_MAX_NB_SEGS) {
+			RTE_LOG(ERR, USER1, "PT len %u\n", vec.pt.len);
+			return -EPERM;
+		}
+
+		pt = (uint8_t *)rte_pktmbuf_append(env.mbuf,
+				vec.pt.len + vec.aead.digest.len);
+
+		if (!pt) {
+			RTE_LOG(ERR, USER1, "Error %i: MBUF too small\n",
+					-ENOMEM);
+			return -ENOMEM;
+		}
+
+		memcpy(pt, vec.pt.val, vec.pt.len);
+		sym->aead.data.length = vec.pt.len;
+		sym->aead.digest.data = pt + vec.pt.len;
+		sym->aead.digest.phys_addr = rte_pktmbuf_mtophys_offset(
+				env.mbuf, vec.pt.len);
+	} else {
+		uint8_t *ct;
+
+		if (vec.ct.len > RTE_MBUF_MAX_NB_SEGS) {
+			RTE_LOG(ERR, USER1, "CT len %u\n", vec.ct.len);
+			return -EPERM;
+		}
+
+		ct = (uint8_t *)rte_pktmbuf_append(env.mbuf, vec.ct.len);
+
+		if (!ct) {
+			RTE_LOG(ERR, USER1, "Error %i: MBUF too small\n",
+					-ENOMEM);
+			return -ENOMEM;
+		}
+
+		memcpy(ct, vec.ct.val, vec.ct.len);
+		sym->aead.data.length = vec.ct.len;
+		sym->aead.digest.data = vec.aead.digest.val;
+		sym->aead.digest.phys_addr = rte_malloc_virt2iova(
+				sym->aead.digest.data);
+	}
+
+	rte_crypto_op_attach_sym_session(env.op, env.sess);
+
+	return 0;
+}
+
+static int
+prepare_aes_xform(struct rte_crypto_sym_xform *xform)
+{
+	const struct rte_cryptodev_symmetric_capability *cap;
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	struct rte_crypto_cipher_xform *cipher_xform = &xform->cipher;
+
+	xform->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+
+	cipher_xform->algo = RTE_CRYPTO_CIPHER_AES_CBC;
+	cipher_xform->op = (info.op == FIPS_TEST_ENC_AUTH_GEN) ?
+			RTE_CRYPTO_CIPHER_OP_ENCRYPT :
+			RTE_CRYPTO_CIPHER_OP_DECRYPT;
+	cipher_xform->key.data = vec.cipher_auth.key.val;
+	cipher_xform->key.length = vec.cipher_auth.key.len;
+	cipher_xform->iv.length = vec.iv.len;
+	cipher_xform->iv.offset = IV_OFF;
+
+	cap_idx.algo.cipher = RTE_CRYPTO_CIPHER_AES_CBC;
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+
+	cap = rte_cryptodev_sym_capability_get(env.dev_id, &cap_idx);
+	if (!cap) {
+		RTE_LOG(ERR, USER1, "Failed to get capability for cdev %u\n",
+				env.dev_id);
+		return -EINVAL;
+	}
+
+	if (rte_cryptodev_sym_capability_check_cipher(cap,
+			cipher_xform->key.length,
+			cipher_xform->iv.length) != 0) {
+		RTE_LOG(ERR, USER1, "PMD %s key length %u IV length %u\n",
+				info.device_name, cipher_xform->key.length,
+				cipher_xform->iv.length);
+		return -EPERM;
+	}
+
+	return 0;
+}
+
+static int
+prepare_tdes_xform(struct rte_crypto_sym_xform *xform)
+{
+	const struct rte_cryptodev_symmetric_capability *cap;
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	struct rte_crypto_cipher_xform *cipher_xform = &xform->cipher;
+
+	xform->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+
+	cipher_xform->algo = RTE_CRYPTO_CIPHER_3DES_CBC;
+	cipher_xform->op = (info.op == FIPS_TEST_ENC_AUTH_GEN) ?
+			RTE_CRYPTO_CIPHER_OP_ENCRYPT :
+			RTE_CRYPTO_CIPHER_OP_DECRYPT;
+	cipher_xform->key.data = vec.cipher_auth.key.val;
+	cipher_xform->key.length = vec.cipher_auth.key.len;
+	cipher_xform->iv.length = vec.iv.len;
+	cipher_xform->iv.offset = IV_OFF;
+
+	cap_idx.algo.cipher = RTE_CRYPTO_CIPHER_3DES_CBC;
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+
+	cap = rte_cryptodev_sym_capability_get(env.dev_id, &cap_idx);
+	if (!cap) {
+		RTE_LOG(ERR, USER1, "Failed to get capability for cdev %u\n",
+				env.dev_id);
+		return -EINVAL;
+	}
+
+	if (rte_cryptodev_sym_capability_check_cipher(cap,
+			cipher_xform->key.length,
+			cipher_xform->iv.length) != 0) {
+		RTE_LOG(ERR, USER1, "PMD %s key length %u IV length %u\n",
+				info.device_name, cipher_xform->key.length,
+				cipher_xform->iv.length);
+		return -EPERM;
+	}
+
+	return 0;
+}
+
+static int
+prepare_hmac_xform(struct rte_crypto_sym_xform *xform)
+{
+	const struct rte_cryptodev_symmetric_capability *cap;
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	struct rte_crypto_auth_xform *auth_xform = &xform->auth;
+
+	xform->type = RTE_CRYPTO_SYM_XFORM_AUTH;
+
+	auth_xform->algo = info.interim_info.hmac_data.algo;
+	auth_xform->op = RTE_CRYPTO_AUTH_OP_GENERATE;
+	auth_xform->digest_length = vec.cipher_auth.digest.len;
+	auth_xform->key.data = vec.cipher_auth.key.val;
+	auth_xform->key.length = vec.cipher_auth.key.len;
+
+	cap_idx.algo.auth = auth_xform->algo;
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH;
+
+	cap = rte_cryptodev_sym_capability_get(env.dev_id, &cap_idx);
+	if (!cap) {
+		RTE_LOG(ERR, USER1, "Failed to get capability for cdev %u\n",
+				env.dev_id);
+		return -EINVAL;
+	}
+
+	if (rte_cryptodev_sym_capability_check_auth(cap,
+			auth_xform->key.length,
+			auth_xform->digest_length, 0) != 0) {
+		RTE_LOG(ERR, USER1, "PMD %s key length %u IV length %u\n",
+				info.device_name, auth_xform->key.length,
+				auth_xform->digest_length);
+		return -EPERM;
+	}
+
+	return 0;
+}
+
+static int
+prepare_gcm_xform(struct rte_crypto_sym_xform *xform)
+{
+	const struct rte_cryptodev_symmetric_capability *cap;
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	struct rte_crypto_aead_xform *aead_xform = &xform->aead;
+
+	xform->type = RTE_CRYPTO_SYM_XFORM_AEAD;
+
+	aead_xform->algo = RTE_CRYPTO_AEAD_AES_GCM;
+	aead_xform->aad_length = vec.aead.aad.len;
+	aead_xform->digest_length = vec.aead.digest.len;
+	aead_xform->iv.offset = IV_OFF;
+	aead_xform->iv.length = vec.iv.len;
+	aead_xform->key.data = vec.aead.key.val;
+	aead_xform->key.length = vec.aead.key.len;
+	aead_xform->op = (info.op == FIPS_TEST_ENC_AUTH_GEN) ?
+			RTE_CRYPTO_AEAD_OP_ENCRYPT :
+			RTE_CRYPTO_AEAD_OP_DECRYPT;
+
+	cap_idx.algo.aead = aead_xform->algo;
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AEAD;
+
+	cap = rte_cryptodev_sym_capability_get(env.dev_id, &cap_idx);
+	if (!cap) {
+		RTE_LOG(ERR, USER1, "Failed to get capability for cdev %u\n",
+				env.dev_id);
+		return -EINVAL;
+	}
+
+	if (rte_cryptodev_sym_capability_check_aead(cap,
+			aead_xform->key.length,
+			aead_xform->digest_length, aead_xform->aad_length,
+			aead_xform->iv.length) != 0) {
+		RTE_LOG(ERR, USER1,
+			"PMD %s key_len %u tag_len %u aad_len %u iv_len %u\n",
+				info.device_name, aead_xform->key.length,
+				aead_xform->digest_length,
+				aead_xform->aad_length,
+				aead_xform->iv.length);
+		return -EPERM;
+	}
+
+	return 0;
+}
+
+static int
+prepare_cmac_xform(struct rte_crypto_sym_xform *xform)
+{
+	const struct rte_cryptodev_symmetric_capability *cap;
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	struct rte_crypto_auth_xform *auth_xform = &xform->auth;
+
+	xform->type = RTE_CRYPTO_SYM_XFORM_AUTH;
+
+	auth_xform->algo = RTE_CRYPTO_AUTH_AES_CMAC;
+	auth_xform->op = (info.op == FIPS_TEST_ENC_AUTH_GEN) ?
+			RTE_CRYPTO_AUTH_OP_GENERATE : RTE_CRYPTO_AUTH_OP_VERIFY;
+	auth_xform->digest_length = vec.cipher_auth.digest.len;
+	auth_xform->key.data = vec.cipher_auth.key.val;
+	auth_xform->key.length = vec.cipher_auth.key.len;
+
+	cap_idx.algo.auth = auth_xform->algo;
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH;
+
+	cap = rte_cryptodev_sym_capability_get(env.dev_id, &cap_idx);
+	if (!cap) {
+		RTE_LOG(ERR, USER1, "Failed to get capability for cdev %u\n",
+				env.dev_id);
+		return -EINVAL;
+	}
+
+	if (rte_cryptodev_sym_capability_check_auth(cap,
+			auth_xform->key.length,
+			auth_xform->digest_length, 0) != 0) {
+		RTE_LOG(ERR, USER1, "PMD %s key length %u IV length %u\n",
+				info.device_name, auth_xform->key.length,
+				auth_xform->digest_length);
+		return -EPERM;
+	}
+
+	return 0;
+}
+
+static int
+prepare_ccm_xform(struct rte_crypto_sym_xform *xform)
+{
+	const struct rte_cryptodev_symmetric_capability *cap;
+	struct rte_cryptodev_sym_capability_idx cap_idx;
+	struct rte_crypto_aead_xform *aead_xform = &xform->aead;
+
+	xform->type = RTE_CRYPTO_SYM_XFORM_AEAD;
+
+	aead_xform->algo = RTE_CRYPTO_AEAD_AES_CCM;
+	aead_xform->aad_length = vec.aead.aad.len;
+	aead_xform->digest_length = vec.aead.digest.len;
+	aead_xform->iv.offset = IV_OFF;
+	aead_xform->iv.length = vec.iv.len;
+	aead_xform->key.data = vec.aead.key.val;
+	aead_xform->key.length = vec.aead.key.len;
+	aead_xform->op = (info.op == FIPS_TEST_ENC_AUTH_GEN) ?
+			RTE_CRYPTO_AEAD_OP_ENCRYPT :
+			RTE_CRYPTO_AEAD_OP_DECRYPT;
+
+	cap_idx.algo.aead = aead_xform->algo;
+	cap_idx.type = RTE_CRYPTO_SYM_XFORM_AEAD;
+
+	cap = rte_cryptodev_sym_capability_get(env.dev_id, &cap_idx);
+	if (!cap) {
+		RTE_LOG(ERR, USER1, "Failed to get capability for cdev %u\n",
+				env.dev_id);
+		return -EINVAL;
+	}
+
+	if (rte_cryptodev_sym_capability_check_aead(cap,
+			aead_xform->key.length,
+			aead_xform->digest_length, aead_xform->aad_length,
+			aead_xform->iv.length) != 0) {
+		RTE_LOG(ERR, USER1,
+			"PMD %s key_len %u tag_len %u aad_len %u iv_len %u\n",
+				info.device_name, aead_xform->key.length,
+				aead_xform->digest_length,
+				aead_xform->aad_length,
+				aead_xform->iv.length);
+		return -EPERM;
+	}
+
+	return 0;
+}
+
+static void
+get_writeback_data(struct fips_val *val)
+{
+	val->val = rte_pktmbuf_mtod(env.mbuf, uint8_t *);
+	val->len = rte_pktmbuf_pkt_len(env.mbuf);
+}
+
+static int
+fips_run_test(void)
+{
+	struct rte_crypto_sym_xform xform = {0};
+	uint16_t n_deqd;
+	int ret;
+
+	ret = test_ops.prepare_xform(&xform);
+	if (ret < 0)
+		return ret;
+
+	env.sess = rte_cryptodev_sym_session_create(env.mpool);
+	if (!env.sess)
+		return -ENOMEM;
+
+	ret = rte_cryptodev_sym_session_init(env.dev_id,
+			env.sess, &xform, env.mpool);
+	if (ret < 0) {
+		RTE_LOG(ERR, USER1, "Error %i: Init session\n",
+				ret);
+		return ret;
+	}
+
+	ret = test_ops.prepare_op();
+	if (ret < 0) {
+		RTE_LOG(ERR, USER1, "Error %i: Prepare op\n",
+				ret);
+		return ret;
+	}
+
+	if (rte_cryptodev_enqueue_burst(env.dev_id, 0, &env.op, 1) < 1) {
+		RTE_LOG(ERR, USER1, "Error: Failed enqueue\n");
+		return ret;
+	}
+
+	do {
+		struct rte_crypto_op *deqd_op;
+
+		n_deqd = rte_cryptodev_dequeue_burst(env.dev_id, 0, &deqd_op,
+				1);
+	} while (n_deqd == 0);
+
+	vec.status = env.op->status;
+
+	rte_cryptodev_sym_session_clear(env.dev_id, env.sess);
+	rte_cryptodev_sym_session_free(env.sess);
+	env.sess = NULL;
+
+	return ret;
+}
+
+static int
+fips_generic_test(void)
+{
+	struct fips_val val;
+	int ret;
+
+	fips_test_write_one_case();
+
+	ret = fips_run_test();
+	if (ret < 0) {
+		if (ret == -EPERM) {
+			fprintf(info.fp_wr, "Bypass\n\n");
+			return 0;
+		}
+
+		return ret;
+	}
+
+	get_writeback_data(&val);
+
+	switch (info.file_type) {
+	case FIPS_TYPE_REQ:
+	case FIPS_TYPE_RSP:
+		if (info.parse_writeback == NULL)
+			return -EPERM;
+		ret = info.parse_writeback(&val);
+		if (ret < 0)
+			return ret;
+		break;
+	case FIPS_TYPE_FAX:
+		if (info.kat_check == NULL)
+			return -EPERM;
+		ret = info.kat_check(&val);
+		if (ret < 0)
+			return ret;
+		break;
+	}
+
+	fprintf(info.fp_wr, "\n");
+
+	return 0;
+}
+
+static int
+fips_mct_tdes_test(void)
+{
+#define TDES_BLOCK_SIZE		8
+#define TDES_EXTERN_ITER	400
+#define TDES_INTERN_ITER	10000
+	struct fips_val val, val_key;
+	uint8_t prev_out[TDES_BLOCK_SIZE];
+	uint8_t prev_prev_out[TDES_BLOCK_SIZE];
+	uint8_t prev_in[TDES_BLOCK_SIZE];
+	uint32_t i, j, k;
+	int ret;
+
+	for (i = 0; i < TDES_EXTERN_ITER; i++) {
+		if (i != 0)
+			update_info_vec(i);
+
+		fips_test_write_one_case();
+
+		for (j = 0; j < TDES_INTERN_ITER; j++) {
+			ret = fips_run_test();
+			if (ret < 0) {
+				if (ret == -EPERM) {
+					fprintf(info.fp_wr, "Bypass\n");
+					return 0;
+				}
+
+				return ret;
+			}
+
+			get_writeback_data(&val);
+
+			if (info.op == FIPS_TEST_DEC_AUTH_VERIF)
+				memcpy(prev_in, vec.ct.val, TDES_BLOCK_SIZE);
+
+			if (j == 0) {
+				memcpy(prev_out, val.val, TDES_BLOCK_SIZE);
+
+				if (info.op == FIPS_TEST_ENC_AUTH_GEN) {
+					memcpy(vec.pt.val, vec.iv.val,
+							TDES_BLOCK_SIZE);
+					memcpy(vec.iv.val, val.val,
+							TDES_BLOCK_SIZE);
+				} else {
+					memcpy(vec.iv.val, vec.ct.val,
+							TDES_BLOCK_SIZE);
+					memcpy(vec.ct.val, val.val,
+							TDES_BLOCK_SIZE);
+				}
+				continue;
+			}
+
+			if (info.op == FIPS_TEST_ENC_AUTH_GEN) {
+				memcpy(vec.iv.val, val.val, TDES_BLOCK_SIZE);
+				memcpy(vec.pt.val, prev_out, TDES_BLOCK_SIZE);
+			} else {
+				memcpy(vec.iv.val, vec.ct.val, TDES_BLOCK_SIZE);
+				memcpy(vec.ct.val, val.val, TDES_BLOCK_SIZE);
+			}
+
+			if (j == TDES_INTERN_ITER - 1)
+				continue;
+
+			memcpy(prev_out, val.val, TDES_BLOCK_SIZE);
+
+			if (j == TDES_INTERN_ITER - 3)
+				memcpy(prev_prev_out, val.val, TDES_BLOCK_SIZE);
+		}
+
+		info.parse_writeback(&val);
+		fprintf(info.fp_wr, "\n");
+
+		if (i == TDES_EXTERN_ITER - 1)
+			continue;
+
+		/** update key */
+		memcpy(&val_key, &vec.cipher_auth.key, sizeof(val_key));
+
+		if (info.interim_info.tdes_data.nb_keys == 0) {
+			if (memcmp(val_key.val, val_key.val + 8, 8) == 0)
+				info.interim_info.tdes_data.nb_keys = 1;
+			else if (memcmp(val_key.val, val_key.val + 16, 8) == 0)
+				info.interim_info.tdes_data.nb_keys = 2;
+			else
+				info.interim_info.tdes_data.nb_keys = 3;
+
+		}
+
+		for (k = 0; k < TDES_BLOCK_SIZE; k++) {
+
+			switch (info.interim_info.tdes_data.nb_keys) {
+			case 3:
+				val_key.val[k] ^= val.val[k];
+				val_key.val[k + 8] ^= prev_out[k];
+				val_key.val[k + 16] ^= prev_prev_out[k];
+				break;
+			case 2:
+				val_key.val[k] ^= val.val[k];
+				val_key.val[k + 8] ^= prev_out[k];
+				val_key.val[k + 16] ^= val.val[k];
+				break;
+			default: /* case 1 */
+				val_key.val[k] ^= val.val[k];
+				val_key.val[k + 8] ^= val.val[k];
+				val_key.val[k + 16] ^= val.val[k];
+				break;
+			}
+
+		}
+
+		for (k = 0; k < 24; k++)
+			val_key.val[k] = (__builtin_popcount(val_key.val[k]) &
+					0x1) ?
+					val_key.val[k] : (val_key.val[k] ^ 0x1);
+
+		if (info.op == FIPS_TEST_ENC_AUTH_GEN) {
+			memcpy(vec.iv.val, val.val, TDES_BLOCK_SIZE);
+			memcpy(vec.pt.val, prev_out, TDES_BLOCK_SIZE);
+		} else {
+			memcpy(vec.iv.val, prev_out, TDES_BLOCK_SIZE);
+			memcpy(vec.ct.val, val.val, TDES_BLOCK_SIZE);
+		}
+	}
+
+	return 0;
+}
+
+static int
+fips_mct_aes_test(void)
+{
+#define AES_BLOCK_SIZE	16
+#define AES_EXTERN_ITER	100
+#define AES_INTERN_ITER	1000
+	struct fips_val val, val_key;
+	uint8_t prev_out[AES_BLOCK_SIZE] = {0};
+	uint8_t prev_in[AES_BLOCK_SIZE] = {0};
+	uint32_t i, j, k;
+	int ret;
+
+	for (i = 0; i < AES_EXTERN_ITER; i++) {
+		if (i != 0)
+			update_info_vec(i);
+
+		fips_test_write_one_case();
+
+		for (j = 0; j < AES_INTERN_ITER; j++) {
+			ret = fips_run_test();
+			if (ret < 0) {
+				if (ret == -EPERM) {
+					fprintf(info.fp_wr, "Bypass\n");
+					return 0;
+				}
+
+				return ret;
+			}
+
+			get_writeback_data(&val);
+
+			if (info.op == FIPS_TEST_DEC_AUTH_VERIF)
+				memcpy(prev_in, vec.ct.val, AES_BLOCK_SIZE);
+
+			if (j == 0) {
+				memcpy(prev_out, val.val, AES_BLOCK_SIZE);
+
+				if (info.op == FIPS_TEST_ENC_AUTH_GEN) {
+					memcpy(vec.pt.val, vec.iv.val,
+							AES_BLOCK_SIZE);
+					memcpy(vec.iv.val, val.val,
+							AES_BLOCK_SIZE);
+				} else {
+					memcpy(vec.ct.val, vec.iv.val,
+							AES_BLOCK_SIZE);
+					memcpy(vec.iv.val, prev_in,
+							AES_BLOCK_SIZE);
+				}
+				continue;
+			}
+
+			if (info.op == FIPS_TEST_ENC_AUTH_GEN) {
+				memcpy(vec.iv.val, val.val, AES_BLOCK_SIZE);
+				memcpy(vec.pt.val, prev_out, AES_BLOCK_SIZE);
+			} else {
+				memcpy(vec.iv.val, prev_in, AES_BLOCK_SIZE);
+				memcpy(vec.ct.val, prev_out, AES_BLOCK_SIZE);
+			}
+
+			if (j == AES_INTERN_ITER - 1)
+				continue;
+
+			memcpy(prev_out, val.val, AES_BLOCK_SIZE);
+		}
+
+		info.parse_writeback(&val);
+		fprintf(info.fp_wr, "\n");
+
+		if (i == AES_EXTERN_ITER - 1)
+			continue;
+
+		/** update key */
+		memcpy(&val_key, &vec.cipher_auth.key, sizeof(val_key));
+		for (k = 0; k < vec.cipher_auth.key.len; k++) {
+			switch (vec.cipher_auth.key.len) {
+			case 16:
+				val_key.val[k] ^= val.val[k];
+				break;
+			case 24:
+				if (k < 8)
+					val_key.val[k] ^= prev_out[k + 8];
+				else
+					val_key.val[k] ^= val.val[k - 8];
+				break;
+			case 32:
+				if (k < 16)
+					val_key.val[k] ^= prev_out[k];
+				else
+					val_key.val[k] ^= val.val[k - 16];
+				break;
+			default:
+				return -1;
+			}
+		}
+
+		if (info.op == FIPS_TEST_DEC_AUTH_VERIF)
+			memcpy(vec.iv.val, val.val, AES_BLOCK_SIZE);
+	}
+
+	return 0;
+}
+
+static int
+init_test_ops(void)
+{
+	switch (info.algo) {
+	case FIPS_TEST_ALGO_AES:
+		test_ops.prepare_op = prepare_cipher_op;
+		test_ops.prepare_xform  = prepare_aes_xform;
+		if (info.interim_info.aes_data.test_type == AESAVS_TYPE_MCT)
+			test_ops.test = fips_mct_aes_test;
+		else
+			test_ops.test = fips_generic_test;
+		break;
+	case FIPS_TEST_ALGO_HMAC:
+		test_ops.prepare_op = prepare_auth_op;
+		test_ops.prepare_xform = prepare_hmac_xform;
+		test_ops.test = fips_generic_test;
+		break;
+	case FIPS_TEST_ALGO_TDES:
+		test_ops.prepare_op = prepare_cipher_op;
+		test_ops.prepare_xform  = prepare_tdes_xform;
+		if (info.interim_info.tdes_data.test_type == TDES_MCT)
+			test_ops.test = fips_mct_tdes_test;
+		else
+			test_ops.test = fips_generic_test;
+		break;
+	case FIPS_TEST_ALGO_AES_GCM:
+		test_ops.prepare_op = prepare_aead_op;
+		test_ops.prepare_xform = prepare_gcm_xform;
+		test_ops.test = fips_generic_test;
+		break;
+	case FIPS_TEST_ALGO_AES_CMAC:
+		test_ops.prepare_op = prepare_auth_op;
+		test_ops.prepare_xform = prepare_cmac_xform;
+		test_ops.test = fips_generic_test;
+		break;
+	case FIPS_TEST_ALGO_AES_CCM:
+		test_ops.prepare_op = prepare_aead_op;
+		test_ops.prepare_xform = prepare_ccm_xform;
+		test_ops.test = fips_generic_test;
+		break;
+	default:
+		return -1;
+	}
+
+	return 0;
+}
+
+static void
+print_test_block(void)
+{
+	uint32_t i;
+
+	for (i = 0; i < info.nb_vec_lines; i++)
+		printf("%s\n", info.vec[i]);
+
+	printf("\n");
+}
+
+static int
+fips_test_one_file(void)
+{
+	int fetch_ret = 0, ret;
+
+
+	ret = init_test_ops();
+	if (ret < 0) {
+		RTE_LOG(ERR, USER1, "Error %i: Init test op\n", ret);
+		return ret;
+	}
+
+	while (ret >= 0 && fetch_ret == 0) {
+		fetch_ret = fips_test_fetch_one_block();
+		if (fetch_ret < 0) {
+			RTE_LOG(ERR, USER1, "Error %i: Fetch block\n",
+					fetch_ret);
+			ret = fetch_ret;
+			goto error_one_case;
+		}
+
+		if (info.nb_vec_lines == 0) {
+			if (fetch_ret == -EOF)
+				break;
+
+			fprintf(info.fp_wr, "\n");
+			continue;
+		}
+
+		ret = fips_test_parse_one_case();
+		switch (ret) {
+		case 0:
+			ret = test_ops.test();
+			if (ret == 0)
+				break;
+			RTE_LOG(ERR, USER1, "Error %i: test block\n",
+					ret);
+			goto error_one_case;
+		case 1:
+			break;
+		default:
+			RTE_LOG(ERR, USER1, "Error %i: Parse block\n",
+					ret);
+			goto error_one_case;
+		}
+
+		continue;
+error_one_case:
+		print_test_block();
+	}
+
+	fips_test_clear();
+
+	return ret;
+
+}
diff --git a/examples/fips_validation/meson.build b/examples/fips_validation/meson.build
new file mode 100644
index 00000000..498c9ba9
--- /dev/null
+++ b/examples/fips_validation/meson.build
@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+# meson file, for building this example as part of a main DPDK build.
+#
+# To build this example as a standalone application with an already-installed
+# DPDK instance, use 'make'
+
+deps += ['cryptodev']
+allow_experimental_apis = true
+sources = files(
+	'fips_validation_aes.c',
+	'fips_validation.c',
+	'fips_validation_hmac.c',
+	'fips_validation_tdes.c',
+	'fips_validation_gcm.c',
+	'fips_validation_cmac.c',
+	'fips_validation_ccm.c',
+	'main.c'
+)
diff --git a/examples/flow_filtering/flow_blocks.c b/examples/flow_filtering/flow_blocks.c
index 4da45928..bae71169 100644
--- a/examples/flow_filtering/flow_blocks.c
+++ b/examples/flow_filtering/flow_blocks.c
@@ -46,8 +46,6 @@ generate_ipv4_flow(uint16_t port_id, uint16_t rx_q,
 	struct rte_flow_action_queue queue = { .index = rx_q };
 	struct rte_flow_item_eth eth_spec;
 	struct rte_flow_item_eth eth_mask;
-	struct rte_flow_item_vlan vlan_spec;
-	struct rte_flow_item_vlan vlan_mask;
 	struct rte_flow_item_ipv4 ip_spec;
 	struct rte_flow_item_ipv4 ip_mask;
 	int res;
@@ -85,17 +83,6 @@ generate_ipv4_flow(uint16_t port_id, uint16_t rx_q,
 	pattern[0].mask = &eth_mask;
 
 	/*
-	 * setting the second level of the pattern (vlan).
-	 * since in this example we just want to get the
-	 * ipv4 we also set this level to allow all.
-	 */
-	memset(&vlan_spec, 0, sizeof(struct rte_flow_item_vlan));
-	memset(&vlan_mask, 0, sizeof(struct rte_flow_item_vlan));
-	pattern[1].type = RTE_FLOW_ITEM_TYPE_VLAN;
-	pattern[1].spec = &vlan_spec;
-	pattern[1].mask = &vlan_mask;
-
-	/*
 	 * setting the third level of the pattern (ip).
 	 * in this example this is the level we care about
 	 * so we set it according to the parameters.
@@ -106,12 +93,12 @@ generate_ipv4_flow(uint16_t port_id, uint16_t rx_q,
 	ip_mask.hdr.dst_addr = dest_mask;
 	ip_spec.hdr.src_addr = htonl(src_ip);
 	ip_mask.hdr.src_addr = src_mask;
-	pattern[2].type = RTE_FLOW_ITEM_TYPE_IPV4;
-	pattern[2].spec = &ip_spec;
-	pattern[2].mask = &ip_mask;
+	pattern[1].type = RTE_FLOW_ITEM_TYPE_IPV4;
+	pattern[1].spec = &ip_spec;
+	pattern[1].mask = &ip_mask;
 
 	/* the final level must be always type end */
-	pattern[3].type = RTE_FLOW_ITEM_TYPE_END;
+	pattern[2].type = RTE_FLOW_ITEM_TYPE_END;
 
 	res = rte_flow_validate(port_id, &attr, pattern, action, error);
 	if (!res)
diff --git a/examples/flow_filtering/main.c b/examples/flow_filtering/main.c
index ce91e8a6..27e287ae 100644
--- a/examples/flow_filtering/main.c
+++ b/examples/flow_filtering/main.c
@@ -121,7 +121,6 @@ init_port(void)
 	struct rte_eth_conf port_conf = {
 		.rxmode = {
 			.split_hdr_size = 0,
-			.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 		},
 		.txmode = {
 			.offloads =
@@ -132,27 +131,13 @@ init_port(void)
 				DEV_TX_OFFLOAD_SCTP_CKSUM  |
 				DEV_TX_OFFLOAD_TCP_TSO,
 		},
-		/*
-		 * Initialize fdir_conf of rte_eth_conf.
-		 * Fdir is used in flow filtering for I40e,
-		 * so rte_flow rules involve some fdir
-		 * configurations. In long term it's better
-		 * that drivers don't require any fdir
-		 * configuration for rte_flow, but we need to
-		 * get this workaround so that sample app can
-		 * run on I40e.
-		 */
-		.fdir_conf = {
-			.mode = RTE_FDIR_MODE_PERFECT,
-			.pballoc = RTE_FDIR_PBALLOC_64K,
-			.status = RTE_FDIR_REPORT_STATUS,
-			.drop_queue = 127,
-		},
 	};
 	struct rte_eth_txconf txq_conf;
 	struct rte_eth_rxconf rxq_conf;
 	struct rte_eth_dev_info dev_info;
 
+	rte_eth_dev_info_get(port_id, &dev_info);
+	port_conf.txmode.offloads &= dev_info.rx_offload_capa;
 	printf(":: initializing port: %d\n", port_id);
 	ret = rte_eth_dev_configure(port_id,
 				nr_queues, nr_queues, &port_conf);
@@ -162,7 +147,6 @@ init_port(void)
 			ret, port_id);
 	}
 
-	rte_eth_dev_info_get(port_id, &dev_info);
 	rxq_conf = dev_info.default_rxconf;
 	rxq_conf.offloads = port_conf.rxmode.offloads;
 	/* only set Rx queues: something we care only so far */
diff --git a/examples/ip_fragmentation/main.c b/examples/ip_fragmentation/main.c
index 5306d767..17a877da 100644
--- a/examples/ip_fragmentation/main.c
+++ b/examples/ip_fragmentation/main.c
@@ -141,8 +141,7 @@ static struct rte_eth_conf port_conf = {
 		.max_rx_pkt_len = JUMBO_FRAME_MAX_SIZE,
 		.split_hdr_size = 0,
 		.offloads = (DEV_RX_OFFLOAD_CHECKSUM |
-			     DEV_RX_OFFLOAD_JUMBO_FRAME |
-			     DEV_RX_OFFLOAD_CRC_STRIP),
+			     DEV_RX_OFFLOAD_JUMBO_FRAME),
 	},
 	.txmode = {
 		.mq_mode = ETH_MQ_TX_NONE,
diff --git a/examples/ip_pipeline/Makefile b/examples/ip_pipeline/Makefile
index 3fb98ce3..41ba7df2 100644
--- a/examples/ip_pipeline/Makefile
+++ b/examples/ip_pipeline/Makefile
@@ -18,6 +18,7 @@ SRCS-y += swq.c
 SRCS-y += tap.c
 SRCS-y += thread.c
 SRCS-y += tmgr.c
+SRCS-y += cryptodev.c
 
 # Build using pkg-config variables if possible
 $(shell pkg-config --exists libdpdk)
diff --git a/examples/ip_pipeline/action.c b/examples/ip_pipeline/action.c
index a29c2b36..d2104aad 100644
--- a/examples/ip_pipeline/action.c
+++ b/examples/ip_pipeline/action.c
@@ -7,9 +7,9 @@
 #include <string.h>
 
 #include <rte_string_fns.h>
+#include <rte_table_hash_func.h>
 
 #include "action.h"
-#include "hash_func.h"
 
 /**
  * Input port
@@ -57,35 +57,35 @@ port_in_action_profile_create(const char *name,
 		(params->lb.f_hash == NULL)) {
 		switch (params->lb.key_size) {
 		case  8:
-			params->lb.f_hash = hash_default_key8;
+			params->lb.f_hash = rte_table_hash_crc_key8;
 			break;
 
 		case 16:
-			params->lb.f_hash = hash_default_key16;
+			params->lb.f_hash = rte_table_hash_crc_key16;
 			break;
 
 		case 24:
-			params->lb.f_hash = hash_default_key24;
+			params->lb.f_hash = rte_table_hash_crc_key24;
 			break;
 
 		case 32:
-			params->lb.f_hash = hash_default_key32;
+			params->lb.f_hash = rte_table_hash_crc_key32;
 			break;
 
 		case 40:
-			params->lb.f_hash = hash_default_key40;
+			params->lb.f_hash = rte_table_hash_crc_key40;
 			break;
 
 		case 48:
-			params->lb.f_hash = hash_default_key48;
+			params->lb.f_hash = rte_table_hash_crc_key48;
 			break;
 
 		case 56:
-			params->lb.f_hash = hash_default_key56;
+			params->lb.f_hash = rte_table_hash_crc_key56;
 			break;
 
 		case 64:
-			params->lb.f_hash = hash_default_key64;
+			params->lb.f_hash = rte_table_hash_crc_key64;
 			break;
 
 		default:
@@ -192,35 +192,35 @@ table_action_profile_create(const char *name,
 		(params->lb.f_hash == NULL)) {
 		switch (params->lb.key_size) {
 		case 8:
-			params->lb.f_hash = hash_default_key8;
+			params->lb.f_hash = rte_table_hash_crc_key8;
 			break;
 
 		case 16:
-			params->lb.f_hash = hash_default_key16;
+			params->lb.f_hash = rte_table_hash_crc_key16;
 			break;
 
 		case 24:
-			params->lb.f_hash = hash_default_key24;
+			params->lb.f_hash = rte_table_hash_crc_key24;
 			break;
 
 		case 32:
-			params->lb.f_hash = hash_default_key32;
+			params->lb.f_hash = rte_table_hash_crc_key32;
 			break;
 
 		case 40:
-			params->lb.f_hash = hash_default_key40;
+			params->lb.f_hash = rte_table_hash_crc_key40;
 			break;
 
 		case 48:
-			params->lb.f_hash = hash_default_key48;
+			params->lb.f_hash = rte_table_hash_crc_key48;
 			break;
 
 		case 56:
-			params->lb.f_hash = hash_default_key56;
+			params->lb.f_hash = rte_table_hash_crc_key56;
 			break;
 
 		case 64:
-			params->lb.f_hash = hash_default_key64;
+			params->lb.f_hash = rte_table_hash_crc_key64;
 			break;
 
 		default:
@@ -333,6 +333,39 @@ table_action_profile_create(const char *name,
 		}
 	}
 
+	if (params->action_mask & (1LLU << RTE_TABLE_ACTION_SYM_CRYPTO)) {
+		status = rte_table_action_profile_action_register(ap,
+			RTE_TABLE_ACTION_SYM_CRYPTO,
+			&params->sym_crypto);
+
+		if (status) {
+			rte_table_action_profile_free(ap);
+			return NULL;
+		}
+	}
+
+	if (params->action_mask & (1LLU << RTE_TABLE_ACTION_TAG)) {
+		status = rte_table_action_profile_action_register(ap,
+			RTE_TABLE_ACTION_TAG,
+			NULL);
+
+		if (status) {
+			rte_table_action_profile_free(ap);
+			return NULL;
+		}
+	}
+
+	if (params->action_mask & (1LLU << RTE_TABLE_ACTION_DECAP)) {
+		status = rte_table_action_profile_action_register(ap,
+			RTE_TABLE_ACTION_DECAP,
+			NULL);
+
+		if (status) {
+			rte_table_action_profile_free(ap);
+			return NULL;
+		}
+	}
+
 	status = rte_table_action_profile_freeze(ap);
 	if (status) {
 		rte_table_action_profile_free(ap);
diff --git a/examples/ip_pipeline/action.h b/examples/ip_pipeline/action.h
index 417200e8..cde17e69 100644
--- a/examples/ip_pipeline/action.h
+++ b/examples/ip_pipeline/action.h
@@ -53,6 +53,7 @@ struct table_action_profile_params {
 	struct rte_table_action_nat_config nat;
 	struct rte_table_action_ttl_config ttl;
 	struct rte_table_action_stats_config stats;
+	struct rte_table_action_sym_crypto_config sym_crypto;
 };
 
 struct table_action_profile {
diff --git a/examples/ip_pipeline/cli.c b/examples/ip_pipeline/cli.c
index 102a1d6b..3de62068 100644
--- a/examples/ip_pipeline/cli.c
+++ b/examples/ip_pipeline/cli.c
@@ -12,6 +12,8 @@
 #include <rte_ethdev.h>
 
 #include "cli.h"
+
+#include "cryptodev.h"
 #include "kni.h"
 #include "link.h"
 #include "mempool.h"
@@ -785,6 +787,65 @@ cmd_kni(char **tokens,
 	}
 }
 
+static const char cmd_cryptodev_help[] =
+"cryptodev <cryptodev_name>\n"
+"   dev <device_name> | dev_id <device_id>\n"
+"   queue <n_queues> <queue_size>\n";
+
+static void
+cmd_cryptodev(char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	struct cryptodev_params params;
+	char *name;
+
+	memset(&params, 0, sizeof(params));
+	if (n_tokens != 7) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	name = tokens[1];
+
+	if (strcmp(tokens[2], "dev") == 0)
+		params.dev_name = tokens[3];
+	else if (strcmp(tokens[2], "dev_id") == 0) {
+		if (parser_read_uint32(&params.dev_id, tokens[3]) < 0) {
+			snprintf(out, out_size,	MSG_ARG_INVALID,
+				"dev_id");
+			return;
+		}
+	} else {
+		snprintf(out, out_size,	MSG_ARG_INVALID,
+			"cryptodev");
+		return;
+	}
+
+	if (strcmp(tokens[4], "queue")) {
+		snprintf(out, out_size,	MSG_ARG_NOT_FOUND,
+			"4");
+		return;
+	}
+
+	if (parser_read_uint32(&params.n_queues, tokens[5]) < 0) {
+		snprintf(out, out_size,	MSG_ARG_INVALID,
+			"q");
+		return;
+	}
+
+	if (parser_read_uint32(&params.queue_size, tokens[6]) < 0) {
+		snprintf(out, out_size,	MSG_ARG_INVALID,
+			"queue_size");
+		return;
+	}
+
+	if (cryptodev_create(name, &params) == NULL) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
 
 static const char cmd_port_in_action_profile_help[] =
 "port in action profile <profile_name>\n"
@@ -961,13 +1022,19 @@ static const char cmd_table_action_profile_help[] =
 "       tc <n_tc>\n"
 "       stats none | pkts | bytes | both]\n"
 "   [tm spp <n_subports_per_port> pps <n_pipes_per_subport>]\n"
-"   [encap ether | vlan | qinq | mpls | pppoe]\n"
+"   [encap ether | vlan | qinq | mpls | pppoe |\n"
+"       vxlan offset <ether_offset> ipv4 | ipv6 vlan on | off]\n"
 "   [nat src | dst\n"
 "       proto udp | tcp]\n"
 "   [ttl drop | fwd\n"
 "       stats none | pkts]\n"
 "   [stats pkts | bytes | both]\n"
-"   [time]\n";
+"   [time]\n"
+"   [sym_crypto dev <CRYPTODEV_NAME> offset <op_offset> "
+"       mempool_create <mempool_name>\n"
+"       mempool_init <mempool_name>]\n"
+"   [tag]\n"
+"   [decap]\n";
 
 static void
 cmd_table_action_profile(char **tokens,
@@ -1157,6 +1224,8 @@ cmd_table_action_profile(char **tokens,
 	} /* tm */
 
 	if ((t0 < n_tokens) && (strcmp(tokens[t0], "encap") == 0)) {
+		uint32_t n_extra_tokens = 0;
+
 		if (n_tokens < t0 + 2) {
 			snprintf(out, out_size, MSG_ARG_MISMATCH,
 				"action profile encap");
@@ -1173,13 +1242,61 @@ cmd_table_action_profile(char **tokens,
 			p.encap.encap_mask = 1LLU << RTE_TABLE_ACTION_ENCAP_MPLS;
 		else if (strcmp(tokens[t0 + 1], "pppoe") == 0)
 			p.encap.encap_mask = 1LLU << RTE_TABLE_ACTION_ENCAP_PPPOE;
-		else {
+		else if (strcmp(tokens[t0 + 1], "vxlan") == 0) {
+			if (n_tokens < t0 + 2 + 5) {
+				snprintf(out, out_size, MSG_ARG_MISMATCH,
+					"action profile encap vxlan");
+				return;
+			}
+
+			if (strcmp(tokens[t0 + 2], "offset") != 0) {
+				snprintf(out, out_size, MSG_ARG_NOT_FOUND,
+					"vxlan: offset");
+				return;
+			}
+
+			if (parser_read_uint32(&p.encap.vxlan.data_offset,
+				tokens[t0 + 2 + 1]) != 0) {
+				snprintf(out, out_size, MSG_ARG_INVALID,
+					"vxlan: ether_offset");
+				return;
+			}
+
+			if (strcmp(tokens[t0 + 2 + 2], "ipv4") == 0)
+				p.encap.vxlan.ip_version = 1;
+			else if (strcmp(tokens[t0 + 2 + 2], "ipv6") == 0)
+				p.encap.vxlan.ip_version = 0;
+			else {
+				snprintf(out, out_size, MSG_ARG_INVALID,
+					"vxlan: ipv4 or ipv6");
+				return;
+			}
+
+			if (strcmp(tokens[t0 + 2 + 3], "vlan") != 0) {
+				snprintf(out, out_size, MSG_ARG_NOT_FOUND,
+					"vxlan: vlan");
+				return;
+			}
+
+			if (strcmp(tokens[t0 + 2 + 4], "on") == 0)
+				p.encap.vxlan.vlan = 1;
+			else if (strcmp(tokens[t0 + 2 + 4], "off") == 0)
+				p.encap.vxlan.vlan = 0;
+			else {
+				snprintf(out, out_size, MSG_ARG_INVALID,
+					"vxlan: on or off");
+				return;
+			}
+
+			p.encap.encap_mask = 1LLU << RTE_TABLE_ACTION_ENCAP_VXLAN;
+			n_extra_tokens = 5;
+		} else {
 			snprintf(out, out_size, MSG_ARG_MISMATCH, "encap");
 			return;
 		}
 
 		p.action_mask |= 1LLU << RTE_TABLE_ACTION_ENCAP;
-		t0 += 2;
+		t0 += 2 + n_extra_tokens;
 	} /* encap */
 
 	if ((t0 < n_tokens) && (strcmp(tokens[t0], "nat") == 0)) {
@@ -1285,6 +1402,67 @@ cmd_table_action_profile(char **tokens,
 		t0 += 1;
 	} /* time */
 
+	if ((t0 < n_tokens) && (strcmp(tokens[t0], "sym_crypto") == 0)) {
+		struct cryptodev *cryptodev;
+		struct mempool *mempool;
+
+		if (n_tokens < t0 + 9 ||
+				strcmp(tokens[t0 + 1], "dev") ||
+				strcmp(tokens[t0 + 3], "offset") ||
+				strcmp(tokens[t0 + 5], "mempool_create") ||
+				strcmp(tokens[t0 + 7], "mempool_init")) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"table action profile sym_crypto");
+			return;
+		}
+
+		cryptodev = cryptodev_find(tokens[t0 + 2]);
+		if (cryptodev == NULL) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"table action profile sym_crypto");
+			return;
+		}
+
+		p.sym_crypto.cryptodev_id = cryptodev->dev_id;
+
+		if (parser_read_uint32(&p.sym_crypto.op_offset,
+				tokens[t0 + 4]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+					"table action profile sym_crypto");
+			return;
+		}
+
+		mempool = mempool_find(tokens[t0 + 6]);
+		if (mempool == NULL) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"table action profile sym_crypto");
+			return;
+		}
+		p.sym_crypto.mp_create = mempool->m;
+
+		mempool = mempool_find(tokens[t0 + 8]);
+		if (mempool == NULL) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"table action profile sym_crypto");
+			return;
+		}
+		p.sym_crypto.mp_init = mempool->m;
+
+		p.action_mask |= 1LLU << RTE_TABLE_ACTION_SYM_CRYPTO;
+
+		t0 += 9;
+	} /* sym_crypto */
+
+	if ((t0 < n_tokens) && (strcmp(tokens[t0], "tag") == 0)) {
+		p.action_mask |= 1LLU << RTE_TABLE_ACTION_TAG;
+		t0 += 1;
+	} /* tag */
+
+	if ((t0 < n_tokens) && (strcmp(tokens[t0], "decap") == 0)) {
+		p.action_mask |= 1LLU << RTE_TABLE_ACTION_DECAP;
+		t0 += 1;
+	} /* decap */
+
 	if (t0 < n_tokens) {
 		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
 		return;
@@ -1366,6 +1544,7 @@ static const char cmd_pipeline_port_in_help[] =
 "   | tap <tap_name> mempool <mempool_name> mtu <mtu>\n"
 "   | kni <kni_name>\n"
 "   | source mempool <mempool_name> file <file_name> bpp <n_bytes_per_pkt>\n"
+"   | cryptodev <cryptodev_name> rxq <queue_id>\n"
 "   [action <port_in_action_profile_name>]\n"
 "   [disabled]\n";
 
@@ -1538,6 +1717,26 @@ cmd_pipeline_port_in(char **tokens,
 		}
 
 		t0 += 7;
+	} else if (strcmp(tokens[t0], "cryptodev") == 0) {
+		if (n_tokens < t0 + 3) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"pipeline port in cryptodev");
+			return;
+		}
+
+		p.type = PORT_IN_CRYPTODEV;
+
+		p.dev_name = tokens[t0 + 1];
+		if (parser_read_uint16(&p.rxq.queue_id, tokens[t0 + 3]) != 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"rxq");
+			return;
+		}
+
+		p.cryptodev.arg_callback = NULL;
+		p.cryptodev.f_callback = NULL;
+
+		t0 += 4;
 	} else {
 		snprintf(out, out_size, MSG_ARG_INVALID, tokens[0]);
 		return;
@@ -1584,7 +1783,8 @@ static const char cmd_pipeline_port_out_help[] =
 "   | tmgr <tmgr_name>\n"
 "   | tap <tap_name>\n"
 "   | kni <kni_name>\n"
-"   | sink [file <file_name> pkts <max_n_pkts>]\n";
+"   | sink [file <file_name> pkts <max_n_pkts>]\n"
+"   | cryptodev <cryptodev_name> txq <txq_id> offset <crypto_op_offset>\n";
 
 static void
 cmd_pipeline_port_out(char **tokens,
@@ -1718,6 +1918,41 @@ cmd_pipeline_port_out(char **tokens,
 				return;
 			}
 		}
+
+	} else if (strcmp(tokens[6], "cryptodev") == 0) {
+		if (n_tokens != 12) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"pipeline port out cryptodev");
+			return;
+		}
+
+		p.type = PORT_OUT_CRYPTODEV;
+
+		p.dev_name = tokens[7];
+
+		if (strcmp(tokens[8], "txq")) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"pipeline port out cryptodev");
+			return;
+		}
+
+		if (parser_read_uint16(&p.cryptodev.queue_id, tokens[9])
+				!= 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "queue_id");
+			return;
+		}
+
+		if (strcmp(tokens[10], "offset")) {
+			snprintf(out, out_size, MSG_ARG_MISMATCH,
+				"pipeline port out cryptodev");
+			return;
+		}
+
+		if (parser_read_uint32(&p.cryptodev.op_offset, tokens[11])
+				!= 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID, "queue_id");
+			return;
+		}
 	} else {
 		snprintf(out, out_size, MSG_ARG_INVALID, tokens[0]);
 		return;
@@ -2861,11 +3096,31 @@ parse_match(char **tokens,
  *          [label1 <label> <tc> <ttl>
  *          [label2 <label> <tc> <ttl>
  *          [label3 <label> <tc> <ttl>]]]
- *       | pppoe <da> <sa> <session_id>]
+ *       | pppoe <da> <sa> <session_id>
+ *       | vxlan ether <da> <sa>
+ *          [vlan <pcp> <dei> <vid>]
+ *          ipv4 <sa> <da> <dscp> <ttl>
+ *          | ipv6 <sa> <da> <flow_label> <dscp> <hop_limit>
+ *          udp <sp> <dp>
+ *          vxlan <vni>]
  *    [nat ipv4 | ipv6 <addr> <port>]
  *    [ttl dec | keep]
  *    [stats]
  *    [time]
+ *    [sym_crypto
+ *       encrypt | decrypt
+ *       type
+ *       | cipher
+ *          cipher_algo <algo> cipher_key <key> cipher_iv <iv>
+ *       | cipher_auth
+ *          cipher_algo <algo> cipher_key <key> cipher_iv <iv>
+ *          auth_algo <algo> auth_key <key> digest_size <size>
+ *       | aead
+ *          aead_algo <algo> aead_key <key> aead_iv <iv> aead_aad <aad>
+ *          digest_size <size>
+ *       data_offset <data_offset>]
+ *    [tag <tag>]
+ *    [decap <n>]
  *
  * where:
  *    <pa> ::= g | y | r | drop
@@ -3254,6 +3509,122 @@ parse_table_action_encap(char **tokens,
 		return 1 + 4;
 	}
 
+	/* vxlan */
+	if (n_tokens && (strcmp(tokens[0], "vxlan") == 0)) {
+		uint32_t n = 0;
+
+		n_tokens--;
+		tokens++;
+		n++;
+
+		/* ether <da> <sa> */
+		if ((n_tokens < 3) ||
+			strcmp(tokens[0], "ether") ||
+			parse_mac_addr(tokens[1], &a->encap.vxlan.ether.da) ||
+			parse_mac_addr(tokens[2], &a->encap.vxlan.ether.sa))
+			return 0;
+
+		n_tokens -= 3;
+		tokens += 3;
+		n += 3;
+
+		/* [vlan <pcp> <dei> <vid>] */
+		if (strcmp(tokens[0], "vlan") == 0) {
+			uint32_t pcp, dei, vid;
+
+			if ((n_tokens < 4) ||
+				parser_read_uint32(&pcp, tokens[1]) ||
+				(pcp > 7) ||
+				parser_read_uint32(&dei, tokens[2]) ||
+				(dei > 1) ||
+				parser_read_uint32(&vid, tokens[3]) ||
+				(vid > 0xFFF))
+				return 0;
+
+			a->encap.vxlan.vlan.pcp = pcp;
+			a->encap.vxlan.vlan.dei = dei;
+			a->encap.vxlan.vlan.vid = vid;
+
+			n_tokens -= 4;
+			tokens += 4;
+			n += 4;
+		}
+
+		/* ipv4 <sa> <da> <dscp> <ttl>
+		   | ipv6 <sa> <da> <flow_label> <dscp> <hop_limit> */
+		if (strcmp(tokens[0], "ipv4") == 0) {
+			struct in_addr sa, da;
+			uint8_t dscp, ttl;
+
+			if ((n_tokens < 5) ||
+				parse_ipv4_addr(tokens[1], &sa) ||
+				parse_ipv4_addr(tokens[2], &da) ||
+				parser_read_uint8(&dscp, tokens[3]) ||
+				(dscp > 64) ||
+				parser_read_uint8(&ttl, tokens[4]))
+				return 0;
+
+			a->encap.vxlan.ipv4.sa = rte_be_to_cpu_32(sa.s_addr);
+			a->encap.vxlan.ipv4.da = rte_be_to_cpu_32(da.s_addr);
+			a->encap.vxlan.ipv4.dscp = dscp;
+			a->encap.vxlan.ipv4.ttl = ttl;
+
+			n_tokens -= 5;
+			tokens += 5;
+			n += 5;
+		} else if (strcmp(tokens[0], "ipv6") == 0) {
+			struct in6_addr sa, da;
+			uint32_t flow_label;
+			uint8_t dscp, hop_limit;
+
+			if ((n_tokens < 6) ||
+				parse_ipv6_addr(tokens[1], &sa) ||
+				parse_ipv6_addr(tokens[2], &da) ||
+				parser_read_uint32(&flow_label, tokens[3]) ||
+				parser_read_uint8(&dscp, tokens[4]) ||
+				(dscp > 64) ||
+				parser_read_uint8(&hop_limit, tokens[5]))
+				return 0;
+
+			memcpy(a->encap.vxlan.ipv6.sa, sa.s6_addr, 16);
+			memcpy(a->encap.vxlan.ipv6.da, da.s6_addr, 16);
+			a->encap.vxlan.ipv6.flow_label = flow_label;
+			a->encap.vxlan.ipv6.dscp = dscp;
+			a->encap.vxlan.ipv6.hop_limit = hop_limit;
+
+			n_tokens -= 6;
+			tokens += 6;
+			n += 6;
+		} else
+			return 0;
+
+		/* udp <sp> <dp> */
+		if ((n_tokens < 3) ||
+			strcmp(tokens[0], "udp") ||
+			parser_read_uint16(&a->encap.vxlan.udp.sp, tokens[1]) ||
+			parser_read_uint16(&a->encap.vxlan.udp.dp, tokens[2]))
+			return 0;
+
+		n_tokens -= 3;
+		tokens += 3;
+		n += 3;
+
+		/* vxlan <vni> */
+		if ((n_tokens < 2) ||
+			strcmp(tokens[0], "vxlan") ||
+			parser_read_uint32(&a->encap.vxlan.vxlan.vni, tokens[1]) ||
+			(a->encap.vxlan.vxlan.vni > 0xFFFFFF))
+			return 0;
+
+		n_tokens -= 2;
+		tokens += 2;
+		n += 2;
+
+		a->encap.type = RTE_TABLE_ACTION_ENCAP_VXLAN;
+		a->action_mask |= 1 << RTE_TABLE_ACTION_ENCAP;
+		return 1 + n;
+	}
+
 	return 0;
 }
 
@@ -3348,6 +3719,400 @@ parse_table_action_time(char **tokens,
 	return 1;
 }
 
+static void
+parse_free_sym_crypto_param_data(struct rte_table_action_sym_crypto_params *p)
+{
+	struct rte_crypto_sym_xform *xform[2] = {NULL};
+	uint32_t i;
+
+	xform[0] = p->xform;
+	if (xform[0])
+		xform[1] = xform[0]->next;
+
+	for (i = 0; i < 2; i++) {
+		if (xform[i] == NULL)
+			continue;
+
+		switch (xform[i]->type) {
+		case RTE_CRYPTO_SYM_XFORM_CIPHER:
+			if (xform[i]->cipher.key.data)
+				free(xform[i]->cipher.key.data);
+			if (p->cipher_auth.cipher_iv.val)
+				free(p->cipher_auth.cipher_iv.val);
+			if (p->cipher_auth.cipher_iv_update.val)
+				free(p->cipher_auth.cipher_iv_update.val);
+			break;
+		case RTE_CRYPTO_SYM_XFORM_AUTH:
+			if (xform[i]->auth.key.data)
+				free(xform[i]->cipher.key.data);
+			if (p->cipher_auth.auth_iv.val)
+				free(p->cipher_auth.cipher_iv.val);
+			if (p->cipher_auth.auth_iv_update.val)
+				free(p->cipher_auth.cipher_iv_update.val);
+			break;
+		case RTE_CRYPTO_SYM_XFORM_AEAD:
+			if (xform[i]->aead.key.data)
+				free(xform[i]->cipher.key.data);
+			if (p->aead.iv.val)
+				free(p->aead.iv.val);
+			if (p->aead.aad.val)
+				free(p->aead.aad.val);
+			break;
+		default:
+			continue;
+		}
+	}
+
+}
+
+static struct rte_crypto_sym_xform *
+parse_table_action_cipher(struct rte_table_action_sym_crypto_params *p,
+		char **tokens, uint32_t n_tokens, uint32_t encrypt,
+		uint32_t *used_n_tokens)
+{
+	struct rte_crypto_sym_xform *xform_cipher;
+	int status;
+	size_t len;
+
+	if (n_tokens < 7 || strcmp(tokens[1], "cipher_algo") ||
+			strcmp(tokens[3], "cipher_key") ||
+			strcmp(tokens[5], "cipher_iv"))
+		return NULL;
+
+	xform_cipher = calloc(1, sizeof(*xform_cipher));
+	if (xform_cipher == NULL)
+		return NULL;
+
+	xform_cipher->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
+	xform_cipher->cipher.op = encrypt ? RTE_CRYPTO_CIPHER_OP_ENCRYPT :
+			RTE_CRYPTO_CIPHER_OP_DECRYPT;
+
+	/* cipher_algo */
+	status = rte_cryptodev_get_cipher_algo_enum(
+			&xform_cipher->cipher.algo, tokens[2]);
+	if (status < 0)
+		goto error_exit;
+
+	/* cipher_key */
+	len = strlen(tokens[4]);
+	xform_cipher->cipher.key.data = calloc(1, len / 2 + 1);
+	if (xform_cipher->cipher.key.data == NULL)
+		goto error_exit;
+
+	status = parse_hex_string(tokens[4],
+			xform_cipher->cipher.key.data,
+			(uint32_t *)&len);
+	if (status < 0)
+		goto error_exit;
+
+	xform_cipher->cipher.key.length = (uint16_t)len;
+
+	/* cipher_iv */
+	len = strlen(tokens[6]);
+
+	p->cipher_auth.cipher_iv.val = calloc(1, len / 2 + 1);
+	if (p->cipher_auth.cipher_iv.val == NULL)
+		goto error_exit;
+
+	status = parse_hex_string(tokens[6],
+			p->cipher_auth.cipher_iv.val,
+			(uint32_t *)&len);
+	if (status < 0)
+		goto error_exit;
+
+	xform_cipher->cipher.iv.length = (uint16_t)len;
+	xform_cipher->cipher.iv.offset = RTE_TABLE_ACTION_SYM_CRYPTO_IV_OFFSET;
+	p->cipher_auth.cipher_iv.length = (uint32_t)len;
+	*used_n_tokens = 7;
+
+	return xform_cipher;
+
+error_exit:
+	if (xform_cipher->cipher.key.data)
+		free(xform_cipher->cipher.key.data);
+
+	if (p->cipher_auth.cipher_iv.val) {
+		free(p->cipher_auth.cipher_iv.val);
+		p->cipher_auth.cipher_iv.val = NULL;
+	}
+
+	free(xform_cipher);
+
+	return NULL;
+}
+
+static struct rte_crypto_sym_xform *
+parse_table_action_cipher_auth(struct rte_table_action_sym_crypto_params *p,
+		char **tokens, uint32_t n_tokens, uint32_t encrypt,
+		uint32_t *used_n_tokens)
+{
+	struct rte_crypto_sym_xform *xform_cipher;
+	struct rte_crypto_sym_xform *xform_auth;
+	int status;
+	size_t len;
+
+	if (n_tokens < 13 ||
+			strcmp(tokens[7], "auth_algo") ||
+			strcmp(tokens[9], "auth_key") ||
+			strcmp(tokens[11], "digest_size"))
+		return NULL;
+
+	xform_auth = calloc(1, sizeof(*xform_auth));
+	if (xform_auth == NULL)
+		return NULL;
+
+	xform_auth->type = RTE_CRYPTO_SYM_XFORM_AUTH;
+	xform_auth->auth.op = encrypt ? RTE_CRYPTO_AUTH_OP_GENERATE :
+			RTE_CRYPTO_AUTH_OP_VERIFY;
+
+	/* auth_algo */
+	status = rte_cryptodev_get_auth_algo_enum(&xform_auth->auth.algo,
+			tokens[8]);
+	if (status < 0)
+		goto error_exit;
+
+	/* auth_key */
+	len = strlen(tokens[10]);
+	xform_auth->auth.key.data = calloc(1, len / 2 + 1);
+	if (xform_auth->auth.key.data == NULL)
+		goto error_exit;
+
+	status = parse_hex_string(tokens[10],
+			xform_auth->auth.key.data, (uint32_t *)&len);
+	if (status < 0)
+		goto error_exit;
+
+	xform_auth->auth.key.length = (uint16_t)len;
+
+	if (strcmp(tokens[11], "digest_size"))
+		goto error_exit;
+
+	status = parser_read_uint16(&xform_auth->auth.digest_length,
+			tokens[12]);
+	if (status < 0)
+		goto error_exit;
+
+	xform_cipher = parse_table_action_cipher(p, tokens, 7, encrypt,
+			used_n_tokens);
+	if (xform_cipher == NULL)
+		goto error_exit;
+
+	*used_n_tokens += 6;
+
+	if (encrypt) {
+		xform_cipher->next = xform_auth;
+		return xform_cipher;
+	} else {
+		xform_auth->next = xform_cipher;
+		return xform_auth;
+	}
+
+error_exit:
+	if (xform_auth->auth.key.data)
+		free(xform_auth->auth.key.data);
+	if (p->cipher_auth.auth_iv.val) {
+		free(p->cipher_auth.auth_iv.val);
+		p->cipher_auth.auth_iv.val = 0;
+	}
+
+	free(xform_auth);
+
+	return NULL;
+}
+
+static struct rte_crypto_sym_xform *
+parse_table_action_aead(struct rte_table_action_sym_crypto_params *p,
+		char **tokens, uint32_t n_tokens, uint32_t encrypt,
+		uint32_t *used_n_tokens)
+{
+	struct rte_crypto_sym_xform *xform_aead;
+	int status;
+	size_t len;
+
+	if (n_tokens < 11 || strcmp(tokens[1], "aead_algo") ||
+			strcmp(tokens[3], "aead_key") ||
+			strcmp(tokens[5], "aead_iv") ||
+			strcmp(tokens[7], "aead_aad") ||
+			strcmp(tokens[9], "digest_size"))
+		return NULL;
+
+	xform_aead = calloc(1, sizeof(*xform_aead));
+	if (xform_aead == NULL)
+		return NULL;
+
+	xform_aead->type = RTE_CRYPTO_SYM_XFORM_AEAD;
+	xform_aead->aead.op = encrypt ? RTE_CRYPTO_AEAD_OP_ENCRYPT :
+			RTE_CRYPTO_AEAD_OP_DECRYPT;
+
+	/* aead_algo */
+	status = rte_cryptodev_get_aead_algo_enum(&xform_aead->aead.algo,
+			tokens[2]);
+	if (status < 0)
+		goto error_exit;
+
+	/* aead_key */
+	len = strlen(tokens[4]);
+	xform_aead->aead.key.data = calloc(1, len / 2 + 1);
+	if (xform_aead->aead.key.data == NULL)
+		goto error_exit;
+
+	status = parse_hex_string(tokens[4], xform_aead->aead.key.data,
+			(uint32_t *)&len);
+	if (status < 0)
+		goto error_exit;
+
+	xform_aead->aead.key.length = (uint16_t)len;
+
+	/* aead_iv */
+	len = strlen(tokens[6]);
+	p->aead.iv.val = calloc(1, len / 2 + 1);
+	if (p->aead.iv.val == NULL)
+		goto error_exit;
+
+	status = parse_hex_string(tokens[6], p->aead.iv.val,
+			(uint32_t *)&len);
+	if (status < 0)
+		goto error_exit;
+
+	xform_aead->aead.iv.length = (uint16_t)len;
+	xform_aead->aead.iv.offset = RTE_TABLE_ACTION_SYM_CRYPTO_IV_OFFSET;
+	p->aead.iv.length = (uint32_t)len;
+
+	/* aead_aad */
+	len = strlen(tokens[8]);
+	p->aead.aad.val = calloc(1, len / 2 + 1);
+	if (p->aead.aad.val == NULL)
+		goto error_exit;
+
+	status = parse_hex_string(tokens[8], p->aead.aad.val, (uint32_t *)&len);
+	if (status < 0)
+		goto error_exit;
+
+	xform_aead->aead.aad_length = (uint16_t)len;
+	p->aead.aad.length = (uint32_t)len;
+
+	/* digest_size */
+	status = parser_read_uint16(&xform_aead->aead.digest_length,
+			tokens[10]);
+	if (status < 0)
+		goto error_exit;
+
+	*used_n_tokens = 11;
+
+	return xform_aead;
+
+error_exit:
+	if (xform_aead->aead.key.data)
+		free(xform_aead->aead.key.data);
+	if (p->aead.iv.val) {
+		free(p->aead.iv.val);
+		p->aead.iv.val = NULL;
+	}
+	if (p->aead.aad.val) {
+		free(p->aead.aad.val);
+		p->aead.aad.val = NULL;
+	}
+
+	free(xform_aead);
+
+	return NULL;
+}
+
+
+static uint32_t
+parse_table_action_sym_crypto(char **tokens,
+	uint32_t n_tokens,
+	struct table_rule_action *a)
+{
+	struct rte_table_action_sym_crypto_params *p = &a->sym_crypto;
+	struct rte_crypto_sym_xform *xform = NULL;
+	uint32_t used_n_tokens;
+	uint32_t encrypt;
+	int status;
+
+	if ((n_tokens < 12) ||
+		strcmp(tokens[0], "sym_crypto") ||
+		strcmp(tokens[2], "type"))
+		return 0;
+
+	memset(p, 0, sizeof(*p));
+
+	if (strcmp(tokens[1], "encrypt") == 0)
+		encrypt = 1;
+	else
+		encrypt = 0;
+
+	status = parser_read_uint32(&p->data_offset, tokens[n_tokens - 1]);
+	if (status < 0)
+		return 0;
+
+	if (strcmp(tokens[3], "cipher") == 0) {
+		tokens += 3;
+		n_tokens -= 3;
+
+		xform = parse_table_action_cipher(p, tokens, n_tokens, encrypt,
+				&used_n_tokens);
+	} else if (strcmp(tokens[3], "cipher_auth") == 0) {
+		tokens += 3;
+		n_tokens -= 3;
+
+		xform = parse_table_action_cipher_auth(p, tokens, n_tokens,
+				encrypt, &used_n_tokens);
+	} else if (strcmp(tokens[3], "aead") == 0) {
+		tokens += 3;
+		n_tokens -= 3;
+
+		xform = parse_table_action_aead(p, tokens, n_tokens, encrypt,
+				&used_n_tokens);
+	}
+
+	if (xform == NULL)
+		return 0;
+
+	p->xform = xform;
+
+	if (strcmp(tokens[used_n_tokens], "data_offset")) {
+		parse_free_sym_crypto_param_data(p);
+		return 0;
+	}
+
+	a->action_mask |= 1 << RTE_TABLE_ACTION_SYM_CRYPTO;
+
+	return used_n_tokens + 5;
+}
+
+static uint32_t
+parse_table_action_tag(char **tokens,
+	uint32_t n_tokens,
+	struct table_rule_action *a)
+{
+	if ((n_tokens < 2) ||
+		strcmp(tokens[0], "tag"))
+		return 0;
+
+	if (parser_read_uint32(&a->tag.tag, tokens[1]))
+		return 0;
+
+	a->action_mask |= 1 << RTE_TABLE_ACTION_TAG;
+	return 2;
+}
+
+static uint32_t
+parse_table_action_decap(char **tokens,
+	uint32_t n_tokens,
+	struct table_rule_action *a)
+{
+	if ((n_tokens < 2) ||
+		strcmp(tokens[0], "decap"))
+		return 0;
+
+	if (parser_read_uint16(&a->decap.n, tokens[1]))
+		return 0;
+
+	a->action_mask |= 1 << RTE_TABLE_ACTION_DECAP;
+	return 2;
+}
+
 static uint32_t
 parse_table_action(char **tokens,
 	uint32_t n_tokens,
@@ -3492,6 +4257,47 @@ parse_table_action(char **tokens,
 		n_tokens -= n;
 	}
 
+	if (n_tokens && (strcmp(tokens[0], "sym_crypto") == 0)) {
+		uint32_t n;
+
+		n = parse_table_action_sym_crypto(tokens, n_tokens, a);
+		if (n == 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"action sym_crypto");
+		}
+
+		tokens += n;
+		n_tokens -= n;
+	}
+
+	if (n_tokens && (strcmp(tokens[0], "tag") == 0)) {
+		uint32_t n;
+
+		n = parse_table_action_tag(tokens, n_tokens, a);
+		if (n == 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"action tag");
+			return 0;
+		}
+
+		tokens += n;
+		n_tokens -= n;
+	}
+
+	if (n_tokens && (strcmp(tokens[0], "decap") == 0)) {
+		uint32_t n;
+
+		n = parse_table_action_decap(tokens, n_tokens, a);
+		if (n == 0) {
+			snprintf(out, out_size, MSG_ARG_INVALID,
+				"action decap");
+			return 0;
+		}
+
+		tokens += n;
+		n_tokens -= n;
+	}
+
 	if (n_tokens0 - n_tokens == 1) {
 		snprintf(out, out_size, MSG_ARG_INVALID, "action");
 		return 0;
@@ -3515,7 +4321,6 @@ cmd_pipeline_table_rule_add(char **tokens,
 	struct table_rule_match m;
 	struct table_rule_action a;
 	char *pipeline_name;
-	void *data;
 	uint32_t table_id, t0, n_tokens_parsed;
 	int status;
 
@@ -3573,12 +4378,14 @@ cmd_pipeline_table_rule_add(char **tokens,
 		return;
 	}
 
-	status = pipeline_table_rule_add(pipeline_name, table_id,
-		&m, &a, &data);
+	status = pipeline_table_rule_add(pipeline_name, table_id, &m, &a);
 	if (status) {
 		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
 		return;
 	}
+
+	if (a.action_mask & 1 << RTE_TABLE_ACTION_SYM_CRYPTO)
+		parse_free_sym_crypto_param_data(&a.sym_crypto);
 }
 
 
@@ -3600,7 +4407,6 @@ cmd_pipeline_table_rule_add_default(char **tokens,
 	size_t out_size)
 {
 	struct table_rule_action action;
-	void *data;
 	char *pipeline_name;
 	uint32_t table_id;
 	int status;
@@ -3706,8 +4512,7 @@ cmd_pipeline_table_rule_add_default(char **tokens,
 
 	status = pipeline_table_rule_add_default(pipeline_name,
 		table_id,
-		&action,
-		&data);
+		&action);
 	if (status) {
 		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
 		return;
@@ -3716,7 +4521,7 @@ cmd_pipeline_table_rule_add_default(char **tokens,
 
 
 static const char cmd_pipeline_table_rule_add_bulk_help[] =
-"pipeline <pipeline_name> table <table_id> rule add bulk <file_name> <n_rules>\n"
+"pipeline <pipeline_name> table <table_id> rule add bulk <file_name>\n"
 "\n"
 "  File <file_name>:\n"
 "  - line format: match <match> action <action>\n";
@@ -3724,8 +4529,7 @@ static const char cmd_pipeline_table_rule_add_bulk_help[] =
 static int
 cli_rule_file_process(const char *file_name,
 	size_t line_len_max,
-	struct table_rule_match *m,
-	struct table_rule_action *a,
+	struct table_rule_list **rule_list,
 	uint32_t *n_rules,
 	uint32_t *line_number,
 	char *out,
@@ -3737,14 +4541,12 @@ cmd_pipeline_table_rule_add_bulk(char **tokens,
 	char *out,
 	size_t out_size)
 {
-	struct table_rule_match *match;
-	struct table_rule_action *action;
-	void **data;
+	struct table_rule_list *list = NULL;
 	char *pipeline_name, *file_name;
-	uint32_t table_id, n_rules, n_rules_parsed, line_number;
+	uint32_t table_id, n_rules, n_rules_added, n_rules_not_added, line_number;
 	int status;
 
-	if (n_tokens != 9) {
+	if (n_tokens != 8) {
 		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
 		return;
 	}
@@ -3778,68 +4580,33 @@ cmd_pipeline_table_rule_add_bulk(char **tokens,
 
 	file_name = tokens[7];
 
-	if ((parser_read_uint32(&n_rules, tokens[8]) != 0) ||
-		(n_rules == 0)) {
-		snprintf(out, out_size, MSG_ARG_INVALID, "n_rules");
-		return;
-	}
-
-	/* Memory allocation. */
-	match = calloc(n_rules, sizeof(struct table_rule_match));
-	action = calloc(n_rules, sizeof(struct table_rule_action));
-	data = calloc(n_rules, sizeof(void *));
-	if ((match == NULL) || (action == NULL) || (data == NULL)) {
-		snprintf(out, out_size, MSG_OUT_OF_MEMORY);
-		free(data);
-		free(action);
-		free(match);
-		return;
-	}
-
-	/* Load rule file */
-	n_rules_parsed = n_rules;
+	/* Load rules from file. */
 	status = cli_rule_file_process(file_name,
 		1024,
-		match,
-		action,
-		&n_rules_parsed,
+		&list,
+		&n_rules,
 		&line_number,
 		out,
 		out_size);
 	if (status) {
 		snprintf(out, out_size, MSG_FILE_ERR, file_name, line_number);
-		free(data);
-		free(action);
-		free(match);
-		return;
-	}
-	if (n_rules_parsed != n_rules) {
-		snprintf(out, out_size, MSG_FILE_NOT_ENOUGH, file_name);
-		free(data);
-		free(action);
-		free(match);
 		return;
 	}
 
 	/* Rule bulk add */
 	status = pipeline_table_rule_add_bulk(pipeline_name,
 		table_id,
-		match,
-		action,
-		data,
-		&n_rules);
+		list,
+		&n_rules_added,
+		&n_rules_not_added);
 	if (status) {
 		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
-		free(data);
-		free(action);
-		free(match);
 		return;
 	}
 
-	/* Memory free */
-	free(data);
-	free(action);
-	free(match);
+	snprintf(out, out_size, "Added %u rules out of %u.\n",
+		n_rules_added,
+		n_rules);
 }
 
 
@@ -3972,19 +4739,530 @@ cmd_pipeline_table_rule_delete_default(char **tokens,
 	}
 }
 
+static void
+ether_addr_show(FILE *f, struct ether_addr *addr)
+{
+	fprintf(f, "%02x:%02x:%02x:%02x:%02x:%02x",
+		(uint32_t)addr->addr_bytes[0], (uint32_t)addr->addr_bytes[1],
+		(uint32_t)addr->addr_bytes[2], (uint32_t)addr->addr_bytes[3],
+		(uint32_t)addr->addr_bytes[4], (uint32_t)addr->addr_bytes[5]);
+}
+
+static void
+ipv4_addr_show(FILE *f, uint32_t addr)
+{
+	fprintf(f, "%u.%u.%u.%u",
+		addr >> 24,
+		(addr >> 16) & 0xFF,
+		(addr >> 8) & 0xFF,
+		addr & 0xFF);
+}
+
+static void
+ipv6_addr_show(FILE *f, uint8_t *addr)
+{
+	fprintf(f, "%02x%02x:%02x%02x:%02x%02x:%02x%02x:"
+		"%02x%02x:%02x%02x:%02x%02x:%02x%02x:",
+		(uint32_t)addr[0], (uint32_t)addr[1],
+		(uint32_t)addr[2], (uint32_t)addr[3],
+		(uint32_t)addr[4], (uint32_t)addr[5],
+		(uint32_t)addr[6], (uint32_t)addr[7],
+		(uint32_t)addr[8], (uint32_t)addr[9],
+		(uint32_t)addr[10], (uint32_t)addr[11],
+		(uint32_t)addr[12], (uint32_t)addr[13],
+		(uint32_t)addr[14], (uint32_t)addr[15]);
+}
+
+static const char *
+policer_action_string(enum rte_table_action_policer action) {
+	switch (action) {
+		case RTE_TABLE_ACTION_POLICER_COLOR_GREEN: return "G";
+		case RTE_TABLE_ACTION_POLICER_COLOR_YELLOW: return "Y";
+		case RTE_TABLE_ACTION_POLICER_COLOR_RED: return "R";
+		case RTE_TABLE_ACTION_POLICER_DROP: return "D";
+		default: return "?";
+	}
+}
+
+static int
+table_rule_show(const char *pipeline_name,
+	uint32_t table_id,
+	const char *file_name)
+{
+	struct pipeline *p;
+	struct table *table;
+	struct table_rule *rule;
+	FILE *f = NULL;
+	uint32_t i;
+
+	/* Check input params. */
+	if ((pipeline_name == NULL) ||
+		(file_name == NULL))
+		return -1;
+
+	p = pipeline_find(pipeline_name);
+	if ((p == NULL) ||
+		(table_id >= p->n_tables))
+		return -1;
+
+	table = &p->table[table_id];
+
+	/* Open file. */
+	f = fopen(file_name, "w");
+	if (f == NULL)
+		return -1;
+
+	/* Write table rules to file. */
+	TAILQ_FOREACH(rule, &table->rules, node) {
+		struct table_rule_match *m = &rule->match;
+		struct table_rule_action *a = &rule->action;
+
+		fprintf(f, "match ");
+		switch (m->match_type) {
+		case TABLE_ACL:
+			fprintf(f, "acl priority %u ",
+				m->match.acl.priority);
+
+			fprintf(f, m->match.acl.ip_version ? "ipv4 " : "ipv6 ");
+
+			if (m->match.acl.ip_version)
+				ipv4_addr_show(f, m->match.acl.ipv4.sa);
+			else
+				ipv6_addr_show(f, m->match.acl.ipv6.sa);
+
+			fprintf(f, "%u",	m->match.acl.sa_depth);
+
+			if (m->match.acl.ip_version)
+				ipv4_addr_show(f, m->match.acl.ipv4.da);
+			else
+				ipv6_addr_show(f, m->match.acl.ipv6.da);
+
+			fprintf(f, "%u",	m->match.acl.da_depth);
+
+			fprintf(f, "%u %u %u %u %u ",
+				(uint32_t)m->match.acl.sp0,
+				(uint32_t)m->match.acl.sp1,
+				(uint32_t)m->match.acl.dp0,
+				(uint32_t)m->match.acl.dp1,
+				(uint32_t)m->match.acl.proto);
+			break;
+
+		case TABLE_ARRAY:
+			fprintf(f, "array %u ",
+				m->match.array.pos);
+			break;
+
+		case TABLE_HASH:
+			fprintf(f, "hash raw ");
+			for (i = 0; i < table->params.match.hash.key_size; i++)
+				fprintf(f, "%02x", m->match.hash.key[i]);
+			fprintf(f, " ");
+			break;
+
+		case TABLE_LPM:
+			fprintf(f, "lpm ");
+
+			fprintf(f, m->match.lpm.ip_version ? "ipv4 " : "ipv6 ");
+
+			if (m->match.acl.ip_version)
+				ipv4_addr_show(f, m->match.lpm.ipv4);
+			else
+				ipv6_addr_show(f, m->match.lpm.ipv6);
+
+			fprintf(f, "%u ",
+				(uint32_t)m->match.lpm.depth);
+			break;
+
+		default:
+			fprintf(f, "unknown ");
+		}
+
+		fprintf(f, "action ");
+		if (a->action_mask & (1LLU << RTE_TABLE_ACTION_FWD)) {
+			fprintf(f, "fwd ");
+			switch (a->fwd.action) {
+			case RTE_PIPELINE_ACTION_DROP:
+				fprintf(f, "drop ");
+				break;
+
+			case RTE_PIPELINE_ACTION_PORT:
+				fprintf(f, "port %u ", a->fwd.id);
+				break;
+
+			case RTE_PIPELINE_ACTION_PORT_META:
+				fprintf(f, "meta ");
+				break;
+
+			case RTE_PIPELINE_ACTION_TABLE:
+			default:
+				fprintf(f, "table %u ", a->fwd.id);
+			}
+		}
+
+		if (a->action_mask & (1LLU << RTE_TABLE_ACTION_LB)) {
+			fprintf(f, "balance ");
+			for (i = 0; i < RTE_DIM(a->lb.out); i++)
+				fprintf(f, "%u ", a->lb.out[i]);
+		}
+
+		if (a->action_mask & (1LLU << RTE_TABLE_ACTION_MTR)) {
+			fprintf(f, "mtr ");
+			for (i = 0; i < RTE_TABLE_ACTION_TC_MAX; i++)
+				if (a->mtr.tc_mask & (1 << i)) {
+					struct rte_table_action_mtr_tc_params *p =
+						&a->mtr.mtr[i];
+					enum rte_table_action_policer ga =
+						p->policer[e_RTE_METER_GREEN];
+					enum rte_table_action_policer ya =
+						p->policer[e_RTE_METER_YELLOW];
+					enum rte_table_action_policer ra =
+						p->policer[e_RTE_METER_RED];
+
+					fprintf(f, "tc%u meter %u policer g %s y %s r %s ",
+						i,
+						a->mtr.mtr[i].meter_profile_id,
+						policer_action_string(ga),
+						policer_action_string(ya),
+						policer_action_string(ra));
+				}
+		}
+
+		if (a->action_mask & (1LLU << RTE_TABLE_ACTION_TM))
+			fprintf(f, "tm subport %u pipe %u ",
+				a->tm.subport_id,
+				a->tm.pipe_id);
+
+		if (a->action_mask & (1LLU << RTE_TABLE_ACTION_ENCAP)) {
+			fprintf(f, "encap ");
+			switch (a->encap.type) {
+			case RTE_TABLE_ACTION_ENCAP_ETHER:
+				fprintf(f, "ether ");
+				ether_addr_show(f, &a->encap.ether.ether.da);
+				fprintf(f, " ");
+				ether_addr_show(f, &a->encap.ether.ether.sa);
+				fprintf(f, " ");
+				break;
+
+			case RTE_TABLE_ACTION_ENCAP_VLAN:
+				fprintf(f, "vlan ");
+				ether_addr_show(f, &a->encap.vlan.ether.da);
+				fprintf(f, " ");
+				ether_addr_show(f, &a->encap.vlan.ether.sa);
+				fprintf(f, " pcp %u dei %u vid %u ",
+					a->encap.vlan.vlan.pcp,
+					a->encap.vlan.vlan.dei,
+					a->encap.vlan.vlan.vid);
+				break;
+
+			case RTE_TABLE_ACTION_ENCAP_QINQ:
+				fprintf(f, "qinq ");
+				ether_addr_show(f, &a->encap.qinq.ether.da);
+				fprintf(f, " ");
+				ether_addr_show(f, &a->encap.qinq.ether.sa);
+				fprintf(f, " pcp %u dei %u vid %u pcp %u dei %u vid %u ",
+					a->encap.qinq.svlan.pcp,
+					a->encap.qinq.svlan.dei,
+					a->encap.qinq.svlan.vid,
+					a->encap.qinq.cvlan.pcp,
+					a->encap.qinq.cvlan.dei,
+					a->encap.qinq.cvlan.vid);
+				break;
+
+			case RTE_TABLE_ACTION_ENCAP_MPLS:
+				fprintf(f, "mpls %s ", (a->encap.mpls.unicast) ?
+					"unicast " : "multicast ");
+				ether_addr_show(f, &a->encap.mpls.ether.da);
+				fprintf(f, " ");
+				ether_addr_show(f, &a->encap.mpls.ether.sa);
+				fprintf(f, " ");
+				for (i = 0; i < a->encap.mpls.mpls_count; i++) {
+					struct rte_table_action_mpls_hdr *l =
+						&a->encap.mpls.mpls[i];
+
+					fprintf(f, "label%u %u %u %u ",
+						i,
+						l->label,
+						l->tc,
+						l->ttl);
+				}
+				break;
+
+			case RTE_TABLE_ACTION_ENCAP_PPPOE:
+				fprintf(f, "pppoe ");
+				ether_addr_show(f, &a->encap.pppoe.ether.da);
+				fprintf(f, " ");
+				ether_addr_show(f, &a->encap.pppoe.ether.sa);
+				fprintf(f, " %u ", a->encap.pppoe.pppoe.session_id);
+				break;
+
+			case RTE_TABLE_ACTION_ENCAP_VXLAN:
+				fprintf(f, "vxlan ether ");
+				ether_addr_show(f, &a->encap.vxlan.ether.da);
+				fprintf(f, " ");
+				ether_addr_show(f, &a->encap.vxlan.ether.sa);
+				if (table->ap->params.encap.vxlan.vlan)
+					fprintf(f, " vlan pcp %u dei %u vid %u ",
+						a->encap.vxlan.vlan.pcp,
+						a->encap.vxlan.vlan.dei,
+						a->encap.vxlan.vlan.vid);
+				if (table->ap->params.encap.vxlan.ip_version) {
+					fprintf(f, " ipv4 ");
+					ipv4_addr_show(f, a->encap.vxlan.ipv4.sa);
+					fprintf(f, " ");
+					ipv4_addr_show(f, a->encap.vxlan.ipv4.da);
+					fprintf(f, " %u %u ",
+						(uint32_t)a->encap.vxlan.ipv4.dscp,
+						(uint32_t)a->encap.vxlan.ipv4.ttl);
+				} else {
+					fprintf(f, " ipv6 ");
+					ipv6_addr_show(f, a->encap.vxlan.ipv6.sa);
+					fprintf(f, " ");
+					ipv6_addr_show(f, a->encap.vxlan.ipv6.da);
+					fprintf(f, " %u %u %u ",
+						a->encap.vxlan.ipv6.flow_label,
+						(uint32_t)a->encap.vxlan.ipv6.dscp,
+						(uint32_t)a->encap.vxlan.ipv6.hop_limit);
+					fprintf(f, " udp %u %u vxlan %u ",
+						a->encap.vxlan.udp.sp,
+						a->encap.vxlan.udp.dp,
+						a->encap.vxlan.vxlan.vni);
+				}
+				break;
+
+			default:
+				fprintf(f, "unknown ");
+			}
+		}
+
+		if (a->action_mask & (1LLU << RTE_TABLE_ACTION_NAT)) {
+			fprintf(f, "nat %s ", (a->nat.ip_version) ? "ipv4 " : "ipv6 ");
+			if (a->nat.ip_version)
+				ipv4_addr_show(f, a->nat.addr.ipv4);
+			else
+				ipv6_addr_show(f, a->nat.addr.ipv6);
+			fprintf(f, " %u ", (uint32_t)(a->nat.port));
+		}
+
+		if (a->action_mask & (1LLU << RTE_TABLE_ACTION_TTL))
+			fprintf(f, "ttl %s ", (a->ttl.decrement) ? "dec" : "keep");
+
+		if (a->action_mask & (1LLU << RTE_TABLE_ACTION_STATS))
+			fprintf(f, "stats ");
+
+		if (a->action_mask & (1LLU << RTE_TABLE_ACTION_TIME))
+			fprintf(f, "time ");
+
+		if (a->action_mask & (1LLU << RTE_TABLE_ACTION_SYM_CRYPTO))
+			fprintf(f, "sym_crypto ");
+
+		if (a->action_mask & (1LLU << RTE_TABLE_ACTION_TAG))
+			fprintf(f, "tag %u ", a->tag.tag);
+
+		if (a->action_mask & (1LLU << RTE_TABLE_ACTION_DECAP))
+			fprintf(f, "decap %u ", a->decap.n);
+
+		/* end */
+		fprintf(f, "\n");
+	}
+
+	/* Write table default rule to file. */
+	if (table->rule_default) {
+		struct table_rule_action *a = &table->rule_default->action;
+
+		fprintf(f, "# match default action fwd ");
+
+		switch (a->fwd.action) {
+		case RTE_PIPELINE_ACTION_DROP:
+			fprintf(f, "drop ");
+			break;
+
+		case RTE_PIPELINE_ACTION_PORT:
+			fprintf(f, "port %u ", a->fwd.id);
+			break;
+
+		case RTE_PIPELINE_ACTION_PORT_META:
+			fprintf(f, "meta ");
+			break;
+
+		case RTE_PIPELINE_ACTION_TABLE:
+		default:
+			fprintf(f, "table %u ", a->fwd.id);
+		}
+	} else
+		fprintf(f, "# match default action fwd drop ");
+
+	fprintf(f, "\n");
+
+	/* Close file. */
+	fclose(f);
+
+	return 0;
+}
+
+static const char cmd_pipeline_table_rule_show_help[] =
+"pipeline <pipeline_name> table <table_id> rule show\n"
+"     file <file_name>\n";
+
+static void
+cmd_pipeline_table_rule_show(char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	char *file_name = NULL, *pipeline_name;
+	uint32_t table_id;
+	int status;
+
+	if (n_tokens != 8) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	pipeline_name = tokens[1];
+
+	if (strcmp(tokens[2], "table") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "table");
+		return;
+	}
+
+	if (parser_read_uint32(&table_id, tokens[3]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "table_id");
+		return;
+	}
+
+	if (strcmp(tokens[4], "rule") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "rule");
+		return;
+	}
+
+	if (strcmp(tokens[5], "show") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "show");
+		return;
+	}
+
+	if (strcmp(tokens[6], "file") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "file");
+		return;
+	}
+
+	file_name = tokens[7];
+
+	status = table_rule_show(pipeline_name, table_id, file_name);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+}
 
 static const char cmd_pipeline_table_rule_stats_read_help[] =
-"pipeline <pipeline_name> table <table_id> rule read stats [clear]\n";
+"pipeline <pipeline_name> table <table_id> rule read stats [clear]\n"
+"     match <match>\n";
 
 static void
 cmd_pipeline_table_rule_stats_read(char **tokens,
-	uint32_t n_tokens __rte_unused,
+	uint32_t n_tokens,
 	char *out,
 	size_t out_size)
 {
-	snprintf(out, out_size, MSG_CMD_UNIMPLEM, tokens[0]);
-}
+	struct table_rule_match m;
+	struct rte_table_action_stats_counters stats;
+	char *pipeline_name;
+	uint32_t table_id, n_tokens_parsed;
+	int clear = 0, status;
+
+	if (n_tokens < 7) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	pipeline_name = tokens[1];
+
+	if (strcmp(tokens[2], "table") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "table");
+		return;
+	}
+
+	if (parser_read_uint32(&table_id, tokens[3]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "table_id");
+		return;
+	}
+
+	if (strcmp(tokens[4], "rule") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "rule");
+		return;
+	}
+
+	if (strcmp(tokens[5], "read") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "read");
+		return;
+	}
+
+	if (strcmp(tokens[6], "stats") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "stats");
+		return;
+	}
+
+	n_tokens -= 7;
+	tokens += 7;
+
+	/* clear */
+	if (n_tokens && (strcmp(tokens[0], "clear") == 0)) {
+		clear = 1;
 
+		n_tokens--;
+		tokens++;
+	}
+
+	/* match */
+	if ((n_tokens == 0) || strcmp(tokens[0], "match")) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "match");
+		return;
+	}
+
+	n_tokens_parsed = parse_match(tokens,
+		n_tokens,
+		out,
+		out_size,
+		&m);
+	if (n_tokens_parsed == 0)
+		return;
+	n_tokens -= n_tokens_parsed;
+	tokens += n_tokens_parsed;
+
+	/* end */
+	if (n_tokens) {
+		snprintf(out, out_size, MSG_ARG_INVALID, tokens[0]);
+		return;
+	}
+
+	/* Read table rule stats. */
+	status = pipeline_table_rule_stats_read(pipeline_name,
+		table_id,
+		&m,
+		&stats,
+		clear);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+
+	/* Print stats. */
+	if (stats.n_packets_valid && stats.n_bytes_valid)
+		snprintf(out, out_size, "Packets: %" PRIu64 "; Bytes: %" PRIu64 "\n",
+			stats.n_packets,
+			stats.n_bytes);
+
+	if (stats.n_packets_valid && !stats.n_bytes_valid)
+		snprintf(out, out_size, "Packets: %" PRIu64 "; Bytes: N/A\n",
+			stats.n_packets);
+
+	if (!stats.n_packets_valid && stats.n_bytes_valid)
+		snprintf(out, out_size, "Packets: N/A; Bytes: %" PRIu64 "\n",
+			stats.n_bytes);
+
+	if (!stats.n_packets_valid && !stats.n_bytes_valid)
+		snprintf(out, out_size, "Packets: N/A ; Bytes: N/A\n");
+}
 
 static const char cmd_pipeline_table_meter_profile_add_help[] =
 "pipeline <pipeline_name> table <table_id> meter profile <meter_profile_id>\n"
@@ -4201,15 +5479,98 @@ cmd_pipeline_table_meter_profile_delete(char **tokens,
 
 
 static const char cmd_pipeline_table_rule_meter_read_help[] =
-"pipeline <pipeline_name> table <table_id> rule read meter [clear]\n";
+"pipeline <pipeline_name> table <table_id> rule read meter [clear]\n"
+"     match <match>\n";
 
 static void
 cmd_pipeline_table_rule_meter_read(char **tokens,
-	uint32_t n_tokens __rte_unused,
+	uint32_t n_tokens,
 	char *out,
 	size_t out_size)
 {
-	snprintf(out, out_size, MSG_CMD_UNIMPLEM, tokens[0]);
+	struct table_rule_match m;
+	struct rte_table_action_mtr_counters stats;
+	char *pipeline_name;
+	uint32_t table_id, n_tokens_parsed;
+	int clear = 0, status;
+
+	if (n_tokens < 7) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	pipeline_name = tokens[1];
+
+	if (strcmp(tokens[2], "table") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "table");
+		return;
+	}
+
+	if (parser_read_uint32(&table_id, tokens[3]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "table_id");
+		return;
+	}
+
+	if (strcmp(tokens[4], "rule") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "rule");
+		return;
+	}
+
+	if (strcmp(tokens[5], "read") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "read");
+		return;
+	}
+
+	if (strcmp(tokens[6], "meter") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "meter");
+		return;
+	}
+
+	n_tokens -= 7;
+	tokens += 7;
+
+	/* clear */
+	if (n_tokens && (strcmp(tokens[0], "clear") == 0)) {
+		clear = 1;
+
+		n_tokens--;
+		tokens++;
+	}
+
+	/* match */
+	if ((n_tokens == 0) || strcmp(tokens[0], "match")) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "match");
+		return;
+	}
+
+	n_tokens_parsed = parse_match(tokens,
+		n_tokens,
+		out,
+		out_size,
+		&m);
+	if (n_tokens_parsed == 0)
+		return;
+	n_tokens -= n_tokens_parsed;
+	tokens += n_tokens_parsed;
+
+	/* end */
+	if (n_tokens) {
+		snprintf(out, out_size, MSG_ARG_INVALID, tokens[0]);
+		return;
+	}
+
+	/* Read table rule meter stats. */
+	status = pipeline_table_rule_mtr_read(pipeline_name,
+		table_id,
+		&m,
+		&stats,
+		clear);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+
+	/* Print stats. */
 }
 
 
@@ -4364,17 +5725,188 @@ cmd_pipeline_table_dscp(char **tokens,
 
 
 static const char cmd_pipeline_table_rule_ttl_read_help[] =
-"pipeline <pipeline_name> table <table_id> rule read ttl [clear]\n";
+"pipeline <pipeline_name> table <table_id> rule read ttl [clear]\n"
+"     match <match>\n";
 
 static void
 cmd_pipeline_table_rule_ttl_read(char **tokens,
-	uint32_t n_tokens __rte_unused,
+	uint32_t n_tokens,
 	char *out,
 	size_t out_size)
 {
-	snprintf(out, out_size, MSG_CMD_UNIMPLEM, tokens[0]);
+	struct table_rule_match m;
+	struct rte_table_action_ttl_counters stats;
+	char *pipeline_name;
+	uint32_t table_id, n_tokens_parsed;
+	int clear = 0, status;
+
+	if (n_tokens < 7) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	pipeline_name = tokens[1];
+
+	if (strcmp(tokens[2], "table") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "table");
+		return;
+	}
+
+	if (parser_read_uint32(&table_id, tokens[3]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "table_id");
+		return;
+	}
+
+	if (strcmp(tokens[4], "rule") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "rule");
+		return;
+	}
+
+	if (strcmp(tokens[5], "read") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "read");
+		return;
+	}
+
+	if (strcmp(tokens[6], "ttl") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "ttl");
+		return;
+	}
+
+	n_tokens -= 7;
+	tokens += 7;
+
+	/* clear */
+	if (n_tokens && (strcmp(tokens[0], "clear") == 0)) {
+		clear = 1;
+
+		n_tokens--;
+		tokens++;
+	}
+
+	/* match */
+	if ((n_tokens == 0) || strcmp(tokens[0], "match")) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "match");
+		return;
+	}
+
+	n_tokens_parsed = parse_match(tokens,
+		n_tokens,
+		out,
+		out_size,
+		&m);
+	if (n_tokens_parsed == 0)
+		return;
+	n_tokens -= n_tokens_parsed;
+	tokens += n_tokens_parsed;
+
+	/* end */
+	if (n_tokens) {
+		snprintf(out, out_size, MSG_ARG_INVALID, tokens[0]);
+		return;
+	}
+
+	/* Read table rule TTL stats. */
+	status = pipeline_table_rule_ttl_read(pipeline_name,
+		table_id,
+		&m,
+		&stats,
+		clear);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+
+	/* Print stats. */
+	snprintf(out, out_size, "Packets: %" PRIu64 "\n",
+		stats.n_packets);
 }
 
+static const char cmd_pipeline_table_rule_time_read_help[] =
+"pipeline <pipeline_name> table <table_id> rule read time\n"
+"     match <match>\n";
+
+static void
+cmd_pipeline_table_rule_time_read(char **tokens,
+	uint32_t n_tokens,
+	char *out,
+	size_t out_size)
+{
+	struct table_rule_match m;
+	char *pipeline_name;
+	uint64_t timestamp;
+	uint32_t table_id, n_tokens_parsed;
+	int status;
+
+	if (n_tokens < 7) {
+		snprintf(out, out_size, MSG_ARG_MISMATCH, tokens[0]);
+		return;
+	}
+
+	pipeline_name = tokens[1];
+
+	if (strcmp(tokens[2], "table") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "table");
+		return;
+	}
+
+	if (parser_read_uint32(&table_id, tokens[3]) != 0) {
+		snprintf(out, out_size, MSG_ARG_INVALID, "table_id");
+		return;
+	}
+
+	if (strcmp(tokens[4], "rule") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "rule");
+		return;
+	}
+
+	if (strcmp(tokens[5], "read") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "read");
+		return;
+	}
+
+	if (strcmp(tokens[6], "time") != 0) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "time");
+		return;
+	}
+
+	n_tokens -= 7;
+	tokens += 7;
+
+	/* match */
+	if ((n_tokens == 0) || strcmp(tokens[0], "match")) {
+		snprintf(out, out_size, MSG_ARG_NOT_FOUND, "match");
+		return;
+	}
+
+	n_tokens_parsed = parse_match(tokens,
+		n_tokens,
+		out,
+		out_size,
+		&m);
+	if (n_tokens_parsed == 0)
+		return;
+	n_tokens -= n_tokens_parsed;
+	tokens += n_tokens_parsed;
+
+	/* end */
+	if (n_tokens) {
+		snprintf(out, out_size, MSG_ARG_INVALID, tokens[0]);
+		return;
+	}
+
+	/* Read table rule timestamp. */
+	status = pipeline_table_rule_time_read(pipeline_name,
+		table_id,
+		&m,
+		&timestamp);
+	if (status) {
+		snprintf(out, out_size, MSG_CMD_FAIL, tokens[0]);
+		return;
+	}
+
+	/* Print stats. */
+	snprintf(out, out_size, "Packets: %" PRIu64 "\n", timestamp);
+}
 
 static const char cmd_thread_pipeline_enable_help[] =
 "thread <thread_id> pipeline <pipeline_name> enable\n";
@@ -4499,12 +6031,14 @@ cmd_help(char **tokens, uint32_t n_tokens, char *out, size_t out_size)
 			"\tpipeline table rule add bulk\n"
 			"\tpipeline table rule delete\n"
 			"\tpipeline table rule delete default\n"
+			"\tpipeline table rule show\n"
 			"\tpipeline table rule stats read\n"
 			"\tpipeline table meter profile add\n"
 			"\tpipeline table meter profile delete\n"
 			"\tpipeline table rule meter read\n"
 			"\tpipeline table dscp\n"
 			"\tpipeline table rule ttl read\n"
+			"\tpipeline table rule time read\n"
 			"\tthread pipeline enable\n"
 			"\tthread pipeline disable\n\n");
 		return;
@@ -4570,6 +6104,11 @@ cmd_help(char **tokens, uint32_t n_tokens, char *out, size_t out_size)
 		return;
 	}
 
+	if (strcmp(tokens[0], "cryptodev") == 0) {
+		snprintf(out, out_size, "\n%s\n", cmd_cryptodev_help);
+		return;
+	}
+
 	if ((n_tokens == 4) &&
 		(strcmp(tokens[0], "port") == 0) &&
 		(strcmp(tokens[1], "in") == 0) &&
@@ -4707,6 +6246,14 @@ cmd_help(char **tokens, uint32_t n_tokens, char *out, size_t out_size)
 			return;
 		}
 
+		if ((n_tokens == 4) &&
+			(strcmp(tokens[2], "rule") == 0) &&
+			(strcmp(tokens[3], "show") == 0)) {
+			snprintf(out, out_size, "\n%s\n",
+				cmd_pipeline_table_rule_show_help);
+			return;
+		}
+
 		if ((n_tokens == 5) &&
 			(strcmp(tokens[2], "rule") == 0) &&
 			(strcmp(tokens[3], "stats") == 0) &&
@@ -4751,6 +6298,15 @@ cmd_help(char **tokens, uint32_t n_tokens, char *out, size_t out_size)
 				cmd_pipeline_table_rule_ttl_read_help);
 			return;
 		}
+
+		if ((n_tokens == 5) &&
+			(strcmp(tokens[2], "rule") == 0) &&
+			(strcmp(tokens[3], "time") == 0) &&
+			(strcmp(tokens[4], "read") == 0)) {
+			snprintf(out, out_size, "\n%s\n",
+				cmd_pipeline_table_rule_time_read_help);
+			return;
+		}
 	}
 
 	if ((n_tokens == 3) &&
@@ -4860,6 +6416,11 @@ cli_process(char *in, char *out, size_t out_size)
 		return;
 	}
 
+	if (strcmp(tokens[0], "cryptodev") == 0) {
+		cmd_cryptodev(tokens, n_tokens, out, out_size);
+		return;
+	}
+
 	if (strcmp(tokens[0], "port") == 0) {
 		cmd_port_in_action_profile(tokens, n_tokens, out, out_size);
 		return;
@@ -4997,6 +6558,15 @@ cli_process(char *in, char *out, size_t out_size)
 			return;
 		}
 
+		if ((n_tokens >= 6) &&
+			(strcmp(tokens[2], "table") == 0) &&
+			(strcmp(tokens[4], "rule") == 0) &&
+			(strcmp(tokens[5], "show") == 0)) {
+			cmd_pipeline_table_rule_show(tokens, n_tokens,
+				out, out_size);
+			return;
+		}
+
 		if ((n_tokens >= 7) &&
 			(strcmp(tokens[2], "table") == 0) &&
 			(strcmp(tokens[4], "rule") == 0) &&
@@ -5054,6 +6624,16 @@ cli_process(char *in, char *out, size_t out_size)
 				out, out_size);
 			return;
 		}
+
+		if ((n_tokens >= 7) &&
+			(strcmp(tokens[2], "table") == 0) &&
+			(strcmp(tokens[4], "rule") == 0) &&
+			(strcmp(tokens[5], "read") == 0) &&
+			(strcmp(tokens[6], "time") == 0)) {
+			cmd_pipeline_table_rule_time_read(tokens, n_tokens,
+				out, out_size);
+			return;
+		}
 	}
 
 	if (strcmp(tokens[0], "thread") == 0) {
@@ -5133,44 +6713,56 @@ cli_script_process(const char *file_name,
 static int
 cli_rule_file_process(const char *file_name,
 	size_t line_len_max,
-	struct table_rule_match *m,
-	struct table_rule_action *a,
+	struct table_rule_list **rule_list,
 	uint32_t *n_rules,
 	uint32_t *line_number,
 	char *out,
 	size_t out_size)
 {
-	FILE *f = NULL;
+	struct table_rule_list *list = NULL;
 	char *line = NULL;
-	uint32_t rule_id, line_id;
+	FILE *f = NULL;
+	uint32_t rule_id = 0, line_id = 0;
 	int status = 0;
 
 	/* Check input arguments */
 	if ((file_name == NULL) ||
 		(strlen(file_name) == 0) ||
-		(line_len_max == 0)) {
-		*line_number = 0;
-		return -EINVAL;
+		(line_len_max == 0) ||
+		(rule_list == NULL) ||
+		(n_rules == NULL) ||
+		(line_number == NULL) ||
+		(out == NULL)) {
+		status = -EINVAL;
+		goto cli_rule_file_process_free;
 	}
 
 	/* Memory allocation */
+	list = malloc(sizeof(struct table_rule_list));
+	if (list == NULL) {
+		status = -ENOMEM;
+		goto cli_rule_file_process_free;
+	}
+
+	TAILQ_INIT(list);
+
 	line = malloc(line_len_max + 1);
 	if (line == NULL) {
-		*line_number = 0;
-		return -ENOMEM;
+		status = -ENOMEM;
+		goto cli_rule_file_process_free;
 	}
 
 	/* Open file */
 	f = fopen(file_name, "r");
 	if (f == NULL) {
-		*line_number = 0;
-		free(line);
-		return -EIO;
+		status = -EIO;
+		goto cli_rule_file_process_free;
 	}
 
 	/* Read file */
-	for (line_id = 1, rule_id = 0; rule_id < *n_rules; line_id++) {
+	for (line_id = 1, rule_id = 0; ; line_id++) {
 		char *tokens[CMD_MAX_TOKENS];
+		struct table_rule *rule = NULL;
 		uint32_t n_tokens, n_tokens_parsed, t0;
 
 		/* Read next line from file. */
@@ -5186,7 +6778,7 @@ cli_rule_file_process(const char *file_name,
 		status = parse_tokenize_string(line, tokens, &n_tokens);
 		if (status) {
 			status = -EINVAL;
-			break;
+			goto cli_rule_file_process_free;
 		}
 
 		/* Empty line. */
@@ -5194,15 +6786,24 @@ cli_rule_file_process(const char *file_name,
 			continue;
 		t0 = 0;
 
+		/* Rule alloc and insert. */
+		rule = calloc(1, sizeof(struct table_rule));
+		if (rule == NULL) {
+			status = -ENOMEM;
+			goto cli_rule_file_process_free;
+		}
+
+		TAILQ_INSERT_TAIL(list, rule, node);
+
 		/* Rule match. */
 		n_tokens_parsed = parse_match(tokens + t0,
 			n_tokens - t0,
 			out,
 			out_size,
-			&m[rule_id]);
+			&rule->match);
 		if (n_tokens_parsed == 0) {
 			status = -EINVAL;
-			break;
+			goto cli_rule_file_process_free;
 		}
 		t0 += n_tokens_parsed;
 
@@ -5211,17 +6812,17 @@ cli_rule_file_process(const char *file_name,
 			n_tokens - t0,
 			out,
 			out_size,
-			&a[rule_id]);
+			&rule->action);
 		if (n_tokens_parsed == 0) {
 			status = -EINVAL;
-			break;
+			goto cli_rule_file_process_free;
 		}
 		t0 += n_tokens_parsed;
 
 		/* Line completed. */
 		if (t0 < n_tokens) {
 			status = -EINVAL;
-			break;
+			goto cli_rule_file_process_free;
 		}
 
 		/* Increment rule count */
@@ -5234,7 +6835,31 @@ cli_rule_file_process(const char *file_name,
 	/* Memory free */
 	free(line);
 
+	*rule_list = list;
 	*n_rules = rule_id;
 	*line_number = line_id;
+	return 0;
+
+cli_rule_file_process_free:
+	*rule_list = NULL;
+	*n_rules = rule_id;
+	*line_number = line_id;
+
+	for ( ; ; ) {
+		struct table_rule *rule;
+
+		rule = TAILQ_FIRST(list);
+		if (rule == NULL)
+			break;
+
+		TAILQ_REMOVE(list, rule, node);
+		free(rule);
+	}
+
+	if (f)
+		fclose(f);
+	free(line);
+	free(list);
+
 	return status;
 }
diff --git a/examples/ip_pipeline/conn.c b/examples/ip_pipeline/conn.c
index 6b08e9e8..30fca80c 100644
--- a/examples/ip_pipeline/conn.c
+++ b/examples/ip_pipeline/conn.c
@@ -8,7 +8,6 @@
 #include <unistd.h>
 #include <sys/types.h>
 
-#define __USE_GNU
 #include <sys/socket.h>
 
 #include <sys/epoll.h>
diff --git a/examples/ip_pipeline/cryptodev.c b/examples/ip_pipeline/cryptodev.c
new file mode 100644
index 00000000..c4ba72be
--- /dev/null
+++ b/examples/ip_pipeline/cryptodev.c
@@ -0,0 +1,117 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+
+#include <rte_cryptodev.h>
+#include <rte_cryptodev_pmd.h>
+#include <rte_string_fns.h>
+
+#include "cryptodev.h"
+
+static struct cryptodev_list cryptodev_list;
+
+int
+cryptodev_init(void)
+{
+	TAILQ_INIT(&cryptodev_list);
+
+	return 0;
+}
+
+struct cryptodev *
+cryptodev_find(const char *name)
+{
+	struct cryptodev *cryptodev;
+
+	if (name == NULL)
+		return NULL;
+
+	TAILQ_FOREACH(cryptodev, &cryptodev_list, node)
+		if (strcmp(cryptodev->name, name) == 0)
+			return cryptodev;
+
+	return NULL;
+}
+
+struct cryptodev *
+cryptodev_next(struct cryptodev *cryptodev)
+{
+	return (cryptodev == NULL) ?
+			TAILQ_FIRST(&cryptodev_list) :
+			TAILQ_NEXT(cryptodev, node);
+}
+
+struct cryptodev *
+cryptodev_create(const char *name, struct cryptodev_params *params)
+{
+	struct rte_cryptodev_info dev_info;
+	struct rte_cryptodev_config dev_conf;
+	struct rte_cryptodev_qp_conf queue_conf;
+	struct cryptodev *cryptodev;
+	uint32_t dev_id, i;
+	uint32_t socket_id;
+	int status;
+
+	/* Check input params */
+	if ((name == NULL) ||
+		cryptodev_find(name) ||
+		(params->n_queues == 0) ||
+		(params->queue_size == 0))
+		return NULL;
+
+	if (params->dev_name) {
+		status = rte_cryptodev_get_dev_id(params->dev_name);
+		if (status == -1)
+			return NULL;
+
+		dev_id = (uint32_t)status;
+	} else {
+		if (rte_cryptodev_pmd_is_valid_dev(params->dev_id) == 0)
+			return NULL;
+
+		dev_id = params->dev_id;
+	}
+
+	socket_id = rte_cryptodev_socket_id(dev_id);
+	rte_cryptodev_info_get(dev_id, &dev_info);
+
+	if (dev_info.max_nb_queue_pairs < params->n_queues)
+		return NULL;
+	if (dev_info.feature_flags & RTE_CRYPTODEV_FF_HW_ACCELERATED)
+		return NULL;
+
+	dev_conf.socket_id = socket_id;
+	dev_conf.nb_queue_pairs = params->n_queues;
+
+	status = rte_cryptodev_configure(dev_id, &dev_conf);
+	if (status < 0)
+		return NULL;
+
+	queue_conf.nb_descriptors = params->queue_size;
+	for (i = 0; i < params->n_queues; i++) {
+		status = rte_cryptodev_queue_pair_setup(dev_id, i,
+				&queue_conf, socket_id, NULL);
+		if (status < 0)
+			return NULL;
+	}
+
+	if (rte_cryptodev_start(dev_id) < 0)
+		return NULL;
+
+	cryptodev = calloc(1, sizeof(struct cryptodev));
+	if (cryptodev == NULL) {
+		rte_cryptodev_stop(dev_id);
+		return NULL;
+	}
+
+	strlcpy(cryptodev->name, name, sizeof(cryptodev->name));
+	cryptodev->dev_id = dev_id;
+	cryptodev->n_queues = params->n_queues;
+
+	TAILQ_INSERT_TAIL(&cryptodev_list, cryptodev, node);
+
+	return cryptodev;
+}
diff --git a/examples/ip_pipeline/cryptodev.h b/examples/ip_pipeline/cryptodev.h
new file mode 100644
index 00000000..d06b3f2f
--- /dev/null
+++ b/examples/ip_pipeline/cryptodev.h
@@ -0,0 +1,43 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#ifndef _INCLUDE_SYM_C_H_
+#define _INCLUDE_SYM_C_H_
+
+#include <stdint.h>
+#include <sys/queue.h>
+
+#include <rte_cryptodev.h>
+
+#include "common.h"
+
+struct cryptodev {
+	TAILQ_ENTRY(cryptodev) node;
+	char name[NAME_SIZE];
+	uint16_t dev_id;
+	uint32_t n_queues;
+};
+
+TAILQ_HEAD(cryptodev_list, cryptodev);
+
+int
+cryptodev_init(void);
+
+struct cryptodev *
+cryptodev_find(const char *name);
+
+struct cryptodev *
+cryptodev_next(struct cryptodev *cryptodev);
+
+struct cryptodev_params {
+	const char *dev_name;
+	uint32_t dev_id; /**< Valid only when *dev_name* is NULL. */
+	uint32_t n_queues;
+	uint32_t queue_size;
+};
+
+struct cryptodev *
+cryptodev_create(const char *name, struct cryptodev_params *params);
+
+#endif
diff --git a/examples/ip_pipeline/examples/flow_crypto.cli b/examples/ip_pipeline/examples/flow_crypto.cli
new file mode 100644
index 00000000..9b639deb
--- /dev/null
+++ b/examples/ip_pipeline/examples/flow_crypto.cli
@@ -0,0 +1,58 @@
+; SPDX-License-Identifier: BSD-3-Clause
+; Copyright(c) 2018 Intel Corporation
+
+;                 ________________
+; LINK0 RXQ0 --->|                |---> CRYPTO0 TXQ0
+;                |      Flow      |
+; CRYPTO0 RXQ0-->| Classification |---> LINK0 TXQ0
+;                |________________|
+;                        |
+;                        +-----------> SINK0 (flow lookup miss)
+;
+; Input packet: Ethernet/IPv4
+;
+; Packet buffer layout:
+; #   Field Name       Offset (Bytes)   Size (Bytes)
+; 0   Mbuf             0                128
+; 1   Headroom         128              128
+; 2   Ethernet header  256              14
+; 3   IPv4 header      280              20
+; 4   Packet           256              1536
+; 5   Crypto Operation 1792             160
+
+mempool MEMPOOL0 buffer 2304 pool 32K cache 256 cpu 1
+mempool MEMPOOL_SESSION0 buffer 1024 pool 1024 cache 128 cpu 1
+
+link LINK0 dev 0000:81:00.0 rxq 1 128 MEMPOOL0 txq 1 512 promiscuous on
+
+#Cryptodev
+cryptodev CRYPTO0 dev crypto_aesni_gcm0 queue 1 1024
+
+table action profile AP0 ipv4 offset 270 fwd sym_crypto dev CRYPTO0 offset 1792 mempool_create MEMPOOL_SESSION0 mempool_init MEMPOOL_SESSION0
+table action profile AP1 ipv4 offset 270 fwd
+
+pipeline PIPELINE0 period 10 offset_port_id 0 cpu 1
+
+pipeline PIPELINE0 port in bsz 32 link LINK0 rxq 0
+pipeline PIPELINE0 port in bsz 32 cryptodev CRYPTO0 rxq 0
+
+pipeline PIPELINE0 port out bsz 32 cryptodev CRYPTO0 txq 0 offset 1792
+pipeline PIPELINE0 port out bsz 32 link LINK0 txq 0
+pipeline PIPELINE0 port out bsz 32 sink
+
+pipeline PIPELINE0 table match hash ext key 8 mask FFFFFFFF00000000 offset 282 buckets 1K size 4K action AP0
+pipeline PIPELINE0 table match stub action AP1
+
+pipeline PIPELINE0 port in 0 table 0
+pipeline PIPELINE0 port in 1 table 1
+
+thread 24 pipeline PIPELINE0 enable
+
+pipeline PIPELINE0 table 0 rule add match default action fwd port 2
+
+#AES-GCM encrypt
+pipeline PIPELINE0 table 0 rule add match hash ipv4_addr 100.0.0.10 action fwd port 0 sym_crypto encrypt type aead aead_algo aes-gcm aead_key 000102030405060708090a0b0c0d0e0f aead_iv 000102030405060708090a0b aead_aad 000102030405060708090a0b0c0d0e0f digest_size 8 data_offset 290
+#AES-GCM decrypt
+#pipeline PIPELINE0 table 0 rule add match hash ipv4_addr 100.0.0.10 action fwd port 0 sym_crypto decrypt type aead aead_algo aes-gcm aead_key 000102030405060708090a0b0c0d0e0f aead_iv 000102030405060708090a0b aead_aad 000102030405060708090a0b0c0d0e0f digest_size 8 data_offset 290
+
+pipeline PIPELINE0 table 1 rule add match default action fwd port 1
diff --git a/examples/ip_pipeline/hash_func.h b/examples/ip_pipeline/hash_func.h
deleted file mode 100644
index f1b9d944..00000000
--- a/examples/ip_pipeline/hash_func.h
+++ /dev/null
@@ -1,357 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2010-2018 Intel Corporation
- */
-
-#ifndef __INCLUDE_HASH_FUNC_H__
-#define __INCLUDE_HASH_FUNC_H__
-
-static inline uint64_t
-hash_xor_key8(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t xor0;
-
-	xor0 = seed ^ (k[0] & m[0]);
-
-	return (xor0 >> 32) ^ xor0;
-}
-
-static inline uint64_t
-hash_xor_key16(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t xor0;
-
-	xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
-
-	return (xor0 >> 32) ^ xor0;
-}
-
-static inline uint64_t
-hash_xor_key24(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t xor0;
-
-	xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
-
-	xor0 ^= k[2] & m[2];
-
-	return (xor0 >> 32) ^ xor0;
-}
-
-static inline uint64_t
-hash_xor_key32(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t xor0, xor1;
-
-	xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
-	xor1 = (k[2] & m[2]) ^ (k[3] & m[3]);
-
-	xor0 ^= xor1;
-
-	return (xor0 >> 32) ^ xor0;
-}
-
-static inline uint64_t
-hash_xor_key40(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t xor0, xor1;
-
-	xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
-	xor1 = (k[2] & m[2]) ^ (k[3] & m[3]);
-
-	xor0 ^= xor1;
-
-	xor0 ^= k[4] & m[4];
-
-	return (xor0 >> 32) ^ xor0;
-}
-
-static inline uint64_t
-hash_xor_key48(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t xor0, xor1, xor2;
-
-	xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
-	xor1 = (k[2] & m[2]) ^ (k[3] & m[3]);
-	xor2 = (k[4] & m[4]) ^ (k[5] & m[5]);
-
-	xor0 ^= xor1;
-
-	xor0 ^= xor2;
-
-	return (xor0 >> 32) ^ xor0;
-}
-
-static inline uint64_t
-hash_xor_key56(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t xor0, xor1, xor2;
-
-	xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
-	xor1 = (k[2] & m[2]) ^ (k[3] & m[3]);
-	xor2 = (k[4] & m[4]) ^ (k[5] & m[5]);
-
-	xor0 ^= xor1;
-	xor2 ^= k[6] & m[6];
-
-	xor0 ^= xor2;
-
-	return (xor0 >> 32) ^ xor0;
-}
-
-static inline uint64_t
-hash_xor_key64(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t xor0, xor1, xor2, xor3;
-
-	xor0 = ((k[0] & m[0]) ^ seed) ^ (k[1] & m[1]);
-	xor1 = (k[2] & m[2]) ^ (k[3] & m[3]);
-	xor2 = (k[4] & m[4]) ^ (k[5] & m[5]);
-	xor3 = (k[6] & m[6]) ^ (k[7] & m[7]);
-
-	xor0 ^= xor1;
-	xor2 ^= xor3;
-
-	xor0 ^= xor2;
-
-	return (xor0 >> 32) ^ xor0;
-}
-
-#if defined(RTE_ARCH_X86_64)
-
-#include <x86intrin.h>
-
-static inline uint64_t
-hash_crc_key8(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t crc0;
-
-	crc0 = _mm_crc32_u64(seed, k[0] & m[0]);
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key16(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t k0, crc0, crc1;
-
-	k0 = k[0] & m[0];
-
-	crc0 = _mm_crc32_u64(k0, seed);
-	crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
-
-	crc0 ^= crc1;
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key24(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t k0, k2, crc0, crc1;
-
-	k0 = k[0] & m[0];
-	k2 = k[2] & m[2];
-
-	crc0 = _mm_crc32_u64(k0, seed);
-	crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
-
-	crc0 = _mm_crc32_u64(crc0, k2);
-
-	crc0 ^= crc1;
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key32(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t k0, k2, crc0, crc1, crc2, crc3;
-
-	k0 = k[0] & m[0];
-	k2 = k[2] & m[2];
-
-	crc0 = _mm_crc32_u64(k0, seed);
-	crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
-
-	crc2 = _mm_crc32_u64(k2, k[3] & m[3]);
-	crc3 = k2 >> 32;
-
-	crc0 = _mm_crc32_u64(crc0, crc1);
-	crc1 = _mm_crc32_u64(crc2, crc3);
-
-	crc0 ^= crc1;
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key40(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t k0, k2, crc0, crc1, crc2, crc3;
-
-	k0 = k[0] & m[0];
-	k2 = k[2] & m[2];
-
-	crc0 = _mm_crc32_u64(k0, seed);
-	crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
-
-	crc2 = _mm_crc32_u64(k2, k[3] & m[3]);
-	crc3 = _mm_crc32_u64(k2 >> 32, k[4] & m[4]);
-
-	crc0 = _mm_crc32_u64(crc0, crc1);
-	crc1 = _mm_crc32_u64(crc2, crc3);
-
-	crc0 ^= crc1;
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key48(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t k0, k2, k5, crc0, crc1, crc2, crc3;
-
-	k0 = k[0] & m[0];
-	k2 = k[2] & m[2];
-	k5 = k[5] & m[5];
-
-	crc0 = _mm_crc32_u64(k0, seed);
-	crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
-
-	crc2 = _mm_crc32_u64(k2, k[3] & m[3]);
-	crc3 = _mm_crc32_u64(k2 >> 32, k[4] & m[4]);
-
-	crc0 = _mm_crc32_u64(crc0, (crc1 << 32) ^ crc2);
-	crc1 = _mm_crc32_u64(crc3, k5);
-
-	crc0 ^= crc1;
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key56(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t k0, k2, k5, crc0, crc1, crc2, crc3, crc4, crc5;
-
-	k0 = k[0] & m[0];
-	k2 = k[2] & m[2];
-	k5 = k[5] & m[5];
-
-	crc0 = _mm_crc32_u64(k0, seed);
-	crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
-
-	crc2 = _mm_crc32_u64(k2, k[3] & m[3]);
-	crc3 = _mm_crc32_u64(k2 >> 32, k[4] & m[4]);
-
-	crc4 = _mm_crc32_u64(k5, k[6] & m[6]);
-	crc5 = k5 >> 32;
-
-	crc0 = _mm_crc32_u64(crc0, (crc1 << 32) ^ crc2);
-	crc1 = _mm_crc32_u64(crc3, (crc4 << 32) ^ crc5);
-
-	crc0 ^= crc1;
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key64(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint64_t k0, k2, k5, crc0, crc1, crc2, crc3, crc4, crc5;
-
-	k0 = k[0] & m[0];
-	k2 = k[2] & m[2];
-	k5 = k[5] & m[5];
-
-	crc0 = _mm_crc32_u64(k0, seed);
-	crc1 = _mm_crc32_u64(k0 >> 32, k[1] & m[1]);
-
-	crc2 = _mm_crc32_u64(k2, k[3] & m[3]);
-	crc3 = _mm_crc32_u64(k2 >> 32, k[4] & m[4]);
-
-	crc4 = _mm_crc32_u64(k5, k[6] & m[6]);
-	crc5 = _mm_crc32_u64(k5 >> 32, k[7] & m[7]);
-
-	crc0 = _mm_crc32_u64(crc0, (crc1 << 32) ^ crc2);
-	crc1 = _mm_crc32_u64(crc3, (crc4 << 32) ^ crc5);
-
-	crc0 ^= crc1;
-
-	return crc0;
-}
-
-#define hash_default_key8			hash_crc_key8
-#define hash_default_key16			hash_crc_key16
-#define hash_default_key24			hash_crc_key24
-#define hash_default_key32			hash_crc_key32
-#define hash_default_key40			hash_crc_key40
-#define hash_default_key48			hash_crc_key48
-#define hash_default_key56			hash_crc_key56
-#define hash_default_key64			hash_crc_key64
-
-#elif defined(RTE_ARCH_ARM64)
-#include "hash_func_arm64.h"
-#else
-
-#define hash_default_key8			hash_xor_key8
-#define hash_default_key16			hash_xor_key16
-#define hash_default_key24			hash_xor_key24
-#define hash_default_key32			hash_xor_key32
-#define hash_default_key40			hash_xor_key40
-#define hash_default_key48			hash_xor_key48
-#define hash_default_key56			hash_xor_key56
-#define hash_default_key64			hash_xor_key64
-
-#endif
-
-#endif
diff --git a/examples/ip_pipeline/hash_func_arm64.h b/examples/ip_pipeline/hash_func_arm64.h
deleted file mode 100644
index 50df8167..00000000
--- a/examples/ip_pipeline/hash_func_arm64.h
+++ /dev/null
@@ -1,232 +0,0 @@
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2017-2018 Linaro Limited.
- */
-#ifndef __HASH_FUNC_ARM64_H__
-#define __HASH_FUNC_ARM64_H__
-
-#define _CRC32CX(crc, val)	\
-	__asm__("crc32cx %w[c], %w[c], %x[v]":[c] "+r" (crc):[v] "r" (val))
-
-static inline uint64_t
-hash_crc_key8(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key;
-	uint64_t *m = mask;
-	uint32_t crc0;
-
-	crc0 = seed;
-	_CRC32CX(crc0, k[0] & m[0]);
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key16(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key, k0;
-	uint64_t *m = mask;
-	uint32_t crc0, crc1;
-
-	k0 = k[0] & m[0];
-
-	crc0 = k0;
-	_CRC32CX(crc0, seed);
-	crc1 = k0 >> 32;
-	_CRC32CX(crc1, k[1] & m[1]);
-
-	crc0 ^= crc1;
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key24(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key, k0, k2;
-	uint64_t *m = mask;
-	uint32_t crc0, crc1;
-
-	k0 = k[0] & m[0];
-	k2 = k[2] & m[2];
-
-	crc0 = k0;
-	_CRC32CX(crc0, seed);
-	crc1 = k0 >> 32;
-	_CRC32CX(crc1, k[1] & m[1]);
-
-	_CRC32CX(crc0, k2);
-
-	crc0 ^= crc1;
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key32(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key, k0, k2;
-	uint64_t *m = mask;
-	uint32_t crc0, crc1, crc2, crc3;
-
-	k0 = k[0] & m[0];
-	k2 = k[2] & m[2];
-
-	crc0 = k0;
-	_CRC32CX(crc0, seed);
-	crc1 = k0 >> 32;
-	_CRC32CX(crc1, k[1] & m[1]);
-
-	crc2 = k2;
-	_CRC32CX(crc2, k[3] & m[3]);
-	crc3 = k2 >> 32;
-
-	_CRC32CX(crc0, crc1);
-	_CRC32CX(crc2, crc3);
-
-	crc0 ^= crc2;
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key40(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key, k0, k2;
-	uint64_t *m = mask;
-	uint32_t crc0, crc1, crc2, crc3;
-
-	k0 = k[0] & m[0];
-	k2 = k[2] & m[2];
-
-	crc0 = k0;
-	_CRC32CX(crc0, seed);
-	crc1 = k0 >> 32;
-	_CRC32CX(crc1, k[1] & m[1]);
-
-	crc2 = k2;
-	_CRC32CX(crc2, k[3] & m[3]);
-	crc3 = k2 >> 32;
-	_CRC32CX(crc3, k[4] & m[4]);
-
-	_CRC32CX(crc0, crc1);
-	_CRC32CX(crc2, crc3);
-
-	crc0 ^= crc2;
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key48(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key, k0, k2, k5;
-	uint64_t *m = mask;
-	uint32_t crc0, crc1, crc2, crc3;
-
-	k0 = k[0] & m[0];
-	k2 = k[2] & m[2];
-	k5 = k[5] & m[5];
-
-	crc0 = k0;
-	_CRC32CX(crc0, seed);
-	crc1 = k0 >> 32;
-	_CRC32CX(crc1, k[1] & m[1]);
-
-	crc2 = k2;
-	_CRC32CX(crc2, k[3] & m[3]);
-	crc3 = k2 >> 32;
-	_CRC32CX(crc3, k[4] & m[4]);
-
-	_CRC32CX(crc0, ((uint64_t)crc1 << 32) ^ crc2);
-	_CRC32CX(crc3, k5);
-
-	crc0 ^= crc3;
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key56(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key, k0, k2, k5;
-	uint64_t *m = mask;
-	uint32_t crc0, crc1, crc2, crc3, crc4, crc5;
-
-	k0 = k[0] & m[0];
-	k2 = k[2] & m[2];
-	k5 = k[5] & m[5];
-
-	crc0 = k0;
-	_CRC32CX(crc0, seed);
-	crc1 = k0 >> 32;
-	_CRC32CX(crc1, k[1] & m[1]);
-
-	crc2 = k2;
-	_CRC32CX(crc2, k[3] & m[3]);
-	crc3 = k2 >> 32;
-	_CRC32CX(crc3, k[4] & m[4]);
-
-	crc4 = k5;
-	 _CRC32CX(crc4, k[6] & m[6]);
-	crc5 = k5 >> 32;
-
-	_CRC32CX(crc0, ((uint64_t)crc1 << 32) ^ crc2);
-	_CRC32CX(crc3, ((uint64_t)crc4 << 32) ^ crc5);
-
-	crc0 ^= crc3;
-
-	return crc0;
-}
-
-static inline uint64_t
-hash_crc_key64(void *key, void *mask, __rte_unused uint32_t key_size,
-	uint64_t seed)
-{
-	uint64_t *k = key, k0, k2, k5;
-	uint64_t *m = mask;
-	uint32_t crc0, crc1, crc2, crc3, crc4, crc5;
-
-	k0 = k[0] & m[0];
-	k2 = k[2] & m[2];
-	k5 = k[5] & m[5];
-
-	crc0 = k0;
-	_CRC32CX(crc0, seed);
-	crc1 = k0 >> 32;
-	_CRC32CX(crc1, k[1] & m[1]);
-
-	crc2 = k2;
-	_CRC32CX(crc2, k[3] & m[3]);
-	crc3 = k2 >> 32;
-	_CRC32CX(crc3, k[4] & m[4]);
-
-	crc4 = k5;
-	 _CRC32CX(crc4, k[6] & m[6]);
-	crc5 = k5 >> 32;
-	_CRC32CX(crc5, k[7] & m[7]);
-
-	_CRC32CX(crc0, ((uint64_t)crc1 << 32) ^ crc2);
-	_CRC32CX(crc3, ((uint64_t)crc4 << 32) ^ crc5);
-
-	crc0 ^= crc3;
-
-	return crc0;
-}
-
-#define hash_default_key8			hash_crc_key8
-#define hash_default_key16			hash_crc_key16
-#define hash_default_key24			hash_crc_key24
-#define hash_default_key32			hash_crc_key32
-#define hash_default_key40			hash_crc_key40
-#define hash_default_key48			hash_crc_key48
-#define hash_default_key56			hash_crc_key56
-#define hash_default_key64			hash_crc_key64
-
-#endif
diff --git a/examples/ip_pipeline/link.c b/examples/ip_pipeline/link.c
index 392a890f..787eb866 100644
--- a/examples/ip_pipeline/link.c
+++ b/examples/ip_pipeline/link.c
@@ -48,7 +48,6 @@ static struct rte_eth_conf port_conf_default = {
 		.mq_mode = ETH_MQ_RX_NONE,
 		.max_rx_pkt_len = 9000, /* Jumbo frame max packet len */
 		.split_hdr_size = 0, /* Header split buffer size */
-		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.rx_adv_conf = {
 		.rss_conf = {
diff --git a/examples/ip_pipeline/main.c b/examples/ip_pipeline/main.c
index a69facee..97d1e91c 100644
--- a/examples/ip_pipeline/main.c
+++ b/examples/ip_pipeline/main.c
@@ -14,6 +14,7 @@
 #include "cli.h"
 #include "conn.h"
 #include "kni.h"
+#include "cryptodev.h"
 #include "link.h"
 #include "mempool.h"
 #include "pipeline.h"
@@ -210,6 +211,14 @@ main(int argc, char **argv)
 		return status;
 	}
 
+	/* Sym Crypto */
+	status = cryptodev_init();
+	if (status) {
+		printf("Error: Cryptodev initialization failed (%d)\n",
+				status);
+		return status;
+	}
+
 	/* Action */
 	status = port_in_action_profile_init();
 	if (status) {
diff --git a/examples/ip_pipeline/meson.build b/examples/ip_pipeline/meson.build
index a9f2ea44..5e5fe647 100644
--- a/examples/ip_pipeline/meson.build
+++ b/examples/ip_pipeline/meson.build
@@ -21,5 +21,6 @@ sources = files(
 	'swq.c',
 	'tap.c',
 	'thread.c',
-	'tmgr.c'
+	'tmgr.c',
+	'cryptodev.c'
 )
diff --git a/examples/ip_pipeline/pipeline.c b/examples/ip_pipeline/pipeline.c
index 43fe8677..78d590d7 100644
--- a/examples/ip_pipeline/pipeline.c
+++ b/examples/ip_pipeline/pipeline.c
@@ -18,10 +18,12 @@
 #include <rte_port_source_sink.h>
 #include <rte_port_fd.h>
 #include <rte_port_sched.h>
+#include <rte_port_sym_crypto.h>
 
 #include <rte_table_acl.h>
 #include <rte_table_array.h>
 #include <rte_table_hash.h>
+#include <rte_table_hash_func.h>
 #include <rte_table_lpm.h>
 #include <rte_table_lpm_ipv6.h>
 #include <rte_table_stub.h>
@@ -35,8 +37,7 @@
 #include "tap.h"
 #include "tmgr.h"
 #include "swq.h"
-
-#include "hash_func.h"
+#include "cryptodev.h"
 
 #ifndef PIPELINE_MSGQ_SIZE
 #define PIPELINE_MSGQ_SIZE                                 64
@@ -163,6 +164,7 @@ pipeline_port_in_create(const char *pipeline_name,
 		struct rte_port_kni_reader_params kni;
 #endif
 		struct rte_port_source_params source;
+		struct rte_port_sym_crypto_reader_params sym_crypto;
 	} pp;
 
 	struct pipeline *pipeline;
@@ -296,6 +298,27 @@ pipeline_port_in_create(const char *pipeline_name,
 		break;
 	}
 
+	case PORT_IN_CRYPTODEV:
+	{
+		struct cryptodev *cryptodev;
+
+		cryptodev = cryptodev_find(params->dev_name);
+		if (cryptodev == NULL)
+			return -1;
+
+		if (params->rxq.queue_id > cryptodev->n_queues - 1)
+			return -1;
+
+		pp.sym_crypto.cryptodev_id = cryptodev->dev_id;
+		pp.sym_crypto.queue_id = params->cryptodev.queue_id;
+		pp.sym_crypto.f_callback = params->cryptodev.f_callback;
+		pp.sym_crypto.arg_callback = params->cryptodev.arg_callback;
+		p.ops = &rte_port_sym_crypto_reader_ops;
+		p.arg_create = &pp.sym_crypto;
+
+		break;
+	}
+
 	default:
 		return -1;
 	}
@@ -385,6 +408,7 @@ pipeline_port_out_create(const char *pipeline_name,
 		struct rte_port_kni_writer_params kni;
 #endif
 		struct rte_port_sink_params sink;
+		struct rte_port_sym_crypto_writer_params sym_crypto;
 	} pp;
 
 	union {
@@ -394,6 +418,7 @@ pipeline_port_out_create(const char *pipeline_name,
 #ifdef RTE_LIBRTE_KNI
 		struct rte_port_kni_writer_nodrop_params kni;
 #endif
+		struct rte_port_sym_crypto_writer_nodrop_params sym_crypto;
 	} pp_nodrop;
 
 	struct pipeline *pipeline;
@@ -549,6 +574,40 @@ pipeline_port_out_create(const char *pipeline_name,
 		break;
 	}
 
+	case PORT_OUT_CRYPTODEV:
+	{
+		struct cryptodev *cryptodev;
+
+		cryptodev = cryptodev_find(params->dev_name);
+		if (cryptodev == NULL)
+			return -1;
+
+		if (params->cryptodev.queue_id >= cryptodev->n_queues)
+			return -1;
+
+		pp.sym_crypto.cryptodev_id = cryptodev->dev_id;
+		pp.sym_crypto.queue_id = params->cryptodev.queue_id;
+		pp.sym_crypto.tx_burst_sz = params->burst_size;
+		pp.sym_crypto.crypto_op_offset = params->cryptodev.op_offset;
+
+		pp_nodrop.sym_crypto.cryptodev_id = cryptodev->dev_id;
+		pp_nodrop.sym_crypto.queue_id = params->cryptodev.queue_id;
+		pp_nodrop.sym_crypto.tx_burst_sz = params->burst_size;
+		pp_nodrop.sym_crypto.n_retries = params->retry;
+		pp_nodrop.sym_crypto.crypto_op_offset =
+				params->cryptodev.op_offset;
+
+		if (params->retry == 0) {
+			p.ops = &rte_port_sym_crypto_writer_ops;
+			p.arg_create = &pp.sym_crypto;
+		} else {
+			p.ops = &rte_port_sym_crypto_writer_nodrop_ops;
+			p.arg_create = &pp_nodrop.sym_crypto;
+		}
+
+		break;
+	}
+
 	default:
 		return -1;
 	}
@@ -818,28 +877,28 @@ pipeline_table_create(const char *pipeline_name,
 
 		switch (params->match.hash.key_size) {
 		case  8:
-			f_hash = hash_default_key8;
+			f_hash = rte_table_hash_crc_key8;
 			break;
 		case 16:
-			f_hash = hash_default_key16;
+			f_hash = rte_table_hash_crc_key16;
 			break;
 		case 24:
-			f_hash = hash_default_key24;
+			f_hash = rte_table_hash_crc_key24;
 			break;
 		case 32:
-			f_hash = hash_default_key32;
+			f_hash = rte_table_hash_crc_key32;
 			break;
 		case 40:
-			f_hash = hash_default_key40;
+			f_hash = rte_table_hash_crc_key40;
 			break;
 		case 48:
-			f_hash = hash_default_key48;
+			f_hash = rte_table_hash_crc_key48;
 			break;
 		case 56:
-			f_hash = hash_default_key56;
+			f_hash = rte_table_hash_crc_key56;
 			break;
 		case 64:
-			f_hash = hash_default_key64;
+			f_hash = rte_table_hash_crc_key64;
 			break;
 		default:
 			return -1;
@@ -982,7 +1041,95 @@ pipeline_table_create(const char *pipeline_name,
 	memcpy(&table->params, params, sizeof(*params));
 	table->ap = ap;
 	table->a = action;
+	TAILQ_INIT(&table->rules);
+	table->rule_default = NULL;
+
 	pipeline->n_tables++;
 
 	return 0;
 }
+
+struct table_rule *
+table_rule_find(struct table *table,
+    struct table_rule_match *match)
+{
+	struct table_rule *rule;
+
+	TAILQ_FOREACH(rule, &table->rules, node)
+		if (memcmp(&rule->match, match, sizeof(*match)) == 0)
+			return rule;
+
+	return NULL;
+}
+
+void
+table_rule_add(struct table *table,
+    struct table_rule *new_rule)
+{
+	struct table_rule *existing_rule;
+
+	existing_rule = table_rule_find(table, &new_rule->match);
+	if (existing_rule == NULL)
+		TAILQ_INSERT_TAIL(&table->rules, new_rule, node);
+	else {
+		TAILQ_INSERT_AFTER(&table->rules, existing_rule, new_rule, node);
+		TAILQ_REMOVE(&table->rules, existing_rule, node);
+		free(existing_rule);
+	}
+}
+
+void
+table_rule_add_bulk(struct table *table,
+    struct table_rule_list *list,
+    uint32_t n_rules)
+{
+	uint32_t i;
+
+	for (i = 0; i < n_rules; i++) {
+		struct table_rule *existing_rule, *new_rule;
+
+		new_rule = TAILQ_FIRST(list);
+		if (new_rule == NULL)
+			break;
+
+		TAILQ_REMOVE(list, new_rule, node);
+
+		existing_rule = table_rule_find(table, &new_rule->match);
+		if (existing_rule == NULL)
+			TAILQ_INSERT_TAIL(&table->rules, new_rule, node);
+		else {
+			TAILQ_INSERT_AFTER(&table->rules, existing_rule, new_rule, node);
+			TAILQ_REMOVE(&table->rules, existing_rule, node);
+			free(existing_rule);
+		}
+	}
+}
+
+void
+table_rule_delete(struct table *table,
+    struct table_rule_match *match)
+{
+	struct table_rule *rule;
+
+	rule = table_rule_find(table, match);
+	if (rule == NULL)
+		return;
+
+	TAILQ_REMOVE(&table->rules, rule, node);
+	free(rule);
+}
+
+void
+table_rule_default_add(struct table *table,
+	struct table_rule *rule)
+{
+	free(table->rule_default);
+	table->rule_default = rule;
+}
+
+void
+table_rule_default_delete(struct table *table)
+{
+	free(table->rule_default);
+	table->rule_default = NULL;
+}
diff --git a/examples/ip_pipeline/pipeline.h b/examples/ip_pipeline/pipeline.h
index a953a29f..278775c2 100644
--- a/examples/ip_pipeline/pipeline.h
+++ b/examples/ip_pipeline/pipeline.h
@@ -27,6 +27,7 @@ enum port_in_type {
 	PORT_IN_TAP,
 	PORT_IN_KNI,
 	PORT_IN_SOURCE,
+	PORT_IN_CRYPTODEV,
 };
 
 struct port_in_params {
@@ -48,6 +49,12 @@ struct port_in_params {
 			const char *file_name;
 			uint32_t n_bytes_per_pkt;
 		} source;
+
+		struct {
+			uint16_t queue_id;
+			void *f_callback;
+			void *arg_callback;
+		} cryptodev;
 	};
 	uint32_t burst_size;
 
@@ -62,6 +69,7 @@ enum port_out_type {
 	PORT_OUT_TAP,
 	PORT_OUT_KNI,
 	PORT_OUT_SINK,
+	PORT_OUT_CRYPTODEV,
 };
 
 struct port_out_params {
@@ -76,6 +84,11 @@ struct port_out_params {
 			const char *file_name;
 			uint32_t max_n_pkts;
 		} sink;
+
+		struct {
+			uint16_t queue_id;
+			uint32_t op_offset;
+		} cryptodev;
 	};
 	uint32_t burst_size;
 	int retry;
@@ -130,6 +143,10 @@ struct table_params {
 	const char *action_profile_name;
 };
 
+struct table_rule;
+
+TAILQ_HEAD(table_rule_list, table_rule);
+
 struct port_in {
 	struct port_in_params params;
 	struct port_in_action_profile *ap;
@@ -140,6 +157,8 @@ struct table {
 	struct table_params params;
 	struct table_action_profile *ap;
 	struct rte_table_action *a;
+	struct table_rule_list rules;
+	struct table_rule *rule_default;
 };
 
 struct pipeline {
@@ -268,6 +287,16 @@ struct table_rule_action {
 	struct rte_table_action_ttl_params ttl;
 	struct rte_table_action_stats_params stats;
 	struct rte_table_action_time_params time;
+	struct rte_table_action_sym_crypto_params sym_crypto;
+	struct rte_table_action_tag_params tag;
+	struct rte_table_action_decap_params decap;
+};
+
+struct table_rule {
+	TAILQ_ENTRY(table_rule) node;
+	struct table_rule_match match;
+	struct table_rule_action action;
+	void *data;
 };
 
 int
@@ -300,22 +329,19 @@ int
 pipeline_table_rule_add(const char *pipeline_name,
 	uint32_t table_id,
 	struct table_rule_match *match,
-	struct table_rule_action *action,
-	void **data);
+	struct table_rule_action *action);
 
 int
 pipeline_table_rule_add_bulk(const char *pipeline_name,
 	uint32_t table_id,
-	struct table_rule_match *match,
-	struct table_rule_action *action,
-	void **data,
-	uint32_t *n_rules);
+	struct table_rule_list *list,
+	uint32_t *n_rules_added,
+	uint32_t *n_rules_not_added);
 
 int
 pipeline_table_rule_add_default(const char *pipeline_name,
 	uint32_t table_id,
-	struct table_rule_action *action,
-	void **data);
+	struct table_rule_action *action);
 
 int
 pipeline_table_rule_delete(const char *pipeline_name,
@@ -329,7 +355,7 @@ pipeline_table_rule_delete_default(const char *pipeline_name,
 int
 pipeline_table_rule_stats_read(const char *pipeline_name,
 	uint32_t table_id,
-	void *data,
+	struct table_rule_match *match,
 	struct rte_table_action_stats_counters *stats,
 	int clear);
 
@@ -347,8 +373,7 @@ pipeline_table_mtr_profile_delete(const char *pipeline_name,
 int
 pipeline_table_rule_mtr_read(const char *pipeline_name,
 	uint32_t table_id,
-	void *data,
-	uint32_t tc_mask,
+	struct table_rule_match *match,
 	struct rte_table_action_mtr_counters *stats,
 	int clear);
 
@@ -361,8 +386,38 @@ pipeline_table_dscp_table_update(const char *pipeline_name,
 int
 pipeline_table_rule_ttl_read(const char *pipeline_name,
 	uint32_t table_id,
-	void *data,
+	struct table_rule_match *match,
 	struct rte_table_action_ttl_counters *stats,
 	int clear);
 
+int
+pipeline_table_rule_time_read(const char *pipeline_name,
+	uint32_t table_id,
+	struct table_rule_match *match,
+	uint64_t *timestamp);
+
+struct table_rule *
+table_rule_find(struct table *table,
+	struct table_rule_match *match);
+
+void
+table_rule_add(struct table *table,
+	struct table_rule *rule);
+
+void
+table_rule_add_bulk(struct table *table,
+	struct table_rule_list *list,
+	uint32_t n_rules);
+
+void
+table_rule_delete(struct table *table,
+	struct table_rule_match *match);
+
+void
+table_rule_default_add(struct table *table,
+	struct table_rule *rule);
+
+void
+table_rule_default_delete(struct table *table);
+
 #endif /* _INCLUDE_PIPELINE_H_ */
diff --git a/examples/ip_pipeline/thread.c b/examples/ip_pipeline/thread.c
index 7fc03332..272fbbee 100644
--- a/examples/ip_pipeline/thread.c
+++ b/examples/ip_pipeline/thread.c
@@ -584,6 +584,7 @@ enum pipeline_req_type {
 	PIPELINE_REQ_TABLE_RULE_MTR_READ,
 	PIPELINE_REQ_TABLE_DSCP_TABLE_UPDATE,
 	PIPELINE_REQ_TABLE_RULE_TTL_READ,
+	PIPELINE_REQ_TABLE_RULE_TIME_READ,
 	PIPELINE_REQ_MAX
 };
 
@@ -609,10 +610,7 @@ struct pipeline_msg_req_table_rule_add_default {
 };
 
 struct pipeline_msg_req_table_rule_add_bulk {
-	struct table_rule_match *match;
-	struct table_rule_action *action;
-	void **data;
-	uint32_t n_rules;
+	struct table_rule_list *list;
 	int bulk;
 };
 
@@ -650,6 +648,10 @@ struct pipeline_msg_req_table_rule_ttl_read {
 	int clear;
 };
 
+struct pipeline_msg_req_table_rule_time_read {
+	void *data;
+};
+
 struct pipeline_msg_req {
 	enum pipeline_req_type type;
 	uint32_t id; /* Port IN, port OUT or table ID */
@@ -669,6 +671,7 @@ struct pipeline_msg_req {
 		struct pipeline_msg_req_table_rule_mtr_read table_rule_mtr_read;
 		struct pipeline_msg_req_table_dscp_table_update table_dscp_table_update;
 		struct pipeline_msg_req_table_rule_ttl_read table_rule_ttl_read;
+		struct pipeline_msg_req_table_rule_time_read table_rule_time_read;
 	};
 };
 
@@ -708,6 +711,10 @@ struct pipeline_msg_rsp_table_rule_ttl_read {
 	struct rte_table_action_ttl_counters stats;
 };
 
+struct pipeline_msg_rsp_table_rule_time_read {
+	uint64_t timestamp;
+};
+
 struct pipeline_msg_rsp {
 	int status;
 
@@ -722,6 +729,7 @@ struct pipeline_msg_rsp {
 		struct pipeline_msg_rsp_table_rule_stats_read table_rule_stats_read;
 		struct pipeline_msg_rsp_table_rule_mtr_read table_rule_mtr_read;
 		struct pipeline_msg_rsp_table_rule_ttl_read table_rule_ttl_read;
+		struct pipeline_msg_rsp_table_rule_time_read table_rule_time_read;
 	};
 };
 
@@ -812,7 +820,7 @@ pipeline_port_in_stats_read(const char *pipeline_name,
 
 	/* Read response */
 	status = rsp->status;
-	if (status)
+	if (status == 0)
 		memcpy(stats, &rsp->port_in_stats_read.stats, sizeof(*stats));
 
 	/* Free response */
@@ -960,7 +968,7 @@ pipeline_port_out_stats_read(const char *pipeline_name,
 
 	/* Read response */
 	status = rsp->status;
-	if (status)
+	if (status == 0)
 		memcpy(stats, &rsp->port_out_stats_read.stats, sizeof(*stats));
 
 	/* Free response */
@@ -1016,7 +1024,7 @@ pipeline_table_stats_read(const char *pipeline_name,
 
 	/* Read response */
 	status = rsp->status;
-	if (status)
+	if (status == 0)
 		memcpy(stats, &rsp->table_stats_read.stats, sizeof(*stats));
 
 	/* Free response */
@@ -1206,23 +1214,145 @@ action_convert(struct rte_table_action *a,
 	struct table_rule_action *action,
 	struct rte_pipeline_table_entry *data);
 
+struct table_ll {
+	struct rte_pipeline *p;
+	int table_id;
+	struct rte_table_action *a;
+	int bulk_supported;
+};
+
+static int
+table_rule_add_bulk_ll(struct table_ll *table,
+	struct table_rule_list *list,
+	uint32_t *n_rules)
+{
+	union table_rule_match_low_level *match_ll = NULL;
+	uint8_t *action_ll = NULL;
+	void **match_ll_ptr = NULL;
+	struct rte_pipeline_table_entry **action_ll_ptr = NULL;
+	struct rte_pipeline_table_entry **entries_ptr = NULL;
+	int *found = NULL;
+	struct table_rule *rule;
+	uint32_t n, i;
+	int status = 0;
+
+	n = 0;
+	TAILQ_FOREACH(rule, list, node)
+		n++;
+
+	/* Memory allocation */
+	match_ll = calloc(n, sizeof(union table_rule_match_low_level));
+	action_ll = calloc(n, TABLE_RULE_ACTION_SIZE_MAX);
+
+	match_ll_ptr = calloc(n, sizeof(void *));
+	action_ll_ptr = calloc(n, sizeof(struct rte_pipeline_table_entry *));
+
+	entries_ptr = calloc(n, sizeof(struct rte_pipeline_table_entry *));
+	found = calloc(n, sizeof(int));
+
+	if (match_ll == NULL ||
+		action_ll == NULL ||
+		match_ll_ptr == NULL ||
+		action_ll_ptr == NULL ||
+		entries_ptr == NULL ||
+		found == NULL) {
+			status = -ENOMEM;
+			goto table_rule_add_bulk_ll_free;
+	}
+
+	/* Init */
+	for (i = 0; i < n; i++) {
+		match_ll_ptr[i] = (void *)&match_ll[i];
+		action_ll_ptr[i] = (struct rte_pipeline_table_entry *)
+			&action_ll[i * TABLE_RULE_ACTION_SIZE_MAX];
+	}
+
+	/* Rule (match, action) conversion */
+	i = 0;
+	TAILQ_FOREACH(rule, list, node) {
+		status = match_convert(&rule->match, match_ll_ptr[i], 1);
+		if (status)
+			goto table_rule_add_bulk_ll_free;
+
+		status = action_convert(table->a, &rule->action, action_ll_ptr[i]);
+		if (status)
+			goto table_rule_add_bulk_ll_free;
+
+		i++;
+	}
+
+	/* Add rule (match, action) to table */
+	if (table->bulk_supported) {
+		status = rte_pipeline_table_entry_add_bulk(table->p,
+			table->table_id,
+			match_ll_ptr,
+			action_ll_ptr,
+			n,
+			found,
+			entries_ptr);
+		if (status)
+			goto table_rule_add_bulk_ll_free;
+	} else
+		for (i = 0; i < n; i++) {
+			status = rte_pipeline_table_entry_add(table->p,
+				table->table_id,
+				match_ll_ptr[i],
+				action_ll_ptr[i],
+				&found[i],
+				&entries_ptr[i]);
+			if (status) {
+				if (i == 0)
+					goto table_rule_add_bulk_ll_free;
+
+				/* No roll-back. */
+				status = 0;
+				n = i;
+				break;
+			}
+		}
+
+	/* Write back to the rule list. */
+	i = 0;
+	TAILQ_FOREACH(rule, list, node) {
+		if (i >= n)
+			break;
+
+		rule->data = entries_ptr[i];
+
+		i++;
+	}
+
+	*n_rules = n;
+
+	/* Free */
+table_rule_add_bulk_ll_free:
+	free(found);
+	free(entries_ptr);
+	free(action_ll_ptr);
+	free(match_ll_ptr);
+	free(action_ll);
+	free(match_ll);
+
+	return status;
+}
+
 int
 pipeline_table_rule_add(const char *pipeline_name,
 	uint32_t table_id,
 	struct table_rule_match *match,
-	struct table_rule_action *action,
-	void **data)
+	struct table_rule_action *action)
 {
 	struct pipeline *p;
+	struct table *table;
 	struct pipeline_msg_req *req;
 	struct pipeline_msg_rsp *rsp;
+	struct table_rule *rule;
 	int status;
 
 	/* Check input params */
 	if ((pipeline_name == NULL) ||
 		(match == NULL) ||
-		(action == NULL) ||
-		(data == NULL))
+		(action == NULL))
 		return -1;
 
 	p = pipeline_find(pipeline_name);
@@ -1232,16 +1362,26 @@ pipeline_table_rule_add(const char *pipeline_name,
 		action_check(action, p, table_id))
 		return -1;
 
+	table = &p->table[table_id];
+
+	rule = calloc(1, sizeof(struct table_rule));
+	if (rule == NULL)
+		return -1;
+
+	memcpy(&rule->match, match, sizeof(*match));
+	memcpy(&rule->action, action, sizeof(*action));
+
 	if (!pipeline_is_running(p)) {
-		struct rte_table_action *a = p->table[table_id].a;
 		union table_rule_match_low_level match_ll;
 		struct rte_pipeline_table_entry *data_in, *data_out;
 		int key_found;
 		uint8_t *buffer;
 
 		buffer = calloc(TABLE_RULE_ACTION_SIZE_MAX, sizeof(uint8_t));
-		if (buffer == NULL)
+		if (buffer == NULL) {
+			free(rule);
 			return -1;
+		}
 
 		/* Table match-action rule conversion */
 		data_in = (struct rte_pipeline_table_entry *)buffer;
@@ -1249,12 +1389,14 @@ pipeline_table_rule_add(const char *pipeline_name,
 		status = match_convert(match, &match_ll, 1);
 		if (status) {
 			free(buffer);
+			free(rule);
 			return -1;
 		}
 
-		status = action_convert(a, action, data_in);
+		status = action_convert(table->a, action, data_in);
 		if (status) {
 			free(buffer);
+			free(rule);
 			return -1;
 		}
 
@@ -1267,11 +1409,13 @@ pipeline_table_rule_add(const char *pipeline_name,
 				&data_out);
 		if (status) {
 			free(buffer);
+			free(rule);
 			return -1;
 		}
 
 		/* Write Response */
-		*data = data_out;
+		rule->data = data_out;
+		table_rule_add(table, rule);
 
 		free(buffer);
 		return 0;
@@ -1279,8 +1423,10 @@ pipeline_table_rule_add(const char *pipeline_name,
 
 	/* Allocate request */
 	req = pipeline_msg_alloc();
-	if (req == NULL)
+	if (req == NULL) {
+		free(rule);
 		return -1;
+	}
 
 	/* Write request */
 	req->type = PIPELINE_REQ_TABLE_RULE_ADD;
@@ -1290,13 +1436,18 @@ pipeline_table_rule_add(const char *pipeline_name,
 
 	/* Send request and wait for response */
 	rsp = pipeline_msg_send_recv(p, req);
-	if (rsp == NULL)
+	if (rsp == NULL) {
+		free(rule);
 		return -1;
+	}
 
 	/* Read response */
 	status = rsp->status;
-	if (status == 0)
-		*data = rsp->table_rule_add.data;
+	if (status == 0) {
+		rule->data = rsp->table_rule_add.data;
+		table_rule_add(table, rule);
+	} else
+		free(rule);
 
 	/* Free response */
 	pipeline_msg_free(rsp);
@@ -1307,18 +1458,18 @@ pipeline_table_rule_add(const char *pipeline_name,
 int
 pipeline_table_rule_add_default(const char *pipeline_name,
 	uint32_t table_id,
-	struct table_rule_action *action,
-	void **data)
+	struct table_rule_action *action)
 {
 	struct pipeline *p;
+	struct table *table;
 	struct pipeline_msg_req *req;
 	struct pipeline_msg_rsp *rsp;
+	struct table_rule *rule;
 	int status;
 
 	/* Check input params */
 	if ((pipeline_name == NULL) ||
-		(action == NULL) ||
-		(data == NULL))
+		(action == NULL))
 		return -1;
 
 	p = pipeline_find(pipeline_name);
@@ -1327,13 +1478,23 @@ pipeline_table_rule_add_default(const char *pipeline_name,
 		action_default_check(action, p, table_id))
 		return -1;
 
+	table = &p->table[table_id];
+
+	rule = calloc(1, sizeof(struct table_rule));
+	if (rule == NULL)
+		return -1;
+
+	memcpy(&rule->action, action, sizeof(*action));
+
 	if (!pipeline_is_running(p)) {
 		struct rte_pipeline_table_entry *data_in, *data_out;
 		uint8_t *buffer;
 
 		buffer = calloc(TABLE_RULE_ACTION_SIZE_MAX, sizeof(uint8_t));
-		if (buffer == NULL)
+		if (buffer == NULL) {
+			free(rule);
 			return -1;
+		}
 
 		/* Apply actions */
 		data_in = (struct rte_pipeline_table_entry *)buffer;
@@ -1351,11 +1512,13 @@ pipeline_table_rule_add_default(const char *pipeline_name,
 				&data_out);
 		if (status) {
 			free(buffer);
+			free(rule);
 			return -1;
 		}
 
 		/* Write Response */
-		*data = data_out;
+		rule->data = data_out;
+		table_rule_default_add(table, rule);
 
 		free(buffer);
 		return 0;
@@ -1363,8 +1526,10 @@ pipeline_table_rule_add_default(const char *pipeline_name,
 
 	/* Allocate request */
 	req = pipeline_msg_alloc();
-	if (req == NULL)
+	if (req == NULL) {
+		free(rule);
 		return -1;
+	}
 
 	/* Write request */
 	req->type = PIPELINE_REQ_TABLE_RULE_ADD_DEFAULT;
@@ -1373,13 +1538,18 @@ pipeline_table_rule_add_default(const char *pipeline_name,
 
 	/* Send request and wait for response */
 	rsp = pipeline_msg_send_recv(p, req);
-	if (rsp == NULL)
+	if (rsp == NULL) {
+		free(rule);
 		return -1;
+	}
 
 	/* Read response */
 	status = rsp->status;
-	if (status == 0)
-		*data = rsp->table_rule_add_default.data;
+	if (status == 0) {
+		rule->data = rsp->table_rule_add_default.data;
+		table_rule_default_add(table, rule);
+	} else
+		free(rule);
 
 	/* Free response */
 	pipeline_msg_free(rsp);
@@ -1387,156 +1557,119 @@ pipeline_table_rule_add_default(const char *pipeline_name,
 	return status;
 }
 
+static uint32_t
+table_rule_list_free(struct table_rule_list *list)
+{
+	uint32_t n = 0;
+
+	if (!list)
+		return 0;
+
+	for ( ; ; ) {
+		struct table_rule *rule;
+
+		rule = TAILQ_FIRST(list);
+		if (rule == NULL)
+			break;
+
+		TAILQ_REMOVE(list, rule, node);
+		free(rule);
+		n++;
+	}
+
+	free(list);
+	return n;
+}
+
 int
 pipeline_table_rule_add_bulk(const char *pipeline_name,
 	uint32_t table_id,
-	struct table_rule_match *match,
-	struct table_rule_action *action,
-	void **data,
-	uint32_t *n_rules)
+	struct table_rule_list *list,
+	uint32_t *n_rules_added,
+	uint32_t *n_rules_not_added)
 {
 	struct pipeline *p;
+	struct table *table;
 	struct pipeline_msg_req *req;
 	struct pipeline_msg_rsp *rsp;
-	uint32_t i;
-	int status;
+	struct table_rule *rule;
+	int status = 0;
 
 	/* Check input params */
 	if ((pipeline_name == NULL) ||
-		(match == NULL) ||
-		(action == NULL) ||
-		(data == NULL) ||
-		(n_rules == NULL) ||
-		(*n_rules == 0))
-		return -1;
+		(list == NULL) ||
+		TAILQ_EMPTY(list) ||
+		(n_rules_added == NULL) ||
+		(n_rules_not_added == NULL)) {
+		table_rule_list_free(list);
+		return -EINVAL;
+	}
 
 	p = pipeline_find(pipeline_name);
 	if ((p == NULL) ||
-		(table_id >= p->n_tables))
-		return -1;
-
-	for (i = 0; i < *n_rules; i++)
-		if (match_check(match, p, table_id) ||
-			action_check(action, p, table_id))
-			return -1;
-
-	if (!pipeline_is_running(p)) {
-		struct rte_table_action *a = p->table[table_id].a;
-		union table_rule_match_low_level *match_ll;
-		uint8_t *action_ll;
-		void **match_ll_ptr;
-		struct rte_pipeline_table_entry **action_ll_ptr;
-		struct rte_pipeline_table_entry **entries_ptr =
-			(struct rte_pipeline_table_entry **)data;
-		uint32_t bulk =
-			(p->table[table_id].params.match_type == TABLE_ACL) ? 1 : 0;
-		int *found;
-
-		/* Memory allocation */
-		match_ll = calloc(*n_rules, sizeof(union table_rule_match_low_level));
-		action_ll = calloc(*n_rules, TABLE_RULE_ACTION_SIZE_MAX);
-		match_ll_ptr = calloc(*n_rules, sizeof(void *));
-		action_ll_ptr =
-			calloc(*n_rules, sizeof(struct rte_pipeline_table_entry *));
-		found = calloc(*n_rules, sizeof(int));
-
-		if (match_ll == NULL ||
-			action_ll == NULL ||
-			match_ll_ptr == NULL ||
-			action_ll_ptr == NULL ||
-			found == NULL)
-			goto fail;
-
-		for (i = 0; i < *n_rules; i++) {
-			match_ll_ptr[i] = (void *)&match_ll[i];
-			action_ll_ptr[i] =
-				(struct rte_pipeline_table_entry *)&action_ll[i * TABLE_RULE_ACTION_SIZE_MAX];
-		}
+		(table_id >= p->n_tables)) {
+		table_rule_list_free(list);
+		return -EINVAL;
+	}
 
-		/* Rule match conversion */
-		for (i = 0; i < *n_rules; i++) {
-			status = match_convert(&match[i], match_ll_ptr[i], 1);
-			if (status)
-				goto fail;
-		}
+	table = &p->table[table_id];
 
-		/* Rule action conversion */
-		for (i = 0; i < *n_rules; i++) {
-			status = action_convert(a, &action[i], action_ll_ptr[i]);
-			if (status)
-				goto fail;
+	TAILQ_FOREACH(rule, list, node)
+		if (match_check(&rule->match, p, table_id) ||
+			action_check(&rule->action, p, table_id)) {
+			table_rule_list_free(list);
+			return -EINVAL;
 		}
 
-		/* Add rule (match, action) to table */
-		if (bulk) {
-			status = rte_pipeline_table_entry_add_bulk(p->p,
-				table_id,
-				match_ll_ptr,
-				action_ll_ptr,
-				*n_rules,
-				found,
-				entries_ptr);
-			if (status)
-				*n_rules = 0;
-		} else {
-			for (i = 0; i < *n_rules; i++) {
-				status = rte_pipeline_table_entry_add(p->p,
-					table_id,
-					match_ll_ptr[i],
-					action_ll_ptr[i],
-					&found[i],
-					&entries_ptr[i]);
-				if (status) {
-					*n_rules = i;
-					break;
-				}
-			}
+	if (!pipeline_is_running(p)) {
+		struct table_ll table_ll = {
+			.p = p->p,
+			.table_id = table_id,
+			.a = table->a,
+			.bulk_supported = table->params.match_type == TABLE_ACL,
+		};
+
+		status = table_rule_add_bulk_ll(&table_ll, list, n_rules_added);
+		if (status) {
+			table_rule_list_free(list);
+			return status;
 		}
 
-		/* Free */
-		free(found);
-		free(action_ll_ptr);
-		free(match_ll_ptr);
-		free(action_ll);
-		free(match_ll);
-
-		return status;
-
-fail:
-		free(found);
-		free(action_ll_ptr);
-		free(match_ll_ptr);
-		free(action_ll);
-		free(match_ll);
-
-		*n_rules = 0;
-		return -1;
+		table_rule_add_bulk(table, list, *n_rules_added);
+		*n_rules_not_added = table_rule_list_free(list);
+		return 0;
 	}
 
 	/* Allocate request */
 	req = pipeline_msg_alloc();
-	if (req == NULL)
-		return -1;
+	if (req == NULL) {
+		table_rule_list_free(list);
+		return -ENOMEM;
+	}
 
 	/* Write request */
 	req->type = PIPELINE_REQ_TABLE_RULE_ADD_BULK;
 	req->id = table_id;
-	req->table_rule_add_bulk.match = match;
-	req->table_rule_add_bulk.action = action;
-	req->table_rule_add_bulk.data = data;
-	req->table_rule_add_bulk.n_rules = *n_rules;
-	req->table_rule_add_bulk.bulk =
-		(p->table[table_id].params.match_type == TABLE_ACL) ? 1 : 0;
+	req->table_rule_add_bulk.list = list;
+	req->table_rule_add_bulk.bulk = table->params.match_type == TABLE_ACL;
 
 	/* Send request and wait for response */
 	rsp = pipeline_msg_send_recv(p, req);
-	if (rsp == NULL)
-		return -1;
+	if (rsp == NULL) {
+		table_rule_list_free(list);
+		return -ENOMEM;
+	}
 
 	/* Read response */
 	status = rsp->status;
-	if (status == 0)
-		*n_rules = rsp->table_rule_add_bulk.n_rules;
+	if (status == 0) {
+		*n_rules_added = rsp->table_rule_add_bulk.n_rules;
+
+		table_rule_add_bulk(table, list, *n_rules_added);
+		*n_rules_not_added = table_rule_list_free(list);
+	} else
+		table_rule_list_free(list);
+
 
 	/* Free response */
 	pipeline_msg_free(rsp);
@@ -1550,6 +1683,7 @@ pipeline_table_rule_delete(const char *pipeline_name,
 	struct table_rule_match *match)
 {
 	struct pipeline *p;
+	struct table *table;
 	struct pipeline_msg_req *req;
 	struct pipeline_msg_rsp *rsp;
 	int status;
@@ -1565,6 +1699,8 @@ pipeline_table_rule_delete(const char *pipeline_name,
 		match_check(match, p, table_id))
 		return -1;
 
+	table = &p->table[table_id];
+
 	if (!pipeline_is_running(p)) {
 		union table_rule_match_low_level match_ll;
 		int key_found;
@@ -1579,6 +1715,9 @@ pipeline_table_rule_delete(const char *pipeline_name,
 				&key_found,
 				NULL);
 
+		if (status == 0)
+			table_rule_delete(table, match);
+
 		return status;
 	}
 
@@ -1599,6 +1738,8 @@ pipeline_table_rule_delete(const char *pipeline_name,
 
 	/* Read response */
 	status = rsp->status;
+	if (status == 0)
+		table_rule_delete(table, match);
 
 	/* Free response */
 	pipeline_msg_free(rsp);
@@ -1611,6 +1752,7 @@ pipeline_table_rule_delete_default(const char *pipeline_name,
 	uint32_t table_id)
 {
 	struct pipeline *p;
+	struct table *table;
 	struct pipeline_msg_req *req;
 	struct pipeline_msg_rsp *rsp;
 	int status;
@@ -1624,11 +1766,16 @@ pipeline_table_rule_delete_default(const char *pipeline_name,
 		(table_id >= p->n_tables))
 		return -1;
 
+	table = &p->table[table_id];
+
 	if (!pipeline_is_running(p)) {
 		status = rte_pipeline_table_default_entry_delete(p->p,
 			table_id,
 			NULL);
 
+		if (status == 0)
+			table_rule_default_delete(table);
+
 		return status;
 	}
 
@@ -1648,6 +1795,8 @@ pipeline_table_rule_delete_default(const char *pipeline_name,
 
 	/* Read response */
 	status = rsp->status;
+	if (status == 0)
+		table_rule_default_delete(table);
 
 	/* Free response */
 	pipeline_msg_free(rsp);
@@ -1658,31 +1807,37 @@ pipeline_table_rule_delete_default(const char *pipeline_name,
 int
 pipeline_table_rule_stats_read(const char *pipeline_name,
 	uint32_t table_id,
-	void *data,
+	struct table_rule_match *match,
 	struct rte_table_action_stats_counters *stats,
 	int clear)
 {
 	struct pipeline *p;
+	struct table *table;
 	struct pipeline_msg_req *req;
 	struct pipeline_msg_rsp *rsp;
+	struct table_rule *rule;
 	int status;
 
 	/* Check input params */
 	if ((pipeline_name == NULL) ||
-		(data == NULL) ||
+		(match == NULL) ||
 		(stats == NULL))
 		return -1;
 
 	p = pipeline_find(pipeline_name);
 	if ((p == NULL) ||
-		(table_id >= p->n_tables))
+		(table_id >= p->n_tables) ||
+		match_check(match, p, table_id))
 		return -1;
 
-	if (!pipeline_is_running(p)) {
-		struct rte_table_action *a = p->table[table_id].a;
+	table = &p->table[table_id];
+	rule = table_rule_find(table, match);
+	if (rule == NULL)
+		return -1;
 
-		status = rte_table_action_stats_read(a,
-			data,
+	if (!pipeline_is_running(p)) {
+		status = rte_table_action_stats_read(table->a,
+			rule->data,
 			stats,
 			clear);
 
@@ -1697,7 +1852,7 @@ pipeline_table_rule_stats_read(const char *pipeline_name,
 	/* Write request */
 	req->type = PIPELINE_REQ_TABLE_RULE_STATS_READ;
 	req->id = table_id;
-	req->table_rule_stats_read.data = data;
+	req->table_rule_stats_read.data = rule->data;
 	req->table_rule_stats_read.clear = clear;
 
 	/* Send request and wait for response */
@@ -1707,7 +1862,7 @@ pipeline_table_rule_stats_read(const char *pipeline_name,
 
 	/* Read response */
 	status = rsp->status;
-	if (status)
+	if (status == 0)
 		memcpy(stats, &rsp->table_rule_stats_read.stats, sizeof(*stats));
 
 	/* Free response */
@@ -1827,32 +1982,40 @@ pipeline_table_mtr_profile_delete(const char *pipeline_name,
 int
 pipeline_table_rule_mtr_read(const char *pipeline_name,
 	uint32_t table_id,
-	void *data,
-	uint32_t tc_mask,
+	struct table_rule_match *match,
 	struct rte_table_action_mtr_counters *stats,
 	int clear)
 {
 	struct pipeline *p;
+	struct table *table;
 	struct pipeline_msg_req *req;
 	struct pipeline_msg_rsp *rsp;
+	struct table_rule *rule;
+	uint32_t tc_mask;
 	int status;
 
 	/* Check input params */
 	if ((pipeline_name == NULL) ||
-		(data == NULL) ||
+		(match == NULL) ||
 		(stats == NULL))
 		return -1;
 
 	p = pipeline_find(pipeline_name);
 	if ((p == NULL) ||
-		(table_id >= p->n_tables))
+		(table_id >= p->n_tables) ||
+		match_check(match, p, table_id))
 		return -1;
 
-	if (!pipeline_is_running(p)) {
-		struct rte_table_action *a = p->table[table_id].a;
+	table = &p->table[table_id];
+	tc_mask = (1 << table->ap->params.mtr.n_tc) - 1;
+
+	rule = table_rule_find(table, match);
+	if (rule == NULL)
+		return -1;
 
-		status = rte_table_action_meter_read(a,
-				data,
+	if (!pipeline_is_running(p)) {
+		status = rte_table_action_meter_read(table->a,
+				rule->data,
 				tc_mask,
 				stats,
 				clear);
@@ -1868,7 +2031,7 @@ pipeline_table_rule_mtr_read(const char *pipeline_name,
 	/* Write request */
 	req->type = PIPELINE_REQ_TABLE_RULE_MTR_READ;
 	req->id = table_id;
-	req->table_rule_mtr_read.data = data;
+	req->table_rule_mtr_read.data = rule->data;
 	req->table_rule_mtr_read.tc_mask = tc_mask;
 	req->table_rule_mtr_read.clear = clear;
 
@@ -1879,7 +2042,7 @@ pipeline_table_rule_mtr_read(const char *pipeline_name,
 
 	/* Read response */
 	status = rsp->status;
-	if (status)
+	if (status == 0)
 		memcpy(stats, &rsp->table_rule_mtr_read.stats, sizeof(*stats));
 
 	/* Free response */
@@ -1948,31 +2111,40 @@ pipeline_table_dscp_table_update(const char *pipeline_name,
 int
 pipeline_table_rule_ttl_read(const char *pipeline_name,
 	uint32_t table_id,
-	void *data,
+	struct table_rule_match *match,
 	struct rte_table_action_ttl_counters *stats,
 	int clear)
 {
 	struct pipeline *p;
+	struct table *table;
 	struct pipeline_msg_req *req;
 	struct pipeline_msg_rsp *rsp;
+	struct table_rule *rule;
 	int status;
 
 	/* Check input params */
 	if ((pipeline_name == NULL) ||
-		(data == NULL) ||
+		(match == NULL) ||
 		(stats == NULL))
 		return -1;
 
 	p = pipeline_find(pipeline_name);
 	if ((p == NULL) ||
-		(table_id >= p->n_tables))
+		(table_id >= p->n_tables) ||
+		match_check(match, p, table_id))
 		return -1;
 
-	if (!pipeline_is_running(p)) {
-		struct rte_table_action *a = p->table[table_id].a;
+	table = &p->table[table_id];
+	if (!table->ap->params.ttl.n_packets_enabled)
+		return -1;
 
-		status = rte_table_action_ttl_read(a,
-				data,
+	rule = table_rule_find(table, match);
+	if (rule == NULL)
+		return -1;
+
+	if (!pipeline_is_running(p)) {
+		status = rte_table_action_ttl_read(table->a,
+				rule->data,
 				stats,
 				clear);
 
@@ -1987,7 +2159,7 @@ pipeline_table_rule_ttl_read(const char *pipeline_name,
 	/* Write request */
 	req->type = PIPELINE_REQ_TABLE_RULE_TTL_READ;
 	req->id = table_id;
-	req->table_rule_ttl_read.data = data;
+	req->table_rule_ttl_read.data = rule->data;
 	req->table_rule_ttl_read.clear = clear;
 
 	/* Send request and wait for response */
@@ -1997,7 +2169,7 @@ pipeline_table_rule_ttl_read(const char *pipeline_name,
 
 	/* Read response */
 	status = rsp->status;
-	if (status)
+	if (status == 0)
 		memcpy(stats, &rsp->table_rule_ttl_read.stats, sizeof(*stats));
 
 	/* Free response */
@@ -2006,6 +2178,71 @@ pipeline_table_rule_ttl_read(const char *pipeline_name,
 	return status;
 }
 
+int
+pipeline_table_rule_time_read(const char *pipeline_name,
+	uint32_t table_id,
+	struct table_rule_match *match,
+	uint64_t *timestamp)
+{
+	struct pipeline *p;
+	struct table *table;
+	struct pipeline_msg_req *req;
+	struct pipeline_msg_rsp *rsp;
+	struct table_rule *rule;
+	int status;
+
+	/* Check input params */
+	if ((pipeline_name == NULL) ||
+		(match == NULL) ||
+		(timestamp == NULL))
+		return -1;
+
+	p = pipeline_find(pipeline_name);
+	if ((p == NULL) ||
+		(table_id >= p->n_tables) ||
+		match_check(match, p, table_id))
+		return -1;
+
+	table = &p->table[table_id];
+
+	rule = table_rule_find(table, match);
+	if (rule == NULL)
+		return -1;
+
+	if (!pipeline_is_running(p)) {
+		status = rte_table_action_time_read(table->a,
+				rule->data,
+				timestamp);
+
+		return status;
+	}
+
+	/* Allocate request */
+	req = pipeline_msg_alloc();
+	if (req == NULL)
+		return -1;
+
+	/* Write request */
+	req->type = PIPELINE_REQ_TABLE_RULE_TIME_READ;
+	req->id = table_id;
+	req->table_rule_time_read.data = rule->data;
+
+	/* Send request and wait for response */
+	rsp = pipeline_msg_send_recv(p, req);
+	if (rsp == NULL)
+		return -1;
+
+	/* Read response */
+	status = rsp->status;
+	if (status == 0)
+		*timestamp = rsp->table_rule_time_read.timestamp;
+
+	/* Free response */
+	pipeline_msg_free(rsp);
+
+	return status;
+}
+
 /**
  * Data plane threads: message handling
  */
@@ -2244,29 +2481,37 @@ match_convert(struct table_rule_match *mh,
 				ml->acl_add.field_value[0].mask_range.u8 =
 					mh->match.acl.proto_mask;
 
-				ml->acl_add.field_value[1].value.u32 = sa32[0];
+				ml->acl_add.field_value[1].value.u32 =
+					rte_be_to_cpu_32(sa32[0]);
 				ml->acl_add.field_value[1].mask_range.u32 =
 					sa32_depth[0];
-				ml->acl_add.field_value[2].value.u32 = sa32[1];
+				ml->acl_add.field_value[2].value.u32 =
+					rte_be_to_cpu_32(sa32[1]);
 				ml->acl_add.field_value[2].mask_range.u32 =
 					sa32_depth[1];
-				ml->acl_add.field_value[3].value.u32 = sa32[2];
+				ml->acl_add.field_value[3].value.u32 =
+					rte_be_to_cpu_32(sa32[2]);
 				ml->acl_add.field_value[3].mask_range.u32 =
 					sa32_depth[2];
-				ml->acl_add.field_value[4].value.u32 = sa32[3];
+				ml->acl_add.field_value[4].value.u32 =
+					rte_be_to_cpu_32(sa32[3]);
 				ml->acl_add.field_value[4].mask_range.u32 =
 					sa32_depth[3];
 
-				ml->acl_add.field_value[5].value.u32 = da32[0];
+				ml->acl_add.field_value[5].value.u32 =
+					rte_be_to_cpu_32(da32[0]);
 				ml->acl_add.field_value[5].mask_range.u32 =
 					da32_depth[0];
-				ml->acl_add.field_value[6].value.u32 = da32[1];
+				ml->acl_add.field_value[6].value.u32 =
+					rte_be_to_cpu_32(da32[1]);
 				ml->acl_add.field_value[6].mask_range.u32 =
 					da32_depth[1];
-				ml->acl_add.field_value[7].value.u32 = da32[2];
+				ml->acl_add.field_value[7].value.u32 =
+					rte_be_to_cpu_32(da32[2]);
 				ml->acl_add.field_value[7].mask_range.u32 =
 					da32_depth[2];
-				ml->acl_add.field_value[8].value.u32 = da32[3];
+				ml->acl_add.field_value[8].value.u32 =
+					rte_be_to_cpu_32(da32[3]);
 				ml->acl_add.field_value[8].mask_range.u32 =
 					da32_depth[3];
 
@@ -2308,36 +2553,36 @@ match_convert(struct table_rule_match *mh,
 					mh->match.acl.proto_mask;
 
 				ml->acl_delete.field_value[1].value.u32 =
-					sa32[0];
+					rte_be_to_cpu_32(sa32[0]);
 				ml->acl_delete.field_value[1].mask_range.u32 =
 					sa32_depth[0];
 				ml->acl_delete.field_value[2].value.u32 =
-					sa32[1];
+					rte_be_to_cpu_32(sa32[1]);
 				ml->acl_delete.field_value[2].mask_range.u32 =
 					sa32_depth[1];
 				ml->acl_delete.field_value[3].value.u32 =
-					sa32[2];
+					rte_be_to_cpu_32(sa32[2]);
 				ml->acl_delete.field_value[3].mask_range.u32 =
 					sa32_depth[2];
 				ml->acl_delete.field_value[4].value.u32 =
-					sa32[3];
+					rte_be_to_cpu_32(sa32[3]);
 				ml->acl_delete.field_value[4].mask_range.u32 =
 					sa32_depth[3];
 
 				ml->acl_delete.field_value[5].value.u32 =
-					da32[0];
+					rte_be_to_cpu_32(da32[0]);
 				ml->acl_delete.field_value[5].mask_range.u32 =
 					da32_depth[0];
 				ml->acl_delete.field_value[6].value.u32 =
-					da32[1];
+					rte_be_to_cpu_32(da32[1]);
 				ml->acl_delete.field_value[6].mask_range.u32 =
 					da32_depth[1];
 				ml->acl_delete.field_value[7].value.u32 =
-					da32[2];
+					rte_be_to_cpu_32(da32[2]);
 				ml->acl_delete.field_value[7].mask_range.u32 =
 					da32_depth[2];
 				ml->acl_delete.field_value[8].value.u32 =
-					da32[3];
+					rte_be_to_cpu_32(da32[3]);
 				ml->acl_delete.field_value[8].mask_range.u32 =
 					da32_depth[3];
 
@@ -2476,6 +2721,36 @@ action_convert(struct rte_table_action *a,
 			return status;
 	}
 
+	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_SYM_CRYPTO)) {
+		status = rte_table_action_apply(a,
+			data,
+			RTE_TABLE_ACTION_SYM_CRYPTO,
+			&action->sym_crypto);
+
+		if (status)
+			return status;
+	}
+
+	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_TAG)) {
+		status = rte_table_action_apply(a,
+			data,
+			RTE_TABLE_ACTION_TAG,
+			&action->tag);
+
+		if (status)
+			return status;
+	}
+
+	if (action->action_mask & (1LLU << RTE_TABLE_ACTION_DECAP)) {
+		status = rte_table_action_apply(a,
+			data,
+			RTE_TABLE_ACTION_DECAP,
+			&action->decap);
+
+		if (status)
+			return status;
+	}
+
 	return 0;
 }
 
@@ -2567,107 +2842,32 @@ static struct pipeline_msg_rsp *
 pipeline_msg_handle_table_rule_add_bulk(struct pipeline_data *p,
 	struct pipeline_msg_req *req)
 {
-
 	struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
 
 	uint32_t table_id = req->id;
-	struct table_rule_match *match = req->table_rule_add_bulk.match;
-	struct table_rule_action *action = req->table_rule_add_bulk.action;
-	struct rte_pipeline_table_entry **data =
-		(struct rte_pipeline_table_entry **)req->table_rule_add_bulk.data;
-	uint32_t n_rules = req->table_rule_add_bulk.n_rules;
+	struct table_rule_list *list = req->table_rule_add_bulk.list;
 	uint32_t bulk = req->table_rule_add_bulk.bulk;
 
-	struct rte_table_action *a = p->table_data[table_id].a;
-	union table_rule_match_low_level *match_ll;
-	uint8_t *action_ll;
-	void **match_ll_ptr;
-	struct rte_pipeline_table_entry **action_ll_ptr;
-	int *found, status;
-	uint32_t i;
-
-	/* Memory allocation */
-	match_ll = calloc(n_rules, sizeof(union table_rule_match_low_level));
-	action_ll = calloc(n_rules, TABLE_RULE_ACTION_SIZE_MAX);
-	match_ll_ptr = calloc(n_rules, sizeof(void *));
-	action_ll_ptr =
-		calloc(n_rules, sizeof(struct rte_pipeline_table_entry *));
-	found = calloc(n_rules, sizeof(int));
-
-	if ((match_ll == NULL) ||
-		(action_ll == NULL) ||
-		(match_ll_ptr == NULL) ||
-		(action_ll_ptr == NULL) ||
-		(found == NULL))
-		goto fail;
-
-	for (i = 0; i < n_rules; i++) {
-		match_ll_ptr[i] = (void *)&match_ll[i];
-		action_ll_ptr[i] =
-			(struct rte_pipeline_table_entry *)&action_ll[i * TABLE_RULE_ACTION_SIZE_MAX];
-	}
+	uint32_t n_rules_added;
+	int status;
 
-	/* Rule match conversion */
-	for (i = 0; i < n_rules; i++) {
-		status = match_convert(&match[i], match_ll_ptr[i], 1);
-		if (status)
-			goto fail;
-	}
+	struct table_ll table_ll = {
+		.p = p->p,
+		.table_id = table_id,
+		.a = p->table_data[table_id].a,
+		.bulk_supported = bulk,
+	};
 
-	/* Rule action conversion */
-	for (i = 0; i < n_rules; i++) {
-		status = action_convert(a, &action[i], action_ll_ptr[i]);
-		if (status)
-			goto fail;
+	status = table_rule_add_bulk_ll(&table_ll, list, &n_rules_added);
+	if (status) {
+		rsp->status = -1;
+		rsp->table_rule_add_bulk.n_rules = 0;
+		return rsp;
 	}
 
-	/* Add rule (match, action) to table */
-	if (bulk) {
-		status = rte_pipeline_table_entry_add_bulk(p->p,
-			table_id,
-			match_ll_ptr,
-			action_ll_ptr,
-			n_rules,
-			found,
-			data);
-		if (status)
-			n_rules = 0;
-	} else
-		for (i = 0; i < n_rules; i++) {
-			status = rte_pipeline_table_entry_add(p->p,
-				table_id,
-				match_ll_ptr[i],
-				action_ll_ptr[i],
-				&found[i],
-				&data[i]);
-			if (status) {
-				n_rules = i;
-				break;
-			}
-		}
-
 	/* Write response */
 	rsp->status = 0;
-	rsp->table_rule_add_bulk.n_rules = n_rules;
-
-	/* Free */
-	free(found);
-	free(action_ll_ptr);
-	free(match_ll_ptr);
-	free(action_ll);
-	free(match_ll);
-
-	return rsp;
-
-fail:
-	free(found);
-	free(action_ll_ptr);
-	free(match_ll_ptr);
-	free(action_ll);
-	free(match_ll);
-
-	rsp->status = -1;
-	rsp->table_rule_add_bulk.n_rules = 0;
+	rsp->table_rule_add_bulk.n_rules = n_rules_added;
 	return rsp;
 }
 
@@ -2818,6 +3018,22 @@ pipeline_msg_handle_table_rule_ttl_read(struct pipeline_data *p,
 	return rsp;
 }
 
+static struct pipeline_msg_rsp *
+pipeline_msg_handle_table_rule_time_read(struct pipeline_data *p,
+	struct pipeline_msg_req *req)
+{
+	struct pipeline_msg_rsp *rsp = (struct pipeline_msg_rsp *) req;
+	uint32_t table_id = req->id;
+	void *data = req->table_rule_time_read.data;
+	struct rte_table_action *a = p->table_data[table_id].a;
+
+	rsp->status = rte_table_action_time_read(a,
+		data,
+		&rsp->table_rule_time_read.timestamp);
+
+	return rsp;
+}
+
 static void
 pipeline_msg_handle(struct pipeline_data *p)
 {
@@ -2894,6 +3110,10 @@ pipeline_msg_handle(struct pipeline_data *p)
 			rsp = pipeline_msg_handle_table_rule_ttl_read(p, req);
 			break;
 
+		case PIPELINE_REQ_TABLE_RULE_TIME_READ:
+			rsp = pipeline_msg_handle_table_rule_time_read(p, req);
+			break;
+
 		default:
 			rsp = (struct pipeline_msg_rsp *) req;
 			rsp->status = -1;
diff --git a/examples/ip_reassembly/main.c b/examples/ip_reassembly/main.c
index b830f67a..17b55d4c 100644
--- a/examples/ip_reassembly/main.c
+++ b/examples/ip_reassembly/main.c
@@ -165,8 +165,7 @@ static struct rte_eth_conf port_conf = {
 		.max_rx_pkt_len = JUMBO_FRAME_MAX_SIZE,
 		.split_hdr_size = 0,
 		.offloads = (DEV_RX_OFFLOAD_CHECKSUM |
-			     DEV_RX_OFFLOAD_JUMBO_FRAME |
-			     DEV_RX_OFFLOAD_CRC_STRIP),
+			     DEV_RX_OFFLOAD_JUMBO_FRAME),
 	},
 	.rx_adv_conf = {
 			.rss_conf = {
diff --git a/examples/ipsec-secgw/esp.c b/examples/ipsec-secgw/esp.c
index ee9e590a..e33232c9 100644
--- a/examples/ipsec-secgw/esp.c
+++ b/examples/ipsec-secgw/esp.c
@@ -96,6 +96,7 @@ esp_inbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 
 		switch (sa->cipher_algo) {
 		case RTE_CRYPTO_CIPHER_NULL:
+		case RTE_CRYPTO_CIPHER_3DES_CBC:
 		case RTE_CRYPTO_CIPHER_AES_CBC:
 			/* Copy IV at the end of crypto operation */
 			rte_memcpy(iv_ptr, iv, sa->iv_len);
@@ -326,6 +327,7 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	} else {
 		switch (sa->cipher_algo) {
 		case RTE_CRYPTO_CIPHER_NULL:
+		case RTE_CRYPTO_CIPHER_3DES_CBC:
 		case RTE_CRYPTO_CIPHER_AES_CBC:
 			memset(iv, 0, sa->iv_len);
 			break;
@@ -387,6 +389,7 @@ esp_outbound(struct rte_mbuf *m, struct ipsec_sa *sa,
 	} else {
 		switch (sa->cipher_algo) {
 		case RTE_CRYPTO_CIPHER_NULL:
+		case RTE_CRYPTO_CIPHER_3DES_CBC:
 		case RTE_CRYPTO_CIPHER_AES_CBC:
 			sym_cop->cipher.data.offset = ip_hdr_len +
 				sizeof(struct esp_hdr);
diff --git a/examples/ipsec-secgw/ipsec-secgw.c b/examples/ipsec-secgw/ipsec-secgw.c
index b45b87bd..1bc0b5b5 100644
--- a/examples/ipsec-secgw/ipsec-secgw.c
+++ b/examples/ipsec-secgw/ipsec-secgw.c
@@ -54,7 +54,7 @@
 #define NB_MBUF	(32000)
 
 #define CDEV_QUEUE_DESC 2048
-#define CDEV_MAP_ENTRIES 1024
+#define CDEV_MAP_ENTRIES 16384
 #define CDEV_MP_NB_OBJS 2048
 #define CDEV_MP_CACHE_SZ 64
 #define MAX_QUEUE_PAIRS 1
@@ -197,8 +197,7 @@ static struct rte_eth_conf port_conf = {
 		.mq_mode	= ETH_MQ_RX_RSS,
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.offloads = DEV_RX_OFFLOAD_CHECKSUM |
-			    DEV_RX_OFFLOAD_CRC_STRIP,
+		.offloads = DEV_RX_OFFLOAD_CHECKSUM,
 	},
 	.rx_adv_conf = {
 		.rss_conf = {
@@ -1392,9 +1391,27 @@ cryptodevs_init(void)
 
 	uint32_t max_sess_sz = 0, sess_sz;
 	for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) {
+		void *sec_ctx;
+
+		/* Get crypto priv session size */
 		sess_sz = rte_cryptodev_sym_get_private_session_size(cdev_id);
 		if (sess_sz > max_sess_sz)
 			max_sess_sz = sess_sz;
+
+		/*
+		 * If crypto device is security capable, need to check the
+		 * size of security session as well.
+		 */
+
+		/* Get security context of the crypto device */
+		sec_ctx = rte_cryptodev_get_sec_ctx(cdev_id);
+		if (sec_ctx == NULL)
+			continue;
+
+		/* Get size of security session */
+		sess_sz = rte_security_session_get_size(sec_ctx);
+		if (sess_sz > max_sess_sz)
+			max_sess_sz = sess_sz;
 	}
 	RTE_ETH_FOREACH_DEV(port_id) {
 		void *sec_ctx;
diff --git a/examples/ipsec-secgw/sa.c b/examples/ipsec-secgw/sa.c
index 4ab8e098..d2d3550a 100644
--- a/examples/ipsec-secgw/sa.c
+++ b/examples/ipsec-secgw/sa.c
@@ -81,6 +81,13 @@ const struct supported_cipher_algo cipher_algos[] = {
 		.iv_len = 8,
 		.block_size = 16, /* XXX AESNI MB limition, should be 4 */
 		.key_len = 20
+	},
+	{
+		.keyword = "3des-cbc",
+		.algo = RTE_CRYPTO_CIPHER_3DES_CBC,
+		.iv_len = 8,
+		.block_size = 8,
+		.key_len = 24
 	}
 };
 
@@ -327,7 +334,8 @@ parse_sa_tokens(char **tokens, uint32_t n_tokens,
 			if (status->status < 0)
 				return;
 
-			if (algo->algo == RTE_CRYPTO_CIPHER_AES_CBC)
+			if (algo->algo == RTE_CRYPTO_CIPHER_AES_CBC ||
+				algo->algo == RTE_CRYPTO_CIPHER_3DES_CBC)
 				rule->salt = (uint32_t)rte_rand();
 
 			if (algo->algo == RTE_CRYPTO_CIPHER_AES_CTR) {
@@ -810,6 +818,7 @@ sa_add_rules(struct sa_ctx *sa_ctx, const struct ipsec_sa entries[],
 		} else {
 			switch (sa->cipher_algo) {
 			case RTE_CRYPTO_CIPHER_NULL:
+			case RTE_CRYPTO_CIPHER_3DES_CBC:
 			case RTE_CRYPTO_CIPHER_AES_CBC:
 				iv_length = sa->iv_len;
 				break;
diff --git a/examples/ipv4_multicast/main.c b/examples/ipv4_multicast/main.c
index 331c32e7..4073a490 100644
--- a/examples/ipv4_multicast/main.c
+++ b/examples/ipv4_multicast/main.c
@@ -109,8 +109,7 @@ static struct rte_eth_conf port_conf = {
 	.rxmode = {
 		.max_rx_pkt_len = JUMBO_FRAME_MAX_SIZE,
 		.split_hdr_size = 0,
-		.offloads = (DEV_RX_OFFLOAD_JUMBO_FRAME |
-			     DEV_RX_OFFLOAD_CRC_STRIP),
+		.offloads = DEV_RX_OFFLOAD_JUMBO_FRAME,
 	},
 	.txmode = {
 		.mq_mode = ETH_MQ_TX_NONE,
@@ -774,7 +773,7 @@ main(int argc, char **argv)
 			qconf->tx_queue_id[portid] = queueid;
 			queueid++;
 		}
-
+		rte_eth_allmulticast_enable(portid);
 		/* Start device */
 		ret = rte_eth_dev_start(portid);
 		if (ret < 0)
diff --git a/examples/kni/Makefile b/examples/kni/Makefile
index 7e19d2e2..dd90d7d7 100644
--- a/examples/kni/Makefile
+++ b/examples/kni/Makefile
@@ -20,6 +20,7 @@ static: build/$(APP)-static
 
 PC_FILE := $(shell pkg-config --path libdpdk)
 CFLAGS += -O3 $(shell pkg-config --cflags libdpdk)
+CFLAGS += -DALLOW_EXPERIMENTAL_API
 LDFLAGS_SHARED = $(shell pkg-config --libs libdpdk)
 LDFLAGS_STATIC = -Wl,-Bstatic $(shell pkg-config --static --libs libdpdk)
 
@@ -54,6 +55,7 @@ please change the definition of the RTE_TARGET environment variable)
 endif
 
 CFLAGS += -O3
+CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += $(WERROR_FLAGS)
 
 include $(RTE_SDK)/mk/rte.extapp.mk
diff --git a/examples/kni/main.c b/examples/kni/main.c
index 81336087..e37b1ad3 100644
--- a/examples/kni/main.c
+++ b/examples/kni/main.c
@@ -94,9 +94,6 @@ static struct kni_port_params *kni_port_params_array[RTE_MAX_ETHPORTS];
 
 /* Options for configuring ethernet port */
 static struct rte_eth_conf port_conf = {
-	.rxmode = {
-		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
-	},
 	.txmode = {
 		.mq_mode = ETH_MQ_TX_NONE,
 	},
@@ -109,6 +106,8 @@ static struct rte_mempool * pktmbuf_pool = NULL;
 static uint32_t ports_mask = 0;
 /* Ports set in promiscuous mode off by default. */
 static int promiscuous_on = 0;
+/* Monitor link status continually. off by default. */
+static int monitor_links;
 
 /* Structure type for recording kni interface specific stats */
 struct kni_interface_stats {
@@ -172,14 +171,13 @@ signal_handler(int signum)
 	/* When we receive a USR2 signal, reset stats */
 	if (signum == SIGUSR2) {
 		memset(&kni_stats, 0, sizeof(kni_stats));
-		printf("\n**Statistics have been reset**\n");
+		printf("\n** Statistics have been reset **\n");
 		return;
 	}
 
 	/* When we receive a RTMIN or SIGINT signal, stop kni processing */
 	if (signum == SIGRTMIN || signum == SIGINT){
-		printf("SIGRTMIN is received, and the KNI processing is "
-							"going to stop\n");
+		printf("\nSIGRTMIN/SIGINT received. KNI processing stopping.\n");
 		rte_atomic32_inc(&kni_stop);
 		return;
         }
@@ -225,7 +223,8 @@ kni_ingress(struct kni_port_params *p)
 		}
 		/* Burst tx to kni */
 		num = rte_kni_tx_burst(p->kni[i], pkts_burst, nb_rx);
-		kni_stats[port_id].rx_packets += num;
+		if (num)
+			kni_stats[port_id].rx_packets += num;
 
 		rte_kni_handle_request(p->kni[i]);
 		if (unlikely(num < nb_rx)) {
@@ -262,7 +261,8 @@ kni_egress(struct kni_port_params *p)
 		}
 		/* Burst tx to eth */
 		nb_tx = rte_eth_tx_burst(port_id, 0, pkts_burst, (uint16_t)num);
-		kni_stats[port_id].tx_packets += nb_tx;
+		if (nb_tx)
+			kni_stats[port_id].tx_packets += nb_tx;
 		if (unlikely(nb_tx < num)) {
 			/* Free mbufs not tx to NIC */
 			kni_burst_free_mbufs(&pkts_burst[nb_tx], num - nb_tx);
@@ -328,11 +328,12 @@ main_loop(__rte_unused void *arg)
 static void
 print_usage(const char *prgname)
 {
-	RTE_LOG(INFO, APP, "\nUsage: %s [EAL options] -- -p PORTMASK -P "
+	RTE_LOG(INFO, APP, "\nUsage: %s [EAL options] -- -p PORTMASK -P -m "
 		   "[--config (port,lcore_rx,lcore_tx,lcore_kthread...)"
 		   "[,(port,lcore_rx,lcore_tx,lcore_kthread...)]]\n"
 		   "    -p PORTMASK: hex bitmask of ports to use\n"
 		   "    -P : enable promiscuous mode\n"
+		   "    -m : enable monitoring of port carrier state\n"
 		   "    --config (port,lcore_rx,lcore_tx,lcore_kthread...): "
 		   "port and lcore configurations\n",
 	           prgname);
@@ -513,7 +514,7 @@ parse_args(int argc, char **argv)
 	opterr = 0;
 
 	/* Parse command line */
-	while ((opt = getopt_long(argc, argv, "p:P", longopts,
+	while ((opt = getopt_long(argc, argv, "p:Pm", longopts,
 						&longindex)) != EOF) {
 		switch (opt) {
 		case 'p':
@@ -522,6 +523,9 @@ parse_args(int argc, char **argv)
 		case 'P':
 			promiscuous_on = 1;
 			break;
+		case 'm':
+			monitor_links = 1;
+			break;
 		case 0:
 			if (!strncmp(longopts[longindex].name,
 				     CMDLINE_OPT_CONFIG,
@@ -677,6 +681,55 @@ check_all_ports_link_status(uint32_t port_mask)
 	}
 }
 
+static void
+log_link_state(struct rte_kni *kni, int prev, struct rte_eth_link *link)
+{
+	if (kni == NULL || link == NULL)
+		return;
+
+	if (prev == ETH_LINK_DOWN && link->link_status == ETH_LINK_UP) {
+		RTE_LOG(INFO, APP, "%s NIC Link is Up %d Mbps %s %s.\n",
+			rte_kni_get_name(kni),
+			link->link_speed,
+			link->link_autoneg ?  "(AutoNeg)" : "(Fixed)",
+			link->link_duplex ?  "Full Duplex" : "Half Duplex");
+	} else if (prev == ETH_LINK_UP && link->link_status == ETH_LINK_DOWN) {
+		RTE_LOG(INFO, APP, "%s NIC Link is Down.\n",
+			rte_kni_get_name(kni));
+	}
+}
+
+/*
+ * Monitor the link status of all ports and update the
+ * corresponding KNI interface(s)
+ */
+static void *
+monitor_all_ports_link_status(void *arg)
+{
+	uint16_t portid;
+	struct rte_eth_link link;
+	unsigned int i;
+	struct kni_port_params **p = kni_port_params_array;
+	int prev;
+	(void) arg;
+
+	while (monitor_links) {
+		rte_delay_ms(500);
+		RTE_ETH_FOREACH_DEV(portid) {
+			if ((ports_mask & (1 << portid)) == 0)
+				continue;
+			memset(&link, 0, sizeof(link));
+			rte_eth_link_get_nowait(portid, &link);
+			for (i = 0; i < p[portid]->nb_kni; i++) {
+				prev = rte_kni_update_link(p[portid]->kni[i],
+						link.link_status);
+				log_link_state(p[portid]->kni[i], prev, &link);
+			}
+		}
+	}
+	return NULL;
+}
+
 /* Callback for request of changing MTU */
 static int
 kni_change_mtu(uint16_t port_id, unsigned int new_mtu)
@@ -896,6 +949,9 @@ main(int argc, char** argv)
 	int ret;
 	uint16_t nb_sys_ports, port;
 	unsigned i;
+	void *retval;
+	pthread_t kni_link_tid;
+	int pid;
 
 	/* Associate signal_hanlder function with USR signals */
 	signal(SIGUSR1, signal_handler);
@@ -952,12 +1008,31 @@ main(int argc, char** argv)
 	}
 	check_all_ports_link_status(ports_mask);
 
+	pid = getpid();
+	RTE_LOG(INFO, APP, "========================\n");
+	RTE_LOG(INFO, APP, "KNI Running\n");
+	RTE_LOG(INFO, APP, "kill -SIGUSR1 %d\n", pid);
+	RTE_LOG(INFO, APP, "    Show KNI Statistics.\n");
+	RTE_LOG(INFO, APP, "kill -SIGUSR2 %d\n", pid);
+	RTE_LOG(INFO, APP, "    Zero KNI Statistics.\n");
+	RTE_LOG(INFO, APP, "========================\n");
+	fflush(stdout);
+
+	ret = rte_ctrl_thread_create(&kni_link_tid,
+				     "KNI link status check", NULL,
+				     monitor_all_ports_link_status, NULL);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE,
+			"Could not create link status thread!\n");
+
 	/* Launch per-lcore function on every lcore */
 	rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER);
 	RTE_LCORE_FOREACH_SLAVE(i) {
 		if (rte_eal_wait_lcore(i) < 0)
 			return -1;
 	}
+	monitor_links = 0;
+	pthread_join(kni_link_tid, &retval);
 
 	/* Release resources */
 	RTE_ETH_FOREACH_DEV(port) {
diff --git a/examples/kni/meson.build b/examples/kni/meson.build
index 79131639..fd6ae444 100644
--- a/examples/kni/meson.build
+++ b/examples/kni/meson.build
@@ -12,3 +12,4 @@ deps += ['kni', 'bus_pci']
 sources = files(
 	'main.c'
 )
+allow_experimental_apis = true
diff --git a/examples/l2fwd-cat/Makefile b/examples/l2fwd-cat/Makefile
index aec770c2..b6eeabde 100644
--- a/examples/l2fwd-cat/Makefile
+++ b/examples/l2fwd-cat/Makefile
@@ -23,7 +23,6 @@ CFLAGS += -O3 $(shell pkg-config --cflags libdpdk)
 LDFLAGS_SHARED = $(shell pkg-config --libs libdpdk)
 LDFLAGS_STATIC = -Wl,-Bstatic $(shell pkg-config --static --libs libdpdk)
 
-CFLAGS += -D_GNU_SOURCE
 LDFLAGS += -lpqos
 
 build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build
@@ -66,7 +65,6 @@ endif
 EXTRA_CFLAGS += -O3 -g -Wfatal-errors
 
 CFLAGS += -I$(PQOS_INSTALL_PATH)/../include
-CFLAGS_cat.o := -D_GNU_SOURCE
 
 LDLIBS += -L$(PQOS_INSTALL_PATH)
 LDLIBS += -lpqos
diff --git a/examples/l2fwd-cat/meson.build b/examples/l2fwd-cat/meson.build
index 1234e7b5..4e2777a0 100644
--- a/examples/l2fwd-cat/meson.build
+++ b/examples/l2fwd-cat/meson.build
@@ -9,7 +9,6 @@
 pqos = cc.find_library('pqos', required: false)
 build = pqos.found()
 ext_deps += pqos
-cflags += '-D_GNU_SOURCE'
 cflags += '-I/usr/local/include' # assume pqos lib installed in /usr/local
 sources = files(
 	'cat.c', 'l2fwd-cat.c'
diff --git a/examples/l2fwd-crypto/main.c b/examples/l2fwd-crypto/main.c
index 6061b751..f12fd266 100644
--- a/examples/l2fwd-crypto/main.c
+++ b/examples/l2fwd-crypto/main.c
@@ -213,7 +213,6 @@ static struct rte_eth_conf port_conf = {
 		.mq_mode = ETH_MQ_RX_NONE,
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.txmode = {
 		.mq_mode = ETH_MQ_TX_NONE,
diff --git a/examples/l2fwd-jobstats/main.c b/examples/l2fwd-jobstats/main.c
index af542338..a4d28e17 100644
--- a/examples/l2fwd-jobstats/main.c
+++ b/examples/l2fwd-jobstats/main.c
@@ -90,7 +90,6 @@ struct rte_eth_dev_tx_buffer *tx_buffer[RTE_MAX_ETHPORTS];
 static struct rte_eth_conf port_conf = {
 	.rxmode = {
 		.split_hdr_size = 0,
-		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.txmode = {
 		.mq_mode = ETH_MQ_TX_NONE,
diff --git a/examples/l2fwd-keepalive/main.c b/examples/l2fwd-keepalive/main.c
index 2d8b4d1c..0bf2b533 100644
--- a/examples/l2fwd-keepalive/main.c
+++ b/examples/l2fwd-keepalive/main.c
@@ -81,7 +81,6 @@ struct rte_eth_dev_tx_buffer *tx_buffer[RTE_MAX_ETHPORTS];
 static struct rte_eth_conf port_conf = {
 	.rxmode = {
 		.split_hdr_size = 0,
-		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.txmode = {
 		.mq_mode = ETH_MQ_TX_NONE,
diff --git a/examples/l2fwd/main.c b/examples/l2fwd/main.c
index 9bb4c5bc..6c23215a 100644
--- a/examples/l2fwd/main.c
+++ b/examples/l2fwd/main.c
@@ -82,7 +82,6 @@ static struct rte_eth_dev_tx_buffer *tx_buffer[RTE_MAX_ETHPORTS];
 static struct rte_eth_conf port_conf = {
 	.rxmode = {
 		.split_hdr_size = 0,
-		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.txmode = {
 		.mq_mode = ETH_MQ_TX_NONE,
diff --git a/examples/l3fwd-acl/main.c b/examples/l3fwd-acl/main.c
index 7c063a8d..a322ce4f 100644
--- a/examples/l3fwd-acl/main.c
+++ b/examples/l3fwd-acl/main.c
@@ -127,8 +127,7 @@ static struct rte_eth_conf port_conf = {
 		.mq_mode	= ETH_MQ_RX_RSS,
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.offloads = (DEV_RX_OFFLOAD_CRC_STRIP |
-			     DEV_RX_OFFLOAD_CHECKSUM),
+		.offloads = DEV_RX_OFFLOAD_CHECKSUM,
 	},
 	.rx_adv_conf = {
 		.rss_conf = {
diff --git a/examples/l3fwd-power/Makefile b/examples/l3fwd-power/Makefile
index d7e39a34..772ec7ba 100644
--- a/examples/l3fwd-power/Makefile
+++ b/examples/l3fwd-power/Makefile
@@ -23,6 +23,8 @@ CFLAGS += -O3 $(shell pkg-config --cflags libdpdk)
 LDFLAGS_SHARED = $(shell pkg-config --libs libdpdk)
 LDFLAGS_STATIC = -Wl,-Bstatic $(shell pkg-config --static --libs libdpdk)
 
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+
 build/$(APP)-shared: $(SRCS-y) Makefile $(PC_FILE) | build
 	$(CC) $(CFLAGS) $(SRCS-y) -o $@ $(LDFLAGS) $(LDFLAGS_SHARED)
 
@@ -54,6 +56,7 @@ please change the definition of the RTE_TARGET environment variable)
 all:
 else
 
+CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
 
diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
index d15cd520..0b3f8fe6 100644
--- a/examples/l3fwd-power/main.c
+++ b/examples/l3fwd-power/main.c
@@ -43,6 +43,7 @@
 #include <rte_timer.h>
 #include <rte_power.h>
 #include <rte_spinlock.h>
+#include <rte_power_empty_poll.h>
 
 #include "perf_core.h"
 #include "main.h"
@@ -55,6 +56,8 @@
 
 /* 100 ms interval */
 #define TIMER_NUMBER_PER_SECOND           10
+/* (10ms) */
+#define INTERVALS_PER_SECOND             100
 /* 100000 us */
 #define SCALING_PERIOD                    (1000000/TIMER_NUMBER_PER_SECOND)
 #define SCALING_DOWN_TIME_RATIO_THRESHOLD 0.25
@@ -117,6 +120,17 @@
  */
 #define RTE_TEST_RX_DESC_DEFAULT 1024
 #define RTE_TEST_TX_DESC_DEFAULT 1024
+
+/*
+ * These two thresholds were decided on by running the training algorithm on
+ * a 2.5GHz Xeon. These defaults can be overridden by supplying non-zero values
+ * for the med_threshold and high_threshold parameters on the command line.
+ */
+#define EMPTY_POLL_MED_THRESHOLD 350000UL
+#define EMPTY_POLL_HGH_THRESHOLD 580000UL
+
+
+
 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT;
 static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT;
 
@@ -132,6 +146,14 @@ static uint32_t enabled_port_mask = 0;
 static int promiscuous_on = 0;
 /* NUMA is enabled by default. */
 static int numa_on = 1;
+/* emptypoll is disabled by default. */
+static bool empty_poll_on;
+static bool empty_poll_train;
+volatile bool empty_poll_stop;
+static struct  ep_params *ep_params;
+static struct  ep_policy policy;
+static long  ep_med_edpi, ep_hgh_edpi;
+
 static int parse_ptype; /**< Parse packet type using rx callback, and */
 			/**< disabled by default */
 
@@ -180,8 +202,7 @@ static struct rte_eth_conf port_conf = {
 		.mq_mode        = ETH_MQ_RX_RSS,
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.offloads = (DEV_RX_OFFLOAD_CRC_STRIP |
-			     DEV_RX_OFFLOAD_CHECKSUM),
+		.offloads = DEV_RX_OFFLOAD_CHECKSUM,
 	},
 	.rx_adv_conf = {
 		.rss_conf = {
@@ -331,6 +352,19 @@ static inline uint32_t power_idle_heuristic(uint32_t zero_rx_packet_count);
 static inline enum freq_scale_hint_t power_freq_scaleup_heuristic( \
 		unsigned int lcore_id, uint16_t port_id, uint16_t queue_id);
 
+
+/*
+ * These defaults are using the max frequency index (1), a medium index (9)
+ * and a typical low frequency index (14). These can be adjusted to use
+ * different indexes using the relevant command line parameters.
+ */
+static uint8_t  freq_tlb[] = {14, 9, 1};
+
+static int is_done(void)
+{
+	return empty_poll_stop;
+}
+
 /* exit signal handler */
 static void
 signal_exit_now(int sigtype)
@@ -340,6 +374,10 @@ signal_exit_now(int sigtype)
 	int ret;
 
 	if (sigtype == SIGINT) {
+		if (empty_poll_on)
+			empty_poll_stop = true;
+
+
 		for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
 			if (rte_lcore_is_enabled(lcore_id) == 0)
 				continue;
@@ -352,16 +390,19 @@ signal_exit_now(int sigtype)
 							"core%u\n", lcore_id);
 		}
 
-		RTE_ETH_FOREACH_DEV(portid) {
-			if ((enabled_port_mask & (1 << portid)) == 0)
-				continue;
+		if (!empty_poll_on) {
+			RTE_ETH_FOREACH_DEV(portid) {
+				if ((enabled_port_mask & (1 << portid)) == 0)
+					continue;
 
-			rte_eth_dev_stop(portid);
-			rte_eth_dev_close(portid);
+				rte_eth_dev_stop(portid);
+				rte_eth_dev_close(portid);
+			}
 		}
 	}
 
-	rte_exit(EXIT_SUCCESS, "User forced exit\n");
+	if (!empty_poll_on)
+		rte_exit(EXIT_SUCCESS, "User forced exit\n");
 }
 
 /*  Freqency scale down timer callback */
@@ -826,7 +867,110 @@ static int event_register(struct lcore_conf *qconf)
 
 	return 0;
 }
+/* main processing loop */
+static int
+main_empty_poll_loop(__attribute__((unused)) void *dummy)
+{
+	struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
+	unsigned int lcore_id;
+	uint64_t prev_tsc, diff_tsc, cur_tsc;
+	int i, j, nb_rx;
+	uint8_t queueid;
+	uint16_t portid;
+	struct lcore_conf *qconf;
+	struct lcore_rx_queue *rx_queue;
+
+	const uint64_t drain_tsc =
+		(rte_get_tsc_hz() + US_PER_S - 1) /
+		US_PER_S * BURST_TX_DRAIN_US;
+
+	prev_tsc = 0;
+
+	lcore_id = rte_lcore_id();
+	qconf = &lcore_conf[lcore_id];
+
+	if (qconf->n_rx_queue == 0) {
+		RTE_LOG(INFO, L3FWD_POWER, "lcore %u has nothing to do\n",
+			lcore_id);
+		return 0;
+	}
+
+	for (i = 0; i < qconf->n_rx_queue; i++) {
+		portid = qconf->rx_queue_list[i].port_id;
+		queueid = qconf->rx_queue_list[i].queue_id;
+		RTE_LOG(INFO, L3FWD_POWER, " -- lcoreid=%u portid=%u "
+				"rxqueueid=%hhu\n", lcore_id, portid, queueid);
+	}
+
+	while (!is_done()) {
+		stats[lcore_id].nb_iteration_looped++;
+
+		cur_tsc = rte_rdtsc();
+		/*
+		 * TX burst queue drain
+		 */
+		diff_tsc = cur_tsc - prev_tsc;
+		if (unlikely(diff_tsc > drain_tsc)) {
+			for (i = 0; i < qconf->n_tx_port; ++i) {
+				portid = qconf->tx_port_id[i];
+				rte_eth_tx_buffer_flush(portid,
+						qconf->tx_queue_id[portid],
+						qconf->tx_buffer[portid]);
+			}
+			prev_tsc = cur_tsc;
+		}
+
+		/*
+		 * Read packet from RX queues
+		 */
+		for (i = 0; i < qconf->n_rx_queue; ++i) {
+			rx_queue = &(qconf->rx_queue_list[i]);
+			rx_queue->idle_hint = 0;
+			portid = rx_queue->port_id;
+			queueid = rx_queue->queue_id;
+
+			nb_rx = rte_eth_rx_burst(portid, queueid, pkts_burst,
+					MAX_PKT_BURST);
+
+			stats[lcore_id].nb_rx_processed += nb_rx;
+
+			if (nb_rx == 0) {
+
+				rte_power_empty_poll_stat_update(lcore_id);
+
+				continue;
+			} else {
+				rte_power_poll_stat_update(lcore_id, nb_rx);
+			}
 
+
+			/* Prefetch first packets */
+			for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) {
+				rte_prefetch0(rte_pktmbuf_mtod(
+							pkts_burst[j], void *));
+			}
+
+			/* Prefetch and forward already prefetched packets */
+			for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) {
+				rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[
+							j + PREFETCH_OFFSET],
+							void *));
+				l3fwd_simple_forward(pkts_burst[j], portid,
+						qconf);
+			}
+
+			/* Forward remaining prefetched packets */
+			for (; j < nb_rx; j++) {
+				l3fwd_simple_forward(pkts_burst[j], portid,
+						qconf);
+			}
+
+		}
+
+	}
+
+	return 0;
+}
 /* main processing loop */
 static int
 main_loop(__attribute__((unused)) void *dummy)
@@ -1128,7 +1272,9 @@ print_usage(const char *prgname)
 		"  --no-numa: optional, disable numa awareness\n"
 		"  --enable-jumbo: enable jumbo frame"
 		" which max packet len is PKTLEN in decimal (64-9600)\n"
-		"  --parse-ptype: parse packet type by software\n",
+		"  --parse-ptype: parse packet type by software\n"
+		"  --empty-poll: enable empty poll detection"
+		" follow (training_flag, high_threshold, med_threshold)\n",
 		prgname);
 }
 
@@ -1221,7 +1367,55 @@ parse_config(const char *q_arg)
 
 	return 0;
 }
+static int
+parse_ep_config(const char *q_arg)
+{
+	char s[256];
+	const char *p = q_arg;
+	char *end;
+	int  num_arg;
+
+	char *str_fld[3];
+
+	int training_flag;
+	int med_edpi;
+	int hgh_edpi;
+
+	ep_med_edpi = EMPTY_POLL_MED_THRESHOLD;
+	ep_hgh_edpi = EMPTY_POLL_MED_THRESHOLD;
+
+	snprintf(s, sizeof(s), "%s", p);
+
+	num_arg = rte_strsplit(s, sizeof(s), str_fld, 3, ',');
+
+	empty_poll_train = false;
+
+	if (num_arg == 0)
+		return 0;
+
+	if (num_arg == 3) {
+
+		training_flag = strtoul(str_fld[0], &end, 0);
+		med_edpi = strtoul(str_fld[1], &end, 0);
+		hgh_edpi = strtoul(str_fld[2], &end, 0);
+
+		if (training_flag == 1)
+			empty_poll_train = true;
+
+		if (med_edpi > 0)
+			ep_med_edpi = med_edpi;
+
+		if (med_edpi > 0)
+			ep_hgh_edpi = hgh_edpi;
+
+	} else {
+
+		return -1;
+	}
+
+	return 0;
 
+}
 #define CMD_LINE_OPT_PARSE_PTYPE "parse-ptype"
 
 /* Parse the argument given in the command line of the application */
@@ -1231,6 +1425,7 @@ parse_args(int argc, char **argv)
 	int opt, ret;
 	char **argvopt;
 	int option_index;
+	uint32_t limit;
 	char *prgname = argv[0];
 	static struct option lgopts[] = {
 		{"config", 1, 0, 0},
@@ -1238,13 +1433,14 @@ parse_args(int argc, char **argv)
 		{"high-perf-cores", 1, 0, 0},
 		{"no-numa", 0, 0, 0},
 		{"enable-jumbo", 0, 0, 0},
+		{"empty-poll", 1, 0, 0},
 		{CMD_LINE_OPT_PARSE_PTYPE, 0, 0, 0},
 		{NULL, 0, 0, 0}
 	};
 
 	argvopt = argv;
 
-	while ((opt = getopt_long(argc, argvopt, "p:P",
+	while ((opt = getopt_long(argc, argvopt, "p:l:m:h:P",
 				lgopts, &option_index)) != EOF) {
 
 		switch (opt) {
@@ -1261,7 +1457,18 @@ parse_args(int argc, char **argv)
 			printf("Promiscuous mode selected\n");
 			promiscuous_on = 1;
 			break;
-
+		case 'l':
+			limit = parse_max_pkt_len(optarg);
+			freq_tlb[LOW] = limit;
+			break;
+		case 'm':
+			limit = parse_max_pkt_len(optarg);
+			freq_tlb[MED] = limit;
+			break;
+		case 'h':
+			limit = parse_max_pkt_len(optarg);
+			freq_tlb[HGH] = limit;
+			break;
 		/* long options */
 		case 0:
 			if (!strncmp(lgopts[option_index].name, "config", 6)) {
@@ -1300,6 +1507,20 @@ parse_args(int argc, char **argv)
 			}
 
 			if (!strncmp(lgopts[option_index].name,
+						"empty-poll", 10)) {
+				printf("empty-poll is enabled\n");
+				empty_poll_on = true;
+				ret = parse_ep_config(optarg);
+
+				if (ret) {
+					printf("invalid empty poll config\n");
+					print_usage(prgname);
+					return -1;
+				}
+
+			}
+
+			if (!strncmp(lgopts[option_index].name,
 					"enable-jumbo", 12)) {
 				struct option lenopts =
 					{"max-pkt-len", required_argument, \
@@ -1647,6 +1868,59 @@ init_power_library(void)
 	}
 	return ret;
 }
+static void
+empty_poll_setup_timer(void)
+{
+	int lcore_id = rte_lcore_id();
+	uint64_t hz = rte_get_timer_hz();
+
+	struct  ep_params *ep_ptr = ep_params;
+
+	ep_ptr->interval_ticks = hz / INTERVALS_PER_SECOND;
+
+	rte_timer_reset_sync(&ep_ptr->timer0,
+			ep_ptr->interval_ticks,
+			PERIODICAL,
+			lcore_id,
+			rte_empty_poll_detection,
+			(void *)ep_ptr);
+
+}
+static int
+launch_timer(unsigned int lcore_id)
+{
+	int64_t prev_tsc = 0, cur_tsc, diff_tsc, cycles_10ms;
+
+	RTE_SET_USED(lcore_id);
+
+
+	if (rte_get_master_lcore() != lcore_id) {
+		rte_panic("timer on lcore:%d which is not master core:%d\n",
+				lcore_id,
+				rte_get_master_lcore());
+	}
+
+	RTE_LOG(INFO, POWER, "Bring up the Timer\n");
+
+	empty_poll_setup_timer();
+
+	cycles_10ms = rte_get_timer_hz() / 100;
+
+	while (!is_done()) {
+		cur_tsc = rte_rdtsc();
+		diff_tsc = cur_tsc - prev_tsc;
+		if (diff_tsc > cycles_10ms) {
+			rte_timer_manage();
+			prev_tsc = cur_tsc;
+			cycles_10ms = rte_get_timer_hz() / 100;
+		}
+	}
+
+	RTE_LOG(INFO, POWER, "Timer_subsystem is done\n");
+
+	return 0;
+}
+
 
 int
 main(int argc, char **argv)
@@ -1829,13 +2103,15 @@ main(int argc, char **argv)
 		if (rte_lcore_is_enabled(lcore_id) == 0)
 			continue;
 
-		/* init timer structures for each enabled lcore */
-		rte_timer_init(&power_timers[lcore_id]);
-		hz = rte_get_timer_hz();
-		rte_timer_reset(&power_timers[lcore_id],
-			hz/TIMER_NUMBER_PER_SECOND, SINGLE, lcore_id,
-						power_timer_cb, NULL);
-
+		if (empty_poll_on == false) {
+			/* init timer structures for each enabled lcore */
+			rte_timer_init(&power_timers[lcore_id]);
+			hz = rte_get_timer_hz();
+			rte_timer_reset(&power_timers[lcore_id],
+					hz/TIMER_NUMBER_PER_SECOND,
+					SINGLE, lcore_id,
+					power_timer_cb, NULL);
+		}
 		qconf = &lcore_conf[lcore_id];
 		printf("\nInitializing rx queues on lcore %u ... ", lcore_id );
 		fflush(stdout);
@@ -1906,12 +2182,43 @@ main(int argc, char **argv)
 
 	check_all_ports_link_status(enabled_port_mask);
 
+	if (empty_poll_on == true) {
+
+		if (empty_poll_train) {
+			policy.state = TRAINING;
+		} else {
+			policy.state = MED_NORMAL;
+			policy.med_base_edpi = ep_med_edpi;
+			policy.hgh_base_edpi = ep_hgh_edpi;
+		}
+
+		ret = rte_power_empty_poll_stat_init(&ep_params,
+				freq_tlb,
+				&policy);
+		if (ret < 0)
+			rte_exit(EXIT_FAILURE, "empty poll init failed");
+	}
+
+
 	/* launch per-lcore init on every lcore */
-	rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER);
+	if (empty_poll_on == false) {
+		rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER);
+	} else {
+		empty_poll_stop = false;
+		rte_eal_mp_remote_launch(main_empty_poll_loop, NULL,
+				SKIP_MASTER);
+	}
+
+	if (empty_poll_on == true)
+		launch_timer(rte_lcore_id());
+
 	RTE_LCORE_FOREACH_SLAVE(lcore_id) {
 		if (rte_eal_wait_lcore(lcore_id) < 0)
 			return -1;
 	}
 
+	if (empty_poll_on)
+		rte_power_empty_poll_stat_free();
+
 	return 0;
 }
diff --git a/examples/l3fwd-power/meson.build b/examples/l3fwd-power/meson.build
index 20c80543..a3c5c2f1 100644
--- a/examples/l3fwd-power/meson.build
+++ b/examples/l3fwd-power/meson.build
@@ -9,6 +9,7 @@
 if host_machine.system() != 'linux'
 	build = false
 endif
+allow_experimental_apis = true
 deps += ['power', 'timer', 'lpm', 'hash']
 sources = files(
 	'main.c', 'perf_core.c'
diff --git a/examples/l3fwd-vf/main.c b/examples/l3fwd-vf/main.c
index 5edd91a7..41137f97 100644
--- a/examples/l3fwd-vf/main.c
+++ b/examples/l3fwd-vf/main.c
@@ -161,8 +161,7 @@ static struct rte_eth_conf port_conf = {
 		.mq_mode	= ETH_MQ_RX_RSS,
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.offloads = (DEV_RX_OFFLOAD_CRC_STRIP |
-			     DEV_RX_OFFLOAD_CHECKSUM),
+		.offloads = DEV_RX_OFFLOAD_CHECKSUM,
 	},
 	.rx_adv_conf = {
 		.rss_conf = {
diff --git a/examples/l3fwd/main.c b/examples/l3fwd/main.c
index ab019b9e..e4b99efe 100644
--- a/examples/l3fwd/main.c
+++ b/examples/l3fwd/main.c
@@ -120,8 +120,7 @@ static struct rte_eth_conf port_conf = {
 		.mq_mode = ETH_MQ_RX_RSS,
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.offloads = (DEV_RX_OFFLOAD_CRC_STRIP |
-			     DEV_RX_OFFLOAD_CHECKSUM),
+		.offloads = DEV_RX_OFFLOAD_CHECKSUM,
 	},
 	.rx_adv_conf = {
 		.rss_conf = {
diff --git a/examples/link_status_interrupt/main.c b/examples/link_status_interrupt/main.c
index 3b732076..f3346d23 100644
--- a/examples/link_status_interrupt/main.c
+++ b/examples/link_status_interrupt/main.c
@@ -79,7 +79,6 @@ struct rte_eth_dev_tx_buffer *tx_buffer[RTE_MAX_ETHPORTS];
 static struct rte_eth_conf port_conf = {
 	.rxmode = {
 		.split_hdr_size = 0,
-		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.txmode = {
 		.mq_mode = ETH_MQ_TX_NONE,
diff --git a/examples/load_balancer/Makefile b/examples/load_balancer/Makefile
index fc8df71e..197b019d 100644
--- a/examples/load_balancer/Makefile
+++ b/examples/load_balancer/Makefile
@@ -50,7 +50,6 @@ include $(RTE_SDK)/mk/rte.vars.mk
 
 CFLAGS += -O3 -g
 CFLAGS += $(WERROR_FLAGS)
-CFLAGS_config.o := -D_GNU_SOURCE
 
 # workaround for a gcc bug with noreturn attribute
 # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=12603
diff --git a/examples/load_balancer/init.c b/examples/load_balancer/init.c
index f2045f23..3ab7d021 100644
--- a/examples/load_balancer/init.c
+++ b/examples/load_balancer/init.c
@@ -45,8 +45,7 @@ static struct rte_eth_conf port_conf = {
 	.rxmode = {
 		.mq_mode	= ETH_MQ_RX_RSS,
 		.split_hdr_size = 0,
-		.offloads = (DEV_RX_OFFLOAD_CHECKSUM |
-			     DEV_RX_OFFLOAD_CRC_STRIP),
+		.offloads = DEV_RX_OFFLOAD_CHECKSUM,
 	},
 	.rx_adv_conf = {
 		.rss_conf = {
diff --git a/examples/meson.build b/examples/meson.build
index 4ee7a111..af81c762 100644
--- a/examples/meson.build
+++ b/examples/meson.build
@@ -22,6 +22,10 @@ default_cflags = machine_args
 if cc.has_argument('-Wno-format-truncation')
 	default_cflags += '-Wno-format-truncation'
 endif
+
+# specify -D_GNU_SOURCE unconditionally
+default_cflags += '-D_GNU_SOURCE'
+
 foreach example: examples
 	name = example
 	build = true
diff --git a/examples/multi_process/Makefile b/examples/multi_process/Makefile
index a6708b7e..b76b02fc 100644
--- a/examples/multi_process/Makefile
+++ b/examples/multi_process/Makefile
@@ -13,5 +13,6 @@ include $(RTE_SDK)/mk/rte.vars.mk
 DIRS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += client_server_mp
 DIRS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += simple_mp
 DIRS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += symmetric_mp
+DIRS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += hotplug_mp
 
 include $(RTE_SDK)/mk/rte.extsubdir.mk
diff --git a/examples/multi_process/client_server_mp/mp_server/main.c b/examples/multi_process/client_server_mp/mp_server/main.c
index 93a9a089..0ddc63e9 100644
--- a/examples/multi_process/client_server_mp/mp_server/main.c
+++ b/examples/multi_process/client_server_mp/mp_server/main.c
@@ -12,6 +12,7 @@
 #include <sys/queue.h>
 #include <errno.h>
 #include <netinet/ip.h>
+#include <signal.h>
 
 #include <rte_common.h>
 #include <rte_memory.h>
@@ -264,9 +265,23 @@ do_packet_forwarding(void)
 	}
 }
 
+static void
+signal_handler(int signal)
+{
+	uint16_t port_id;
+
+	if (signal == SIGINT)
+		RTE_ETH_FOREACH_DEV(port_id) {
+			rte_eth_dev_stop(port_id);
+			rte_eth_dev_close(port_id);
+		}
+	exit(0);
+}
+
 int
 main(int argc, char *argv[])
 {
+	signal(SIGINT, signal_handler);
 	/* initialise the system */
 	if (init(argc, argv) < 0 )
 		return -1;
diff --git a/examples/multi_process/hotplug_mp/Makefile b/examples/multi_process/hotplug_mp/Makefile
new file mode 100644
index 00000000..bc36aeae
--- /dev/null
+++ b/examples/multi_process/hotplug_mp/Makefile
@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+ifeq ($(RTE_SDK),)
+$(error "Please define RTE_SDK environment variable")
+endif
+
+# Default target, can be overridden by command line or environment
+RTE_TARGET ?= x86_64-native-linuxapp-gcc
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+# binary name
+APP = hotplug_mp
+
+# all source are stored in SRCS-y
+SRCS-y := main.c commands.c
+
+CFLAGS += -O3
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+
+include $(RTE_SDK)/mk/rte.extapp.mk
diff --git a/examples/multi_process/hotplug_mp/commands.c b/examples/multi_process/hotplug_mp/commands.c
new file mode 100644
index 00000000..b0685939
--- /dev/null
+++ b/examples/multi_process/hotplug_mp/commands.c
@@ -0,0 +1,214 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation.
+ */
+
+#include <cmdline_rdline.h>
+#include <cmdline_parse.h>
+#include <cmdline_parse_ipaddr.h>
+#include <cmdline_parse_num.h>
+#include <cmdline_parse_string.h>
+#include <cmdline.h>
+#include <rte_ethdev.h>
+
+/**********************************************************/
+
+struct cmd_help_result {
+	cmdline_fixed_string_t help;
+};
+
+static void cmd_help_parsed(__attribute__((unused)) void *parsed_result,
+			    struct cmdline *cl,
+			    __attribute__((unused)) void *data)
+{
+	cmdline_printf(cl,
+		       "commands:\n"
+		       "- attach <devargs>\n"
+		       "- detach <devargs>\n"
+		       "- list\n\n");
+}
+
+cmdline_parse_token_string_t cmd_help_help =
+	TOKEN_STRING_INITIALIZER(struct cmd_help_result, help, "help");
+
+cmdline_parse_inst_t cmd_help = {
+	.f = cmd_help_parsed,  /* function to call */
+	.data = NULL,      /* 2nd arg of func */
+	.help_str = "show help",
+	.tokens = {        /* token list, NULL terminated */
+		(void *)&cmd_help_help,
+		NULL,
+	},
+};
+
+/**********************************************************/
+
+struct cmd_quit_result {
+	cmdline_fixed_string_t quit;
+};
+
+static void cmd_quit_parsed(__attribute__((unused)) void *parsed_result,
+			    struct cmdline *cl,
+			    __attribute__((unused)) void *data)
+{
+	cmdline_quit(cl);
+}
+
+cmdline_parse_token_string_t cmd_quit_quit =
+	TOKEN_STRING_INITIALIZER(struct cmd_quit_result, quit, "quit");
+
+cmdline_parse_inst_t cmd_quit = {
+	.f = cmd_quit_parsed,  /* function to call */
+	.data = NULL,      /* 2nd arg of func */
+	.help_str = "quit",
+	.tokens = {        /* token list, NULL terminated */
+		(void *)&cmd_quit_quit,
+		NULL,
+	},
+};
+
+/**********************************************************/
+
+struct cmd_list_result {
+	cmdline_fixed_string_t list;
+};
+
+static void cmd_list_parsed(__attribute__((unused)) void *parsed_result,
+			    struct cmdline *cl,
+			    __attribute__((unused)) void *data)
+{
+	uint16_t port_id;
+	char dev_name[RTE_DEV_NAME_MAX_LEN];
+
+	cmdline_printf(cl, "list all etherdev\n");
+
+	RTE_ETH_FOREACH_DEV(port_id) {
+		rte_eth_dev_get_name_by_port(port_id, dev_name);
+		if (strlen(dev_name) > 0)
+			cmdline_printf(cl, "%d\t%s\n", port_id, dev_name);
+		else
+			printf("empty dev_name is not expected!\n");
+	}
+}
+
+cmdline_parse_token_string_t cmd_list_list =
+	TOKEN_STRING_INITIALIZER(struct cmd_list_result, list, "list");
+
+cmdline_parse_inst_t cmd_list = {
+	.f = cmd_list_parsed,  /* function to call */
+	.data = NULL,      /* 2nd arg of func */
+	.help_str = "list all devices",
+	.tokens = {        /* token list, NULL terminated */
+		(void *)&cmd_list_list,
+		NULL,
+	},
+};
+
+/**********************************************************/
+
+struct cmd_dev_attach_result {
+	cmdline_fixed_string_t attach;
+	cmdline_fixed_string_t devargs;
+};
+
+static void cmd_dev_attach_parsed(void *parsed_result,
+				  struct cmdline *cl,
+				  __attribute__((unused)) void *data)
+{
+	struct cmd_dev_attach_result *res = parsed_result;
+	struct rte_devargs da;
+
+	memset(&da, 0, sizeof(da));
+
+	if (rte_devargs_parsef(&da, "%s", res->devargs)) {
+		cmdline_printf(cl, "cannot parse devargs\n");
+		if (da.args)
+			free(da.args);
+		return;
+	}
+
+	if (!rte_eal_hotplug_add(da.bus->name, da.name, da.args))
+		cmdline_printf(cl, "attached device %s\n", da.name);
+	else
+		cmdline_printf(cl, "failed to attached device %s\n",
+				da.name);
+}
+
+cmdline_parse_token_string_t cmd_dev_attach_attach =
+	TOKEN_STRING_INITIALIZER(struct cmd_dev_attach_result, attach,
+				 "attach");
+cmdline_parse_token_string_t cmd_dev_attach_devargs =
+	TOKEN_STRING_INITIALIZER(struct cmd_dev_attach_result, devargs, NULL);
+
+cmdline_parse_inst_t cmd_attach_device = {
+	.f = cmd_dev_attach_parsed,  /* function to call */
+	.data = NULL,      /* 2nd arg of func */
+	.help_str = "attach a device",
+	.tokens = {        /* token list, NULL terminated */
+		(void *)&cmd_dev_attach_attach,
+		(void *)&cmd_dev_attach_devargs,
+		NULL,
+	},
+};
+
+/**********************************************************/
+
+struct cmd_dev_detach_result {
+	cmdline_fixed_string_t detach;
+	cmdline_fixed_string_t devargs;
+};
+
+static void cmd_dev_detach_parsed(void *parsed_result,
+				   struct cmdline *cl,
+				   __attribute__((unused)) void *data)
+{
+	struct cmd_dev_detach_result *res = parsed_result;
+	struct rte_devargs da;
+
+	memset(&da, 0, sizeof(da));
+
+	if (rte_devargs_parsef(&da, "%s", res->devargs)) {
+		cmdline_printf(cl, "cannot parse devargs\n");
+		if (da.args)
+			free(da.args);
+		return;
+	}
+
+	printf("detaching...\n");
+	if (!rte_eal_hotplug_remove(da.bus->name, da.name))
+		cmdline_printf(cl, "detached device %s\n",
+			da.name);
+	else
+		cmdline_printf(cl, "failed to dettach device %s\n",
+			da.name);
+}
+
+cmdline_parse_token_string_t cmd_dev_detach_detach =
+	TOKEN_STRING_INITIALIZER(struct cmd_dev_detach_result, detach,
+				 "detach");
+
+cmdline_parse_token_string_t cmd_dev_detach_devargs =
+	TOKEN_STRING_INITIALIZER(struct cmd_dev_detach_result, devargs, NULL);
+
+cmdline_parse_inst_t cmd_detach_device = {
+	.f = cmd_dev_detach_parsed,  /* function to call */
+	.data = NULL,      /* 2nd arg of func */
+	.help_str = "detach a device",
+	.tokens = {        /* token list, NULL terminated */
+		(void *)&cmd_dev_detach_detach,
+		(void *)&cmd_dev_detach_devargs,
+		NULL,
+	},
+};
+
+/**********************************************************/
+/**********************************************************/
+/****** CONTEXT (list of instruction) */
+
+cmdline_parse_ctx_t main_ctx[] = {
+	(cmdline_parse_inst_t *)&cmd_help,
+	(cmdline_parse_inst_t *)&cmd_quit,
+	(cmdline_parse_inst_t *)&cmd_list,
+	(cmdline_parse_inst_t *)&cmd_attach_device,
+	(cmdline_parse_inst_t *)&cmd_detach_device,
+	NULL,
+};
diff --git a/examples/multi_process/hotplug_mp/commands.h b/examples/multi_process/hotplug_mp/commands.h
new file mode 100644
index 00000000..afcf177d
--- /dev/null
+++ b/examples/multi_process/hotplug_mp/commands.h
@@ -0,0 +1,10 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#ifndef _COMMANDS_H_
+#define _COMMANDS_H_
+
+extern cmdline_parse_ctx_t main_ctx[];
+
+#endif /* _COMMANDS_H_ */
diff --git a/examples/multi_process/hotplug_mp/main.c b/examples/multi_process/hotplug_mp/main.c
new file mode 100644
index 00000000..d6685807
--- /dev/null
+++ b/examples/multi_process/hotplug_mp/main.c
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <errno.h>
+#include <termios.h>
+#include <sys/queue.h>
+
+#include <cmdline_rdline.h>
+#include <cmdline_parse.h>
+#include <cmdline_socket.h>
+#include <cmdline.h>
+
+#include <rte_memory.h>
+#include <rte_eal.h>
+#include <rte_debug.h>
+
+#include "commands.h"
+
+int main(int argc, char **argv)
+{
+	int ret;
+	struct cmdline *cl;
+
+	ret = rte_eal_init(argc, argv);
+	if (ret < 0)
+		rte_panic("Cannot init EAL\n");
+
+	cl = cmdline_stdin_new(main_ctx, "example> ");
+	if (cl == NULL)
+		rte_panic("Cannot create cmdline instance\n");
+	cmdline_interact(cl);
+	cmdline_stdin_exit(cl);
+
+	rte_eal_cleanup();
+
+	return 0;
+}
diff --git a/examples/multi_process/symmetric_mp/main.c b/examples/multi_process/symmetric_mp/main.c
index c6c6a537..c310e942 100644
--- a/examples/multi_process/symmetric_mp/main.c
+++ b/examples/multi_process/symmetric_mp/main.c
@@ -178,8 +178,7 @@ smp_port_init(uint16_t port, struct rte_mempool *mbuf_pool,
 			.rxmode = {
 				.mq_mode	= ETH_MQ_RX_RSS,
 				.split_hdr_size = 0,
-				.offloads = (DEV_RX_OFFLOAD_CHECKSUM |
-					     DEV_RX_OFFLOAD_CRC_STRIP),
+				.offloads = DEV_RX_OFFLOAD_CHECKSUM,
 			},
 			.rx_adv_conf = {
 				.rss_conf = {
diff --git a/examples/netmap_compat/bridge/bridge.c b/examples/netmap_compat/bridge/bridge.c
index 7afca28c..216e0105 100644
--- a/examples/netmap_compat/bridge/bridge.c
+++ b/examples/netmap_compat/bridge/bridge.c
@@ -26,7 +26,6 @@
 struct rte_eth_conf eth_conf = {
 	.rxmode = {
 		.split_hdr_size = 0,
-		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.txmode = {
 		.mq_mode = ETH_MQ_TX_NONE,
diff --git a/examples/performance-thread/l3fwd-thread/main.c b/examples/performance-thread/l3fwd-thread/main.c
index 5392fcea..4f8747bc 100644
--- a/examples/performance-thread/l3fwd-thread/main.c
+++ b/examples/performance-thread/l3fwd-thread/main.c
@@ -2,8 +2,6 @@
  * Copyright(c) 2010-2016 Intel Corporation
  */
 
-#define _GNU_SOURCE
-
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdint.h>
@@ -306,8 +304,7 @@ static struct rte_eth_conf port_conf = {
 		.mq_mode = ETH_MQ_RX_RSS,
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.offloads = (DEV_RX_OFFLOAD_CHECKSUM |
-			     DEV_RX_OFFLOAD_CRC_STRIP),
+		.offloads = DEV_RX_OFFLOAD_CHECKSUM,
 	},
 	.rx_adv_conf = {
 		.rss_conf = {
diff --git a/examples/performance-thread/pthread_shim/main.c b/examples/performance-thread/pthread_shim/main.c
index 7d0d5811..03ff3943 100644
--- a/examples/performance-thread/pthread_shim/main.c
+++ b/examples/performance-thread/pthread_shim/main.c
@@ -2,7 +2,6 @@
  * Copyright(c) 2015 Intel Corporation
  */
 
-#define _GNU_SOURCE
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdint.h>
diff --git a/examples/performance-thread/pthread_shim/pthread_shim.c b/examples/performance-thread/pthread_shim/pthread_shim.c
index 53f12437..a02de067 100644
--- a/examples/performance-thread/pthread_shim/pthread_shim.c
+++ b/examples/performance-thread/pthread_shim/pthread_shim.c
@@ -6,7 +6,6 @@
 #include <stdlib.h>
 #include <sys/types.h>
 #include <errno.h>
-#define __USE_GNU
 #include <sched.h>
 #include <dlfcn.h>
 
diff --git a/examples/qos_meter/main.c b/examples/qos_meter/main.c
index 5cf4e9df..9b011244 100644
--- a/examples/qos_meter/main.c
+++ b/examples/qos_meter/main.c
@@ -56,8 +56,7 @@ static struct rte_eth_conf port_conf = {
 		.mq_mode	= ETH_MQ_RX_RSS,
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.offloads = (DEV_RX_OFFLOAD_CHECKSUM |
-			     DEV_RX_OFFLOAD_CRC_STRIP),
+		.offloads = DEV_RX_OFFLOAD_CHECKSUM,
 	},
 	.rx_adv_conf = {
 		.rss_conf = {
diff --git a/examples/qos_sched/Makefile b/examples/qos_sched/Makefile
index a7ecf978..45b0a9eb 100644
--- a/examples/qos_sched/Makefile
+++ b/examples/qos_sched/Makefile
@@ -57,8 +57,6 @@ else
 
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
-CFLAGS_args.o := -D_GNU_SOURCE
-CFLAGS_cfg_file.o := -D_GNU_SOURCE
 
 include $(RTE_SDK)/mk/rte.extapp.mk
 
diff --git a/examples/qos_sched/init.c b/examples/qos_sched/init.c
index 94cbb26f..37c2b95f 100644
--- a/examples/qos_sched/init.c
+++ b/examples/qos_sched/init.c
@@ -59,7 +59,6 @@ static struct rte_eth_conf port_conf = {
 	.rxmode = {
 		.max_rx_pkt_len = ETHER_MAX_LEN,
 		.split_hdr_size = 0,
-		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.txmode = {
 		.mq_mode = ETH_DCB_NONE,
diff --git a/examples/quota_watermark/qw/init.c b/examples/quota_watermark/qw/init.c
index 19164385..5a0f64f4 100644
--- a/examples/quota_watermark/qw/init.c
+++ b/examples/quota_watermark/qw/init.c
@@ -24,7 +24,6 @@
 static struct rte_eth_conf port_conf = {
 		.rxmode = {
 			.split_hdr_size = 0,
-			.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 		},
 		.txmode = {
 			.mq_mode = ETH_DCB_NONE,
diff --git a/examples/service_cores/main.c b/examples/service_cores/main.c
index 2cd57290..c7c79281 100644
--- a/examples/service_cores/main.c
+++ b/examples/service_cores/main.c
@@ -118,6 +118,12 @@ apply_profile(int profile_id)
 	struct profile *p = &profiles[profile_id];
 	const uint8_t core_off = 1;
 
+	if (p->num_cores > rte_lcore_count() + 1) {
+		printf("insufficent cores to run (%s)",
+			p->name);
+		return;
+	}
+
 	for (i = 0; i < p->num_cores; i++) {
 		uint32_t core = i + core_off;
 		ret = rte_service_lcore_add(core);
diff --git a/examples/tep_termination/Makefile b/examples/tep_termination/Makefile
index 8ec1a38e..4c156432 100644
--- a/examples/tep_termination/Makefile
+++ b/examples/tep_termination/Makefile
@@ -60,7 +60,6 @@ endif
 CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O3
 CFLAGS += $(WERROR_FLAGS)
-CFLAGS += -D_GNU_SOURCE
 
 include $(RTE_SDK)/mk/rte.extapp.mk
 endif
diff --git a/examples/tep_termination/vxlan_setup.c b/examples/tep_termination/vxlan_setup.c
index b99ab97d..ad7fbe9c 100644
--- a/examples/tep_termination/vxlan_setup.c
+++ b/examples/tep_termination/vxlan_setup.c
@@ -69,7 +69,6 @@ uint8_t tep_filter_type[] = {RTE_TUNNEL_FILTER_IMAC_TENID,
 static struct rte_eth_conf port_conf = {
 	.rxmode = {
 		.split_hdr_size = 0,
-		.offloads = DEV_RX_OFFLOAD_CRC_STRIP,
 	},
 	.txmode = {
 		.mq_mode = ETH_MQ_TX_NONE,
diff --git a/examples/vdpa/Makefile b/examples/vdpa/Makefile
new file mode 100644
index 00000000..42672a2b
--- /dev/null
+++ b/examples/vdpa/Makefile
@@ -0,0 +1,32 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+ifeq ($(RTE_SDK),)
+$(error "Please define RTE_SDK environment variable")
+endif
+
+# Default target, can be overridden by command line or environment
+RTE_TARGET ?= x86_64-native-linuxapp-gcc
+
+include $(RTE_SDK)/mk/rte.vars.mk
+
+ifneq ($(CONFIG_RTE_EXEC_ENV),"linuxapp")
+$(info This application can only operate in a linuxapp environment, \
+please change the definition of the RTE_TARGET environment variable)
+all:
+else
+
+# binary name
+APP = vdpa
+
+# all source are stored in SRCS-y
+SRCS-y := main.c
+
+CFLAGS += -O2 -D_FILE_OFFSET_BITS=64
+CFLAGS += $(WERROR_FLAGS)
+CFLAGS += -D_GNU_SOURCE
+CFLAGS += -DALLOW_EXPERIMENTAL_API
+
+include $(RTE_SDK)/mk/rte.extapp.mk
+
+endif
diff --git a/examples/vdpa/main.c b/examples/vdpa/main.c
new file mode 100644
index 00000000..d2e2cb7c
--- /dev/null
+++ b/examples/vdpa/main.c
@@ -0,0 +1,454 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2018 Intel Corporation
+ */
+
+#include <getopt.h>
+#include <signal.h>
+#include <stdint.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <rte_ethdev.h>
+#include <rte_malloc.h>
+#include <rte_vhost.h>
+#include <rte_vdpa.h>
+#include <rte_pci.h>
+#include <rte_string_fns.h>
+
+#include <cmdline_parse.h>
+#include <cmdline_socket.h>
+#include <cmdline_parse_string.h>
+#include <cmdline.h>
+
+#define MAX_PATH_LEN 128
+#define MAX_VDPA_SAMPLE_PORTS 1024
+#define RTE_LOGTYPE_VDPA RTE_LOGTYPE_USER1
+
+struct vdpa_port {
+	char ifname[MAX_PATH_LEN];
+	int did;
+	int vid;
+	uint64_t flags;
+};
+
+static struct vdpa_port vports[MAX_VDPA_SAMPLE_PORTS];
+
+static char iface[MAX_PATH_LEN];
+static int dev_total;
+static int devcnt;
+static int interactive;
+static int client_mode;
+
+/* display usage */
+static void
+vdpa_usage(const char *prgname)
+{
+	printf("Usage: %s [EAL options] -- "
+				 "	--interactive|-i: run in interactive mode.\n"
+				 "	--iface <path>: specify the path prefix of the socket files, e.g. /tmp/vhost-user-.\n"
+				 "	--client: register a vhost-user socket as client mode.\n",
+				 prgname);
+}
+
+static int
+parse_args(int argc, char **argv)
+{
+	static const char *short_option = "i";
+	static struct option long_option[] = {
+		{"iface", required_argument, NULL, 0},
+		{"interactive", no_argument, &interactive, 1},
+		{"client", no_argument, &client_mode, 1},
+		{NULL, 0, 0, 0},
+	};
+	int opt, idx;
+	char *prgname = argv[0];
+
+	while ((opt = getopt_long(argc, argv, short_option, long_option, &idx))
+			!= EOF) {
+		switch (opt) {
+		case 'i':
+			printf("Interactive-mode selected\n");
+			interactive = 1;
+			break;
+		/* long options */
+		case 0:
+			if (strncmp(long_option[idx].name, "iface",
+						MAX_PATH_LEN) == 0) {
+				rte_strscpy(iface, optarg, MAX_PATH_LEN);
+				printf("iface %s\n", iface);
+			}
+			if (!strcmp(long_option[idx].name, "interactive")) {
+				printf("Interactive-mode selected\n");
+				interactive = 1;
+			}
+			break;
+
+		default:
+			vdpa_usage(prgname);
+			return -1;
+		}
+	}
+
+	if (iface[0] == '\0' && interactive == 0) {
+		vdpa_usage(prgname);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int
+new_device(int vid)
+{
+	char ifname[MAX_PATH_LEN];
+	int i;
+
+	rte_vhost_get_ifname(vid, ifname, sizeof(ifname));
+	for (i = 0; i < MAX_VDPA_SAMPLE_PORTS; i++) {
+		if (strncmp(ifname, vports[i].ifname, MAX_PATH_LEN) == 0) {
+			printf("\nnew port %s, did: %d\n",
+					ifname, vports[i].did);
+			vports[i].vid = vid;
+			break;
+		}
+	}
+
+	if (i >= MAX_VDPA_SAMPLE_PORTS)
+		return -1;
+
+	return 0;
+}
+
+static void
+destroy_device(int vid)
+{
+	char ifname[MAX_PATH_LEN];
+	int i;
+
+	rte_vhost_get_ifname(vid, ifname, sizeof(ifname));
+	for (i = 0; i < MAX_VDPA_SAMPLE_PORTS; i++) {
+		if (strcmp(ifname, vports[i].ifname) == 0) {
+			printf("\ndestroy port %s, did: %d\n",
+					ifname, vports[i].did);
+			break;
+		}
+	}
+}
+
+static const struct vhost_device_ops vdpa_sample_devops = {
+	.new_device = new_device,
+	.destroy_device = destroy_device,
+};
+
+static int
+start_vdpa(struct vdpa_port *vport)
+{
+	int ret;
+	char *socket_path = vport->ifname;
+	int did = vport->did;
+
+	if (client_mode)
+		vport->flags |= RTE_VHOST_USER_CLIENT;
+
+	if (access(socket_path, F_OK) != -1 && !client_mode) {
+		RTE_LOG(ERR, VDPA,
+			"%s exists, please remove it or specify another file and try again.\n",
+			socket_path);
+		return -1;
+	}
+	ret = rte_vhost_driver_register(socket_path, vport->flags);
+	if (ret != 0)
+		rte_exit(EXIT_FAILURE,
+			"register driver failed: %s\n",
+			socket_path);
+
+	ret = rte_vhost_driver_callback_register(socket_path,
+			&vdpa_sample_devops);
+	if (ret != 0)
+		rte_exit(EXIT_FAILURE,
+			"register driver ops failed: %s\n",
+			socket_path);
+
+	ret = rte_vhost_driver_attach_vdpa_device(socket_path, did);
+	if (ret != 0)
+		rte_exit(EXIT_FAILURE,
+			"attach vdpa device failed: %s\n",
+			socket_path);
+
+	if (rte_vhost_driver_start(socket_path) < 0)
+		rte_exit(EXIT_FAILURE,
+			"start vhost driver failed: %s\n",
+			socket_path);
+	return 0;
+}
+
+static void
+close_vdpa(struct vdpa_port *vport)
+{
+	int ret;
+	char *socket_path = vport->ifname;
+
+	ret = rte_vhost_driver_detach_vdpa_device(socket_path);
+	if (ret != 0)
+		RTE_LOG(ERR, VDPA,
+				"detach vdpa device failed: %s\n",
+				socket_path);
+
+	ret = rte_vhost_driver_unregister(socket_path);
+	if (ret != 0)
+		RTE_LOG(ERR, VDPA,
+				"Fail to unregister vhost driver for %s.\n",
+				socket_path);
+}
+
+static void
+vdpa_sample_quit(void)
+{
+	int i;
+	for (i = 0; i < RTE_MIN(MAX_VDPA_SAMPLE_PORTS, dev_total); i++) {
+		if (vports[i].ifname[0] != '\0')
+			close_vdpa(&vports[i]);
+	}
+}
+
+static void
+signal_handler(int signum)
+{
+	if (signum == SIGINT || signum == SIGTERM) {
+		printf("\nSignal %d received, preparing to exit...\n", signum);
+		vdpa_sample_quit();
+		exit(0);
+	}
+}
+
+/* interactive cmds */
+
+/* *** Help command with introduction. *** */
+struct cmd_help_result {
+	cmdline_fixed_string_t help;
+};
+
+static void cmd_help_parsed(__attribute__((unused)) void *parsed_result,
+		struct cmdline *cl,
+		__attribute__((unused)) void *data)
+{
+	cmdline_printf(
+		cl,
+		"\n"
+		"The following commands are currently available:\n\n"
+		"Control:\n"
+		"    help                                      : Show interactive instructions.\n"
+		"    list                                      : list all available vdpa devices.\n"
+		"    create <socket file> <vdev addr>          : create a new vdpa port.\n"
+		"    quit                                      : exit vdpa sample app.\n"
+	);
+}
+
+cmdline_parse_token_string_t cmd_help_help =
+	TOKEN_STRING_INITIALIZER(struct cmd_help_result, help, "help");
+
+cmdline_parse_inst_t cmd_help = {
+	.f = cmd_help_parsed,
+	.data = NULL,
+	.help_str = "show help",
+	.tokens = {
+		(void *)&cmd_help_help,
+		NULL,
+	},
+};
+
+/* *** List all available vdpa devices *** */
+struct cmd_list_result {
+	cmdline_fixed_string_t action;
+};
+
+static void cmd_list_vdpa_devices_parsed(
+		__attribute__((unused)) void *parsed_result,
+		struct cmdline *cl,
+		__attribute__((unused)) void *data)
+{
+	int did;
+	uint32_t queue_num;
+	uint64_t features;
+	struct rte_vdpa_device *vdev;
+	struct rte_pci_addr addr;
+
+	cmdline_printf(cl, "device id\tdevice address\tqueue num\tsupported features\n");
+	for (did = 0; did < dev_total; did++) {
+		vdev = rte_vdpa_get_device(did);
+		if (!vdev)
+			continue;
+		if (vdev->ops->get_queue_num(did, &queue_num) < 0) {
+			RTE_LOG(ERR, VDPA,
+				"failed to get vdpa queue number "
+				"for device id %d.\n", did);
+			continue;
+		}
+		if (vdev->ops->get_features(did, &features) < 0) {
+			RTE_LOG(ERR, VDPA,
+				"failed to get vdpa features "
+				"for device id %d.\n", did);
+			continue;
+		}
+		addr = vdev->addr.pci_addr;
+		cmdline_printf(cl,
+			"%d\t\t" PCI_PRI_FMT "\t%" PRIu32 "\t\t0x%" PRIx64 "\n",
+			did, addr.domain, addr.bus, addr.devid,
+			addr.function, queue_num, features);
+	}
+}
+
+cmdline_parse_token_string_t cmd_action_list =
+	TOKEN_STRING_INITIALIZER(struct cmd_list_result, action, "list");
+
+cmdline_parse_inst_t cmd_list_vdpa_devices = {
+	.f = cmd_list_vdpa_devices_parsed,
+	.data = NULL,
+	.help_str = "list all available vdpa devices",
+	.tokens = {
+		(void *)&cmd_action_list,
+		NULL,
+	},
+};
+
+/* *** Create new vdpa port *** */
+struct cmd_create_result {
+	cmdline_fixed_string_t action;
+	cmdline_fixed_string_t socket_path;
+	cmdline_fixed_string_t bdf;
+};
+
+static void cmd_create_vdpa_port_parsed(void *parsed_result,
+		struct cmdline *cl,
+		__attribute__((unused)) void *data)
+{
+	int did;
+	struct cmd_create_result *res = parsed_result;
+	struct rte_vdpa_dev_addr addr;
+
+	rte_strscpy(vports[devcnt].ifname, res->socket_path, MAX_PATH_LEN);
+	if (rte_pci_addr_parse(res->bdf, &addr.pci_addr) != 0) {
+		cmdline_printf(cl, "Unable to parse the given bdf.\n");
+		return;
+	}
+	addr.type = PCI_ADDR;
+	did = rte_vdpa_find_device_id(&addr);
+	if (did < 0) {
+		cmdline_printf(cl, "Unable to find vdpa device id.\n");
+		return;
+	}
+
+	vports[devcnt].did = did;
+
+	if (start_vdpa(&vports[devcnt]) == 0)
+		devcnt++;
+}
+
+cmdline_parse_token_string_t cmd_action_create =
+	TOKEN_STRING_INITIALIZER(struct cmd_create_result, action, "create");
+cmdline_parse_token_string_t cmd_socket_path =
+	TOKEN_STRING_INITIALIZER(struct cmd_create_result, socket_path, NULL);
+cmdline_parse_token_string_t cmd_bdf =
+	TOKEN_STRING_INITIALIZER(struct cmd_create_result, bdf, NULL);
+
+cmdline_parse_inst_t cmd_create_vdpa_port = {
+	.f = cmd_create_vdpa_port_parsed,
+	.data = NULL,
+	.help_str = "create a new vdpa port",
+	.tokens = {
+		(void *)&cmd_action_create,
+		(void *)&cmd_socket_path,
+		(void *)&cmd_bdf,
+		NULL,
+	},
+};
+
+/* *** QUIT *** */
+struct cmd_quit_result {
+	cmdline_fixed_string_t quit;
+};
+
+static void cmd_quit_parsed(__attribute__((unused)) void *parsed_result,
+		struct cmdline *cl,
+		__attribute__((unused)) void *data)
+{
+	vdpa_sample_quit();
+	cmdline_quit(cl);
+}
+
+cmdline_parse_token_string_t cmd_quit_quit =
+	TOKEN_STRING_INITIALIZER(struct cmd_quit_result, quit, "quit");
+
+cmdline_parse_inst_t cmd_quit = {
+	.f = cmd_quit_parsed,
+	.data = NULL,
+	.help_str = "quit: exit application",
+	.tokens = {
+		(void *)&cmd_quit_quit,
+		NULL,
+	},
+};
+cmdline_parse_ctx_t main_ctx[] = {
+	(cmdline_parse_inst_t *)&cmd_help,
+	(cmdline_parse_inst_t *)&cmd_list_vdpa_devices,
+	(cmdline_parse_inst_t *)&cmd_create_vdpa_port,
+	(cmdline_parse_inst_t *)&cmd_quit,
+	NULL,
+};
+
+int
+main(int argc, char *argv[])
+{
+	char ch;
+	int i;
+	int ret;
+	struct cmdline *cl;
+
+	ret = rte_eal_init(argc, argv);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE, "eal init failed\n");
+	argc -= ret;
+	argv += ret;
+
+	dev_total = rte_vdpa_get_device_num();
+	if (dev_total <= 0)
+		rte_exit(EXIT_FAILURE, "No available vdpa device found\n");
+
+	signal(SIGINT, signal_handler);
+	signal(SIGTERM, signal_handler);
+
+	ret = parse_args(argc, argv);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE, "invalid argument\n");
+
+	if (interactive == 1) {
+		cl = cmdline_stdin_new(main_ctx, "vdpa> ");
+		if (cl == NULL)
+			rte_panic("Cannot create cmdline instance\n");
+		cmdline_interact(cl);
+		cmdline_stdin_exit(cl);
+	} else {
+		for (i = 0; i < RTE_MIN(MAX_VDPA_SAMPLE_PORTS, dev_total);
+				i++) {
+			vports[i].did = i;
+			snprintf(vports[i].ifname, MAX_PATH_LEN, "%s%d",
+					iface, i);
+
+			start_vdpa(&vports[i]);
+		}
+
+		printf("enter \'q\' to quit\n");
+		while (scanf("%c", &ch)) {
+			if (ch == 'q')
+				break;
+			while (ch != '\n') {
+				if (scanf("%c", &ch))
+					printf("%c", ch);
+			}
+			printf("enter \'q\' to quit\n");
+		}
+		vdpa_sample_quit();
+	}
+
+	return 0;
+}
diff --git a/examples/vdpa/meson.build b/examples/vdpa/meson.build
new file mode 100644
index 00000000..2e38a069
--- /dev/null
+++ b/examples/vdpa/meson.build
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+# meson file, for building this example as part of a main DPDK build.
+#
+# To build this example as a standalone application with an already-installed
+# DPDK instance, use 'make'
+
+if host_machine.system() != 'linux'
+	build = false
+endif
+deps += 'vhost'
+allow_experimental_apis = true
+sources = files(
+	'main.c'
+)
\ No newline at end of file
diff --git a/examples/vhost/Makefile b/examples/vhost/Makefile
index a2ea97a0..c6964381 100644
--- a/examples/vhost/Makefile
+++ b/examples/vhost/Makefile
@@ -61,7 +61,6 @@ else
 CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O2 -D_FILE_OFFSET_BITS=64
 CFLAGS += $(WERROR_FLAGS)
-CFLAGS += -D_GNU_SOURCE
 
 include $(RTE_SDK)/mk/rte.extapp.mk
 
diff --git a/examples/vhost/main.c b/examples/vhost/main.c
index 2175c118..dc9ea101 100644
--- a/examples/vhost/main.c
+++ b/examples/vhost/main.c
@@ -58,9 +58,6 @@
 /* Max number of devices. Limited by vmdq. */
 #define MAX_DEVICES 64
 
-/* Size of buffers used for snprintfs. */
-#define MAX_PRINT_BUFF 6072
-
 /* Maximum long option length for option parsing. */
 #define MAX_LONG_OPT_SZ 64
 
@@ -121,8 +118,7 @@ static struct rte_eth_conf vmdq_conf_default = {
 		 * this fixes bug of ipv4 forwarding in guest can't
 		 * forward pakets from one virtio dev to another virtio dev.
 		 */
-		.offloads = (DEV_RX_OFFLOAD_CRC_STRIP |
-			     DEV_RX_OFFLOAD_VLAN_STRIP),
+		.offloads = DEV_RX_OFFLOAD_VLAN_STRIP,
 	},
 
 	.txmode = {
diff --git a/examples/vhost_crypto/Makefile b/examples/vhost_crypto/Makefile
index 83d33101..a620abf4 100644
--- a/examples/vhost_crypto/Makefile
+++ b/examples/vhost_crypto/Makefile
@@ -25,7 +25,6 @@ SRCS-y := main.c
 CFLAGS += -DALLOW_EXPERIMENTAL_API
 CFLAGS += -O2 -D_FILE_OFFSET_BITS=64
 CFLAGS += $(WERROR_FLAGS)
-CFLAGS += -D_GNU_SOURCE
 
 include $(RTE_SDK)/mk/rte.extapp.mk
 
diff --git a/examples/vhost_crypto/main.c b/examples/vhost_crypto/main.c
index f334d712..cbb5e49d 100644
--- a/examples/vhost_crypto/main.c
+++ b/examples/vhost_crypto/main.c
@@ -14,6 +14,7 @@
 #include <rte_vhost.h>
 #include <rte_cryptodev.h>
 #include <rte_vhost_crypto.h>
+#include <rte_string_fns.h>
 
 #include <cmdline_rdline.h>
 #include <cmdline_parse.h>
@@ -29,97 +30,161 @@
 #define SESSION_MAP_ENTRIES		(1024)
 #define REFRESH_TIME_SEC		(3)
 
-#define MAX_NB_SOCKETS			(32)
-#define DEF_SOCKET_FILE			"/tmp/vhost_crypto1.socket"
+#define MAX_NB_SOCKETS			(4)
+#define MAX_NB_WORKER_CORES		(16)
 
-struct vhost_crypto_options {
+struct lcore_option {
+	uint32_t lcore_id;
 	char *socket_files[MAX_NB_SOCKETS];
 	uint32_t nb_sockets;
 	uint8_t cid;
 	uint16_t qid;
-	uint32_t zero_copy;
-	uint32_t guest_polling;
-} options;
+};
 
 struct vhost_crypto_info {
 	int vids[MAX_NB_SOCKETS];
+	uint32_t nb_vids;
 	struct rte_mempool *sess_pool;
 	struct rte_mempool *cop_pool;
-	uint32_t lcore_id;
 	uint8_t cid;
 	uint32_t qid;
-	uint32_t nb_vids;
+	uint32_t nb_inflight_ops;
 	volatile uint32_t initialized[MAX_NB_SOCKETS];
+} __rte_cache_aligned;
 
-} info;
+struct vhost_crypto_options {
+	struct lcore_option los[MAX_NB_WORKER_CORES];
+	struct vhost_crypto_info *infos[MAX_NB_WORKER_CORES];
+	uint32_t nb_los;
+	uint32_t zero_copy;
+	uint32_t guest_polling;
+} options;
 
+#define CONFIG_KEYWORD		"config"
 #define SOCKET_FILE_KEYWORD	"socket-file"
-#define CRYPTODEV_ID_KEYWORD	"cdev-id"
-#define CRYPTODEV_QUEUE_KEYWORD	"cdev-queue-id"
 #define ZERO_COPY_KEYWORD	"zero-copy"
 #define POLLING_KEYWORD		"guest-polling"
 
-uint64_t vhost_cycles[2], last_v_cycles[2];
-uint64_t outpkt_amount;
+#define NB_SOCKET_FIELDS	(2)
+
+static uint32_t
+find_lo(uint32_t lcore_id)
+{
+	uint32_t i;
+
+	for (i = 0; i < options.nb_los; i++)
+		if (options.los[i].lcore_id == lcore_id)
+			return i;
+
+	return UINT32_MAX;
+}
 
 /** support *SOCKET_FILE_PATH:CRYPTODEV_ID* format */
 static int
 parse_socket_arg(char *arg)
 {
-	uint32_t nb_sockets = options.nb_sockets;
-	size_t len = strlen(arg);
+	uint32_t nb_sockets;
+	uint32_t lcore_id;
+	char *str_fld[NB_SOCKET_FIELDS];
+	struct lcore_option *lo;
+	uint32_t idx;
+	char *end;
+
+	if (rte_strsplit(arg, strlen(arg), str_fld, NB_SOCKET_FIELDS, ',') !=
+				NB_SOCKET_FIELDS) {
+		RTE_LOG(ERR, USER1, "Invalid socket parameter '%s'\n", arg);
+		return -EINVAL;
+	}
+
+	errno = 0;
+	lcore_id = strtoul(str_fld[0], &end, 0);
+	if (errno != 0 || end == str_fld[0] || lcore_id > 255)
+		return -EINVAL;
+
+	idx = find_lo(lcore_id);
+	if (idx == UINT32_MAX) {
+		if (options.nb_los == MAX_NB_WORKER_CORES)
+			return -ENOMEM;
+		lo = &options.los[options.nb_los];
+		lo->lcore_id = lcore_id;
+		options.nb_los++;
+	} else
+		lo = &options.los[idx];
+
+	nb_sockets = lo->nb_sockets;
 
 	if (nb_sockets >= MAX_NB_SOCKETS) {
 		RTE_LOG(ERR, USER1, "Too many socket files!\n");
 		return -ENOMEM;
 	}
 
-	options.socket_files[nb_sockets] = rte_malloc(NULL, len, 0);
-	if (!options.socket_files[nb_sockets]) {
+	lo->socket_files[nb_sockets] = strdup(str_fld[1]);
+	if (!lo->socket_files[nb_sockets]) {
 		RTE_LOG(ERR, USER1, "Insufficient memory\n");
 		return -ENOMEM;
 	}
 
-	rte_memcpy(options.socket_files[nb_sockets], arg, len);
-
-	options.nb_sockets++;
+	lo->nb_sockets++;
 
 	return 0;
 }
 
 static int
-parse_cryptodev_id(const char *q_arg)
+parse_config(char *q_arg)
 {
-	char *end = NULL;
-	uint64_t pm;
-
-	/* parse decimal string */
-	pm = strtoul(q_arg, &end, 10);
-	if (pm > rte_cryptodev_count()) {
-		RTE_LOG(ERR, USER1, "Invalid Cryptodev ID %s\n", q_arg);
-		return -1;
-	}
+	struct lcore_option *lo;
+	char s[256];
+	const char *p, *p0 = q_arg;
+	char *end;
+	enum fieldnames {
+		FLD_LCORE = 0,
+		FLD_CID,
+		FLD_QID,
+		_NUM_FLD
+	};
+	uint32_t flds[_NUM_FLD];
+	char *str_fld[_NUM_FLD];
+	uint32_t i;
+	uint32_t size;
 
-	options.cid = (uint8_t)pm;
+	while ((p = strchr(p0, '(')) != NULL) {
+		++p;
+		p0 = strchr(p, ')');
+		if (p0 == NULL)
+			return -1;
 
-	return 0;
-}
+		size = p0 - p;
+		if (size >= sizeof(s))
+			return -1;
 
-static int
-parse_cdev_queue_id(const char *q_arg)
-{
-	char *end = NULL;
-	uint64_t pm;
+		snprintf(s, sizeof(s), "%.*s", size, p);
+		if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') !=
+				_NUM_FLD)
+			return -1;
+		for (i = 0; i < _NUM_FLD; i++) {
+			errno = 0;
+			flds[i] = strtoul(str_fld[i], &end, 0);
+			if (errno != 0 || end == str_fld[i] || flds[i] > 255)
+				return -EINVAL;
+		}
 
-	/* parse decimal string */
-	pm = strtoul(q_arg, &end, 10);
-	if (pm == UINT64_MAX) {
-		RTE_LOG(ERR, USER1, "Invalid Cryptodev Queue ID %s\n", q_arg);
-		return -1;
+		if (flds[FLD_LCORE] > RTE_MAX_LCORE)
+			return -EINVAL;
+
+		i = find_lo(flds[FLD_LCORE]);
+		if (i == UINT32_MAX) {
+			if (options.nb_los == MAX_NB_WORKER_CORES)
+				return -ENOMEM;
+			lo = &options.los[options.nb_los];
+			options.nb_los++;
+		} else
+			lo = &options.los[i];
+
+		lo->lcore_id = flds[FLD_LCORE];
+		lo->cid = flds[FLD_CID];
+		lo->qid = flds[FLD_QID];
 	}
 
-	options.qid = (uint16_t)pm;
-
 	return 0;
 }
 
@@ -127,13 +192,12 @@ static void
 vhost_crypto_usage(const char *prgname)
 {
 	printf("%s [EAL options] --\n"
-		"  --%s SOCKET-FILE-PATH\n"
-		"  --%s CRYPTODEV_ID: crypto device id\n"
-		"  --%s CDEV_QUEUE_ID: crypto device queue id\n"
+		"  --%s <lcore>,SOCKET-FILE-PATH\n"
+		"  --%s (lcore,cdev_id,queue_id)[,(lcore,cdev_id,queue_id)]"
 		"  --%s: zero copy\n"
 		"  --%s: guest polling\n",
-		prgname, SOCKET_FILE_KEYWORD, CRYPTODEV_ID_KEYWORD,
-		CRYPTODEV_QUEUE_KEYWORD, ZERO_COPY_KEYWORD, POLLING_KEYWORD);
+		prgname, SOCKET_FILE_KEYWORD, CONFIG_KEYWORD,
+		ZERO_COPY_KEYWORD, POLLING_KEYWORD);
 }
 
 static int
@@ -145,19 +209,12 @@ vhost_crypto_parse_args(int argc, char **argv)
 	int option_index;
 	struct option lgopts[] = {
 			{SOCKET_FILE_KEYWORD, required_argument, 0, 0},
-			{CRYPTODEV_ID_KEYWORD, required_argument, 0, 0},
-			{CRYPTODEV_QUEUE_KEYWORD, required_argument, 0, 0},
+			{CONFIG_KEYWORD, required_argument, 0, 0},
 			{ZERO_COPY_KEYWORD, no_argument, 0, 0},
 			{POLLING_KEYWORD, no_argument, 0, 0},
 			{NULL, 0, 0, 0}
 	};
 
-	options.cid = 0;
-	options.qid = 0;
-	options.nb_sockets = 0;
-	options.guest_polling = 0;
-	options.zero_copy = RTE_VHOST_CRYPTO_ZERO_COPY_DISABLE;
-
 	argvopt = argv;
 
 	while ((opt = getopt_long(argc, argvopt, "s:",
@@ -173,15 +230,8 @@ vhost_crypto_parse_args(int argc, char **argv)
 					return ret;
 				}
 			} else if (strcmp(lgopts[option_index].name,
-					CRYPTODEV_ID_KEYWORD) == 0) {
-				ret = parse_cryptodev_id(optarg);
-				if (ret < 0) {
-					vhost_crypto_usage(prgname);
-					return ret;
-				}
-			} else if (strcmp(lgopts[option_index].name,
-					CRYPTODEV_QUEUE_KEYWORD) == 0) {
-				ret = parse_cdev_queue_id(optarg);
+					CONFIG_KEYWORD) == 0) {
+				ret = parse_config(optarg);
 				if (ret < 0) {
 					vhost_crypto_usage(prgname);
 					return ret;
@@ -203,22 +253,15 @@ vhost_crypto_parse_args(int argc, char **argv)
 		}
 	}
 
-	if (options.nb_sockets == 0) {
-		options.socket_files[0] = strdup(DEF_SOCKET_FILE);
-		options.nb_sockets = 1;
-		RTE_LOG(INFO, USER1,
-				"VHOST-CRYPTO: use default socket file %s\n",
-				DEF_SOCKET_FILE);
-	}
-
 	return 0;
 }
 
 static int
 new_device(int vid)
 {
+	struct vhost_crypto_info *info = NULL;
 	char path[PATH_MAX];
-	uint32_t idx, i;
+	uint32_t i, j;
 	int ret;
 
 	ret = rte_vhost_get_ifname(vid, path, PATH_MAX);
@@ -227,23 +270,25 @@ new_device(int vid)
 		return ret;
 	}
 
-	for (idx = 0; idx < options.nb_sockets; idx++) {
-		if (strcmp(path, options.socket_files[idx]) == 0)
+	for (i = 0; i < options.nb_los; i++) {
+		for (j = 0; j < options.los[i].nb_sockets; j++) {
+			if (strcmp(path, options.los[i].socket_files[j]) == 0) {
+				info = options.infos[i];
+				break;
+			}
+		}
+
+		if (info)
 			break;
 	}
 
-	if (idx == options.nb_sockets) {
+	if (!info) {
 		RTE_LOG(ERR, USER1, "Cannot find recorded socket\n");
 		return -ENOENT;
 	}
 
-	for (i = 0; i < 2; i++) {
-		vhost_cycles[i] = 0;
-		last_v_cycles[i] = 0;
-	}
-
-	ret = rte_vhost_crypto_create(vid, info.cid, info.sess_pool,
-			rte_lcore_to_socket_id(info.lcore_id));
+	ret = rte_vhost_crypto_create(vid, info->cid, info->sess_pool,
+			rte_lcore_to_socket_id(options.los[i].lcore_id));
 	if (ret) {
 		RTE_LOG(ERR, USER1, "Cannot create vhost crypto\n");
 		return ret;
@@ -256,8 +301,8 @@ new_device(int vid)
 		return ret;
 	}
 
-	info.vids[idx] = vid;
-	info.initialized[idx] = 1;
+	info->vids[j] = vid;
+	info->initialized[j] = 1;
 
 	rte_wmb();
 
@@ -269,19 +314,30 @@ new_device(int vid)
 static void
 destroy_device(int vid)
 {
-	uint32_t i;
-
-	for (i = 0; i < info.nb_vids; i++) {
-		if (vid == info.vids[i])
+	struct vhost_crypto_info *info = NULL;
+	uint32_t i, j;
+
+	for (i = 0; i < options.nb_los; i++) {
+		for (j = 0; j < options.los[i].nb_sockets; j++) {
+			if (options.infos[i]->vids[j] == vid) {
+				info = options.infos[i];
+				break;
+			}
+		}
+		if (info)
 			break;
 	}
 
-	if (i == info.nb_vids) {
+	if (!info) {
 		RTE_LOG(ERR, USER1, "Cannot find socket file from list\n");
 		return;
 	}
 
-	info.initialized[i] = 0;
+	do {
+
+	} while (info->nb_inflight_ops);
+
+	info->initialized[j] = 0;
 
 	rte_wmb();
 
@@ -302,25 +358,24 @@ static void clrscr(void)
 }
 
 static int
-vhost_crypto_worker(__rte_unused void *arg)
+vhost_crypto_worker(void *arg)
 {
 	struct rte_crypto_op *ops[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
 	struct rte_crypto_op *ops_deq[NB_VIRTIO_QUEUES][MAX_PKT_BURST + 1];
-	uint32_t nb_inflight_ops = 0;
+	struct vhost_crypto_info *info = arg;
 	uint16_t nb_callfds;
 	int callfds[VIRTIO_CRYPTO_MAX_NUM_BURST_VQS];
 	uint32_t lcore_id = rte_lcore_id();
 	uint32_t burst_size = MAX_PKT_BURST;
 	uint32_t i, j, k;
 	uint32_t to_fetch, fetched;
-	uint64_t t_start, t_end, interval;
 
 	int ret = 0;
 
 	RTE_LOG(INFO, USER1, "Processing on Core %u started\n", lcore_id);
 
 	for (i = 0; i < NB_VIRTIO_QUEUES; i++) {
-		if (rte_crypto_op_bulk_alloc(info.cop_pool,
+		if (rte_crypto_op_bulk_alloc(info->cop_pool,
 				RTE_CRYPTO_OP_TYPE_SYMMETRIC, ops[i],
 				burst_size) < burst_size) {
 			RTE_LOG(ERR, USER1, "Failed to alloc cops\n");
@@ -330,45 +385,38 @@ vhost_crypto_worker(__rte_unused void *arg)
 	}
 
 	while (1) {
-		for (i = 0; i < info.nb_vids; i++) {
-			if (unlikely(info.initialized[i] == 0))
+		for (i = 0; i < info->nb_vids; i++) {
+			if (unlikely(info->initialized[i] == 0))
 				continue;
 
 			for (j = 0; j < NB_VIRTIO_QUEUES; j++) {
-				t_start = rte_rdtsc_precise();
-
 				to_fetch = RTE_MIN(burst_size,
 						(NB_CRYPTO_DESCRIPTORS -
-						nb_inflight_ops));
+						info->nb_inflight_ops));
 				fetched = rte_vhost_crypto_fetch_requests(
-						info.vids[i], j, ops[j],
+						info->vids[i], j, ops[j],
 						to_fetch);
-				nb_inflight_ops += rte_cryptodev_enqueue_burst(
-						info.cid, info.qid, ops[j],
+				info->nb_inflight_ops +=
+						rte_cryptodev_enqueue_burst(
+						info->cid, info->qid, ops[j],
 						fetched);
 				if (unlikely(rte_crypto_op_bulk_alloc(
-						info.cop_pool,
+						info->cop_pool,
 						RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 						ops[j], fetched) < fetched)) {
 					RTE_LOG(ERR, USER1, "Failed realloc\n");
 					return -1;
 				}
-				t_end = rte_rdtsc_precise();
-				interval = t_end - t_start;
-
-				vhost_cycles[fetched > 0] += interval;
 
-				t_start = t_end;
 				fetched = rte_cryptodev_dequeue_burst(
-						info.cid, info.qid,
+						info->cid, info->qid,
 						ops_deq[j], RTE_MIN(burst_size,
-						nb_inflight_ops));
+						info->nb_inflight_ops));
 				fetched = rte_vhost_crypto_finalize_requests(
 						ops_deq[j], fetched, callfds,
 						&nb_callfds);
 
-				nb_inflight_ops -= fetched;
-				outpkt_amount += fetched;
+				info->nb_inflight_ops -= fetched;
 
 				if (!options.guest_polling) {
 					for (k = 0; k < nb_callfds; k++)
@@ -376,11 +424,8 @@ vhost_crypto_worker(__rte_unused void *arg)
 								(eventfd_t)1);
 				}
 
-				rte_mempool_put_bulk(info.cop_pool,
+				rte_mempool_put_bulk(info->cop_pool,
 						(void **)ops_deq[j], fetched);
-				interval = rte_rdtsc_precise() - t_start;
-
-				vhost_cycles[fetched > 0] += interval;
 			}
 		}
 	}
@@ -388,17 +433,27 @@ exit:
 	return ret;
 }
 
-
 static void
-unregister_drivers(int socket_num)
+free_resource(void)
 {
-	int ret;
+	uint32_t i, j;
+
+	for (i = 0; i < options.nb_los; i++) {
+		struct lcore_option *lo = &options.los[i];
+		struct vhost_crypto_info *info = options.infos[i];
+
+		rte_mempool_free(info->cop_pool);
+		rte_mempool_free(info->sess_pool);
+
+		for (j = 0; j < lo->nb_sockets; j++) {
+			rte_vhost_driver_unregister(lo->socket_files[i]);
+			free(lo->socket_files[i]);
+		}
+
+		rte_free(info);
+	}
 
-	ret = rte_vhost_driver_unregister(options.socket_files[socket_num]);
-	if (ret != 0)
-		RTE_LOG(ERR, USER1,
-			"Fail to unregister vhost driver for %s.\n",
-			options.socket_files[socket_num]);
+	memset(&options, 0, sizeof(options));
 }
 
 int
@@ -407,10 +462,8 @@ main(int argc, char *argv[])
 	struct rte_cryptodev_qp_conf qp_conf = {NB_CRYPTO_DESCRIPTORS};
 	struct rte_cryptodev_config config;
 	struct rte_cryptodev_info dev_info;
-	uint32_t cryptodev_id;
-	uint32_t worker_lcore;
 	char name[128];
-	uint32_t i = 0;
+	uint32_t i, j, lcore;
 	int ret;
 
 	ret = rte_eal_init(argc, argv);
@@ -423,114 +476,121 @@ main(int argc, char *argv[])
 	if (ret < 0)
 		rte_exit(EXIT_FAILURE, "Failed to parse arguments!\n");
 
-	info.cid = options.cid;
-	info.qid = options.qid;
+	for (i = 0; i < options.nb_los; i++) {
+		struct lcore_option *lo = &options.los[i];
+		struct vhost_crypto_info *info;
 
-	worker_lcore = rte_get_next_lcore(0, 1, 0);
-	if (worker_lcore == RTE_MAX_LCORE)
-		rte_exit(EXIT_FAILURE, "Not enough lcore\n");
-
-	cryptodev_id = info.cid;
-	rte_cryptodev_info_get(cryptodev_id, &dev_info);
-	if (dev_info.max_nb_queue_pairs < info.qid + 1) {
-		RTE_LOG(ERR, USER1, "Number of queues cannot over %u",
-				dev_info.max_nb_queue_pairs);
-		goto error_exit;
-	}
+		info = rte_zmalloc_socket(NULL, sizeof(*info),
+				RTE_CACHE_LINE_SIZE, rte_lcore_to_socket_id(
+						lo->lcore_id));
+		if (!info) {
+			ret = -ENOMEM;
+			goto error_exit;
+		}
 
-	config.nb_queue_pairs = dev_info.max_nb_queue_pairs;
-	config.socket_id = rte_lcore_to_socket_id(worker_lcore);
+		info->cid = lo->cid;
+		info->qid = lo->qid;
+		info->nb_vids = lo->nb_sockets;
 
-	ret = rte_cryptodev_configure(cryptodev_id, &config);
-	if (ret < 0) {
-		RTE_LOG(ERR, USER1, "Failed to configure cryptodev %u",
-				cryptodev_id);
-		goto error_exit;
-	}
+		rte_cryptodev_info_get(info->cid, &dev_info);
+		if (dev_info.max_nb_queue_pairs < info->qid + 1) {
+			RTE_LOG(ERR, USER1, "Number of queues cannot over %u",
+					dev_info.max_nb_queue_pairs);
+			goto error_exit;
+		}
 
-	snprintf(name, 127, "SESS_POOL_%u", worker_lcore);
-	info.sess_pool = rte_mempool_create(name, SESSION_MAP_ENTRIES,
-			rte_cryptodev_sym_get_private_session_size(
-			cryptodev_id), 64, 0, NULL, NULL, NULL, NULL,
-			rte_lcore_to_socket_id(worker_lcore), 0);
-	if (!info.sess_pool) {
-		RTE_LOG(ERR, USER1, "Failed to create mempool");
-		goto error_exit;
-	}
+		config.nb_queue_pairs = dev_info.max_nb_queue_pairs;
+		config.socket_id = rte_lcore_to_socket_id(lo->lcore_id);
 
-	snprintf(name, 127, "COPPOOL_%u", worker_lcore);
-	info.cop_pool = rte_crypto_op_pool_create(name,
-			RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MEMPOOL_OBJS,
-			NB_CACHE_OBJS, 0, rte_lcore_to_socket_id(worker_lcore));
+		ret = rte_cryptodev_configure(info->cid, &config);
+		if (ret < 0) {
+			RTE_LOG(ERR, USER1, "Failed to configure cryptodev %u",
+					info->cid);
+			goto error_exit;
+		}
 
-	if (!info.cop_pool) {
-		RTE_LOG(ERR, USER1, "Lcore %u failed to create crypto pool",
-				worker_lcore);
-		ret = -1;
-		goto error_exit;
-	}
+		snprintf(name, 127, "SESS_POOL_%u", lo->lcore_id);
+		info->sess_pool = rte_mempool_create(name, SESSION_MAP_ENTRIES,
+				rte_cryptodev_sym_get_private_session_size(
+				info->cid), 64, 0, NULL, NULL, NULL, NULL,
+				rte_lcore_to_socket_id(lo->lcore_id), 0);
+		if (!info->sess_pool) {
+			RTE_LOG(ERR, USER1, "Failed to create mempool");
+			goto error_exit;
+		}
 
-	info.nb_vids = options.nb_sockets;
-	for (i = 0; i < MAX_NB_SOCKETS; i++)
-		info.vids[i] = -1;
+		snprintf(name, 127, "COPPOOL_%u", lo->lcore_id);
+		info->cop_pool = rte_crypto_op_pool_create(name,
+				RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MEMPOOL_OBJS,
+				NB_CACHE_OBJS, 0,
+				rte_lcore_to_socket_id(lo->lcore_id));
 
-	for (i = 0; i < dev_info.max_nb_queue_pairs; i++) {
-		ret = rte_cryptodev_queue_pair_setup(cryptodev_id, i,
-				&qp_conf, rte_lcore_to_socket_id(worker_lcore),
-				info.sess_pool);
-		if (ret < 0) {
-			RTE_LOG(ERR, USER1, "Failed to configure qp %u\n",
-					info.cid);
+		if (!info->cop_pool) {
+			RTE_LOG(ERR, USER1, "Failed to create crypto pool");
+			ret = -ENOMEM;
 			goto error_exit;
 		}
-	}
 
-	ret = rte_cryptodev_start(cryptodev_id);
-	if (ret < 0) {
-		RTE_LOG(ERR, USER1, "Failed to start cryptodev %u\n", info.cid);
-		goto error_exit;
+		options.infos[i] = info;
+
+		for (j = 0; j < dev_info.max_nb_queue_pairs; j++) {
+			ret = rte_cryptodev_queue_pair_setup(info->cid, j,
+					&qp_conf, rte_lcore_to_socket_id(
+							lo->lcore_id),
+					info->sess_pool);
+			if (ret < 0) {
+				RTE_LOG(ERR, USER1, "Failed to configure qp\n");
+				goto error_exit;
+			}
+		}
 	}
 
-	info.cid = cryptodev_id;
-	info.lcore_id = worker_lcore;
+	for (i = 0; i < options.nb_los; i++) {
+		struct lcore_option *lo = &options.los[i];
+		struct vhost_crypto_info *info = options.infos[i];
 
-	if (rte_eal_remote_launch(vhost_crypto_worker, NULL, worker_lcore)
-			< 0) {
-		RTE_LOG(ERR, USER1, "Failed to start worker lcore");
-		goto error_exit;
-	}
+		ret = rte_cryptodev_start(lo->cid);
+		if (ret < 0) {
+			RTE_LOG(ERR, USER1, "Failed to start cryptodev\n");
+			goto error_exit;
+		}
 
-	for (i = 0; i < options.nb_sockets; i++) {
-		if (rte_vhost_driver_register(options.socket_files[i],
-				RTE_VHOST_USER_DEQUEUE_ZERO_COPY) < 0) {
-			RTE_LOG(ERR, USER1, "socket %s already exists\n",
-					options.socket_files[i]);
+		if (rte_eal_remote_launch(vhost_crypto_worker, info,
+				lo->lcore_id) < 0) {
+			RTE_LOG(ERR, USER1, "Failed to start worker lcore");
 			goto error_exit;
 		}
 
-		rte_vhost_driver_callback_register(options.socket_files[i],
+		for (j = 0; j < lo->nb_sockets; j++) {
+			ret = rte_vhost_driver_register(lo->socket_files[j],
+				RTE_VHOST_USER_DEQUEUE_ZERO_COPY);
+			if (ret < 0) {
+				RTE_LOG(ERR, USER1, "socket %s already exists\n",
+					lo->socket_files[j]);
+				goto error_exit;
+			}
+
+			rte_vhost_driver_callback_register(lo->socket_files[j],
 				&virtio_crypto_device_ops);
 
-		if (rte_vhost_driver_start(options.socket_files[i]) < 0) {
-			RTE_LOG(ERR, USER1, "failed to start vhost driver.\n");
-			goto error_exit;
+			ret = rte_vhost_driver_start(lo->socket_files[j]);
+			if (ret < 0)  {
+				RTE_LOG(ERR, USER1, "failed to start vhost.\n");
+				goto error_exit;
+			}
 		}
 	}
 
-	RTE_LCORE_FOREACH(worker_lcore)
-		rte_eal_wait_lcore(worker_lcore);
+	RTE_LCORE_FOREACH(lcore)
+		rte_eal_wait_lcore(lcore);
 
-	rte_mempool_free(info.sess_pool);
-	rte_mempool_free(info.cop_pool);
+	free_resource();
 
 	return 0;
 
 error_exit:
-	for (i = 0; i < options.nb_sockets; i++)
-		unregister_drivers(i);
 
-	rte_mempool_free(info.cop_pool);
-	rte_mempool_free(info.sess_pool);
+	free_resource();
 
 	return -1;
 }
diff --git a/examples/vhost_crypto/meson.build b/examples/vhost_crypto/meson.build
index 0f4876f0..daf19fb8 100644
--- a/examples/vhost_crypto/meson.build
+++ b/examples/vhost_crypto/meson.build
@@ -8,7 +8,7 @@
 
 allow_experimental_apis = true
 deps += ['vhost', 'cryptodev']
-cflags += ['-D_GNU_SOURCE','-D_FILE_OFFSET_BITS=64']
+cflags += ['-D_FILE_OFFSET_BITS=64']
 sources = files(
 	'main.c'
 )
diff --git a/examples/vhost_scsi/Makefile b/examples/vhost_scsi/Makefile
index fa0cf727..523aee0b 100644
--- a/examples/vhost_scsi/Makefile
+++ b/examples/vhost_scsi/Makefile
@@ -18,7 +18,7 @@ shared: build/$(APP)-shared
 static: build/$(APP)-static
 	ln -sf $(APP)-static build/$(APP)
 
-CFLAGS += -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64
+CFLAGS += -D_FILE_OFFSET_BITS=64
 LDFLAGS += -pthread
 
 PC_FILE := $(shell pkg-config --path libdpdk)
@@ -57,7 +57,7 @@ please change the definition of the RTE_TARGET environment variable)
 all:
 else
 
-CFLAGS += -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64
+CFLAGS += -D_FILE_OFFSET_BITS=64
 CFLAGS += -O2
 CFLAGS += $(WERROR_FLAGS)
 
diff --git a/examples/vhost_scsi/meson.build b/examples/vhost_scsi/meson.build
index 5f92370f..2303bcae 100644
--- a/examples/vhost_scsi/meson.build
+++ b/examples/vhost_scsi/meson.build
@@ -10,7 +10,7 @@ if host_machine.system() != 'linux'
 	build = false
 endif
 deps += 'vhost'
-cflags += ['-D_GNU_SOURCE','-D_FILE_OFFSET_BITS=64']
+cflags += ['-D_FILE_OFFSET_BITS=64']
 sources = files(
 	'scsi.c', 'vhost_scsi.c'
 )
diff --git a/examples/vm_power_manager/Makefile b/examples/vm_power_manager/Makefile
index 13a5205b..50147c05 100644
--- a/examples/vm_power_manager/Makefile
+++ b/examples/vm_power_manager/Makefile
@@ -31,6 +31,12 @@ CFLAGS += $(WERROR_FLAGS)
 
 LDLIBS += -lvirt
 
+JANSSON := $(shell pkg-config --exists jansson; echo $$?)
+ifeq ($(JANSSON), 0)
+LDLIBS += $(shell pkg-config --libs jansson)
+CFLAGS += -DUSE_JANSSON
+endif
+
 ifeq ($(CONFIG_RTE_BUILD_SHARED_LIB),y)
 
 ifeq ($(CONFIG_RTE_LIBRTE_IXGBE_PMD),y)
diff --git a/examples/vm_power_manager/channel_manager.c b/examples/vm_power_manager/channel_manager.c
index 927fc35a..4fac099d 100644
--- a/examples/vm_power_manager/channel_manager.c
+++ b/examples/vm_power_manager/channel_manager.c
@@ -13,6 +13,7 @@
 
 #include <sys/queue.h>
 #include <sys/types.h>
+#include <sys/stat.h>
 #include <sys/socket.h>
 #include <sys/select.h>
 
@@ -43,7 +44,8 @@ static unsigned char *global_cpumaps;
 static virVcpuInfo *global_vircpuinfo;
 static size_t global_maplen;
 
-static unsigned global_n_host_cpus;
+static unsigned int global_n_host_cpus;
+static bool global_hypervisor_available;
 
 /*
  * Represents a single Virtual Machine
@@ -198,7 +200,11 @@ get_pcpus_mask(struct channel_info *chan_info, unsigned vcpu)
 {
 	struct virtual_machine_info *vm_info =
 			(struct virtual_machine_info *)chan_info->priv_info;
-	return rte_atomic64_read(&vm_info->pcpu_mask[vcpu]);
+
+	if (global_hypervisor_available && (vm_info != NULL))
+		return rte_atomic64_read(&vm_info->pcpu_mask[vcpu]);
+	else
+		return 0;
 }
 
 static inline int
@@ -280,6 +286,38 @@ open_non_blocking_channel(struct channel_info *info)
 }
 
 static int
+open_host_channel(struct channel_info *info)
+{
+	int flags;
+
+	info->fd = open(info->channel_path, O_RDWR | O_RSYNC);
+	if (info->fd == -1) {
+		RTE_LOG(ERR, CHANNEL_MANAGER, "Error(%s) opening fifo for '%s'\n",
+				strerror(errno),
+				info->channel_path);
+		return -1;
+	}
+
+	/* Get current flags */
+	flags = fcntl(info->fd, F_GETFL, 0);
+	if (flags < 0) {
+		RTE_LOG(WARNING, CHANNEL_MANAGER, "Error(%s) fcntl get flags socket for"
+				"'%s'\n", strerror(errno), info->channel_path);
+		return 1;
+	}
+	/* Set to Non Blocking */
+	flags |= O_NONBLOCK;
+	if (fcntl(info->fd, F_SETFL, flags) < 0) {
+		RTE_LOG(WARNING, CHANNEL_MANAGER,
+				"Error(%s) setting non-blocking "
+				"socket for '%s'\n",
+				strerror(errno), info->channel_path);
+		return -1;
+	}
+	return 0;
+}
+
+static int
 setup_channel_info(struct virtual_machine_info **vm_info_dptr,
 		struct channel_info **chan_info_dptr, unsigned channel_num)
 {
@@ -289,6 +327,7 @@ setup_channel_info(struct virtual_machine_info **vm_info_dptr,
 	chan_info->channel_num = channel_num;
 	chan_info->priv_info = (void *)vm_info;
 	chan_info->status = CHANNEL_MGR_CHANNEL_DISCONNECTED;
+	chan_info->type = CHANNEL_TYPE_BINARY;
 	if (open_non_blocking_channel(chan_info) < 0) {
 		RTE_LOG(ERR, CHANNEL_MANAGER, "Could not open channel: "
 				"'%s' for VM '%s'\n",
@@ -311,6 +350,42 @@ setup_channel_info(struct virtual_machine_info **vm_info_dptr,
 	return 0;
 }
 
+static void
+fifo_path(char *dst, unsigned int len)
+{
+	snprintf(dst, len, "%sfifo", CHANNEL_MGR_SOCKET_PATH);
+}
+
+static int
+setup_host_channel_info(struct channel_info **chan_info_dptr,
+		unsigned int channel_num)
+{
+	struct channel_info *chan_info = *chan_info_dptr;
+
+	chan_info->channel_num = channel_num;
+	chan_info->priv_info = (void *)NULL;
+	chan_info->status = CHANNEL_MGR_CHANNEL_DISCONNECTED;
+	chan_info->type = CHANNEL_TYPE_JSON;
+
+	fifo_path(chan_info->channel_path, sizeof(chan_info->channel_path));
+
+	if (open_host_channel(chan_info) < 0) {
+		RTE_LOG(ERR, CHANNEL_MANAGER, "Could not open host channel: "
+				"'%s'\n",
+				chan_info->channel_path);
+		return -1;
+	}
+	if (add_channel_to_monitor(&chan_info) < 0) {
+		RTE_LOG(ERR, CHANNEL_MANAGER, "Could add channel: "
+				"'%s' to epoll ctl\n",
+				chan_info->channel_path);
+		return -1;
+
+	}
+	chan_info->status = CHANNEL_MGR_CHANNEL_CONNECTED;
+	return 0;
+}
+
 int
 add_all_channels(const char *vm_name)
 {
@@ -466,6 +541,45 @@ add_channels(const char *vm_name, unsigned *channel_list,
 }
 
 int
+add_host_channel(void)
+{
+	struct channel_info *chan_info;
+	char socket_path[PATH_MAX];
+	int num_channels_enabled = 0;
+	int ret;
+
+	fifo_path(socket_path, sizeof(socket_path));
+
+	ret = mkfifo(socket_path, 0660);
+	if ((errno != EEXIST) && (ret < 0)) {
+		RTE_LOG(ERR, CHANNEL_MANAGER, "Cannot create fifo '%s' error: "
+				"%s\n", socket_path, strerror(errno));
+		return 0;
+	}
+
+	if (access(socket_path, F_OK) < 0) {
+		RTE_LOG(ERR, CHANNEL_MANAGER, "Channel path '%s' error: "
+				"%s\n", socket_path, strerror(errno));
+		return 0;
+	}
+	chan_info = rte_malloc(NULL, sizeof(*chan_info), 0);
+	if (chan_info == NULL) {
+		RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for "
+				"channel '%s'\n", socket_path);
+		return 0;
+	}
+	snprintf(chan_info->channel_path,
+			sizeof(chan_info->channel_path), "%s", socket_path);
+	if (setup_host_channel_info(&chan_info, 0) < 0) {
+		rte_free(chan_info);
+		return 0;
+	}
+	num_channels_enabled++;
+
+	return num_channels_enabled;
+}
+
+int
 remove_channel(struct channel_info **chan_info_dptr)
 {
 	struct virtual_machine_info *vm_info;
@@ -559,6 +673,8 @@ get_all_vm(int *num_vm, int *num_vcpu)
 				VIR_CONNECT_LIST_DOMAINS_PERSISTENT;
 	unsigned int domain_flag = VIR_DOMAIN_VCPU_CONFIG;
 
+	if (!global_hypervisor_available)
+		return;
 
 	memset(global_cpumaps, 0, CHANNEL_CMDS_MAX_CPUS*global_maplen);
 	if (virNodeGetInfo(global_vir_conn_ptr, &node_info)) {
@@ -768,38 +884,42 @@ connect_hypervisor(const char *path)
 	}
 	return 0;
 }
-
 int
-channel_manager_init(const char *path)
+channel_manager_init(const char *path __rte_unused)
 {
 	virNodeInfo info;
 
 	LIST_INIT(&vm_list_head);
 	if (connect_hypervisor(path) < 0) {
-		RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to initialize channel manager\n");
-		return -1;
-	}
-
-	global_maplen = VIR_CPU_MAPLEN(CHANNEL_CMDS_MAX_CPUS);
+		global_n_host_cpus = 64;
+		global_hypervisor_available = 0;
+		RTE_LOG(INFO, CHANNEL_MANAGER, "Unable to initialize channel manager\n");
+	} else {
+		global_hypervisor_available = 1;
+
+		global_maplen = VIR_CPU_MAPLEN(CHANNEL_CMDS_MAX_CPUS);
+
+		global_vircpuinfo = rte_zmalloc(NULL,
+				sizeof(*global_vircpuinfo) *
+				CHANNEL_CMDS_MAX_CPUS, RTE_CACHE_LINE_SIZE);
+		if (global_vircpuinfo == NULL) {
+			RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for CPU Info\n");
+			goto error;
+		}
+		global_cpumaps = rte_zmalloc(NULL,
+				CHANNEL_CMDS_MAX_CPUS * global_maplen,
+				RTE_CACHE_LINE_SIZE);
+		if (global_cpumaps == NULL)
+			goto error;
 
-	global_vircpuinfo = rte_zmalloc(NULL, sizeof(*global_vircpuinfo) *
-			CHANNEL_CMDS_MAX_CPUS, RTE_CACHE_LINE_SIZE);
-	if (global_vircpuinfo == NULL) {
-		RTE_LOG(ERR, CHANNEL_MANAGER, "Error allocating memory for CPU Info\n");
-		goto error;
-	}
-	global_cpumaps = rte_zmalloc(NULL, CHANNEL_CMDS_MAX_CPUS * global_maplen,
-			RTE_CACHE_LINE_SIZE);
-	if (global_cpumaps == NULL) {
-		goto error;
+		if (virNodeGetInfo(global_vir_conn_ptr, &info)) {
+			RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n");
+			goto error;
+		}
+		global_n_host_cpus = (unsigned int)info.cpus;
 	}
 
-	if (virNodeGetInfo(global_vir_conn_ptr, &info)) {
-		RTE_LOG(ERR, CHANNEL_MANAGER, "Unable to retrieve node Info\n");
-		goto error;
-	}
 
-	global_n_host_cpus = (unsigned)info.cpus;
 
 	if (global_n_host_cpus > CHANNEL_CMDS_MAX_CPUS) {
 		RTE_LOG(WARNING, CHANNEL_MANAGER, "The number of host CPUs(%u) exceeds the "
@@ -811,7 +931,8 @@ channel_manager_init(const char *path)
 
 	return 0;
 error:
-	disconnect_hypervisor();
+	if (global_hypervisor_available)
+		disconnect_hypervisor();
 	return -1;
 }
 
@@ -838,7 +959,10 @@ channel_manager_exit(void)
 		rte_free(vm_info);
 	}
 
-	rte_free(global_cpumaps);
-	rte_free(global_vircpuinfo);
-	disconnect_hypervisor();
+	if (global_hypervisor_available) {
+		/* Only needed if hypervisor available */
+		rte_free(global_cpumaps);
+		rte_free(global_vircpuinfo);
+		disconnect_hypervisor();
+	}
 }
diff --git a/examples/vm_power_manager/channel_manager.h b/examples/vm_power_manager/channel_manager.h
index 872ec614..d948b304 100644
--- a/examples/vm_power_manager/channel_manager.h
+++ b/examples/vm_power_manager/channel_manager.h
@@ -37,7 +37,7 @@ struct sockaddr_un _sockaddr_un;
 #define UNIX_PATH_MAX sizeof(_sockaddr_un.sun_path)
 #endif
 
-#define MAX_VMS 4
+#define MAX_CLIENTS 64
 #define MAX_VCPUS 20
 
 
@@ -47,13 +47,20 @@ struct libvirt_vm_info {
 	uint8_t num_cpus;
 };
 
-struct libvirt_vm_info lvm_info[MAX_VMS];
+struct libvirt_vm_info lvm_info[MAX_CLIENTS];
 /* Communication Channel Status */
 enum channel_status { CHANNEL_MGR_CHANNEL_DISCONNECTED = 0,
 	CHANNEL_MGR_CHANNEL_CONNECTED,
 	CHANNEL_MGR_CHANNEL_DISABLED,
 	CHANNEL_MGR_CHANNEL_PROCESSING};
 
+/* Communication Channel Type */
+enum channel_type {
+	CHANNEL_TYPE_BINARY = 0,
+	CHANNEL_TYPE_INI,
+	CHANNEL_TYPE_JSON
+};
+
 /* VM libvirt(qemu/KVM) connection status */
 enum vm_status { CHANNEL_MGR_VM_INACTIVE = 0, CHANNEL_MGR_VM_ACTIVE};
 
@@ -66,6 +73,7 @@ struct channel_info {
 	volatile uint32_t status;    /**< Connection status(enum channel_status) */
 	int fd;                      /**< AF_UNIX socket fd */
 	unsigned channel_num;        /**< CHANNEL_MGR_SOCKET_PATH/<vm_name>.channel_num */
+	enum channel_type type;      /**< Binary, ini, json, etc. */
 	void *priv_info;             /**< Pointer to private info, do not modify */
 };
 
@@ -227,6 +235,15 @@ int add_channels(const char *vm_name, unsigned *channel_list,
 		unsigned num_channels);
 
 /**
+ * Set up a fifo by which host applications can send command an policies
+ * through a fifo to the vm_power_manager
+ *
+ * @return
+ *  - 0 for success
+ */
+int add_host_channel(void);
+
+/**
  * Remove a channel definition from the channel manager. This must only be
  * called from the channel monitor thread.
  *
diff --git a/examples/vm_power_manager/channel_monitor.c b/examples/vm_power_manager/channel_monitor.c
index 7fa47ba9..5da53154 100644
--- a/examples/vm_power_manager/channel_monitor.c
+++ b/examples/vm_power_manager/channel_monitor.c
@@ -9,11 +9,18 @@
 #include <signal.h>
 #include <errno.h>
 #include <string.h>
+#include <fcntl.h>
 #include <sys/types.h>
 #include <sys/epoll.h>
 #include <sys/queue.h>
 #include <sys/time.h>
-
+#include <sys/socket.h>
+#include <sys/select.h>
+#ifdef USE_JANSSON
+#include <jansson.h>
+#else
+#pragma message "Jansson dev libs unavailable, not including JSON parsing"
+#endif
 #include <rte_log.h>
 #include <rte_memory.h>
 #include <rte_malloc.h>
@@ -35,13 +42,243 @@
 
 uint64_t vsi_pkt_count_prev[384];
 uint64_t rdtsc_prev[384];
+#define MAX_JSON_STRING_LEN 1024
+char json_data[MAX_JSON_STRING_LEN];
 
 double time_period_ms = 1;
 static volatile unsigned run_loop = 1;
 static int global_event_fd;
 static unsigned int policy_is_set;
 static struct epoll_event *global_events_list;
-static struct policy policies[MAX_VMS];
+static struct policy policies[MAX_CLIENTS];
+
+#ifdef USE_JANSSON
+
+union PFID {
+	struct ether_addr addr;
+	uint64_t pfid;
+};
+
+static int
+str_to_ether_addr(const char *a, struct ether_addr *ether_addr)
+{
+	int i;
+	char *end;
+	unsigned long o[ETHER_ADDR_LEN];
+
+	i = 0;
+	do {
+		errno = 0;
+		o[i] = strtoul(a, &end, 16);
+		if (errno != 0 || end == a || (end[0] != ':' && end[0] != 0))
+			return -1;
+		a = end + 1;
+	} while (++i != RTE_DIM(o) / sizeof(o[0]) && end[0] != 0);
+
+	/* Junk at the end of line */
+	if (end[0] != 0)
+		return -1;
+
+	/* Support the format XX:XX:XX:XX:XX:XX */
+	if (i == ETHER_ADDR_LEN) {
+		while (i-- != 0) {
+			if (o[i] > UINT8_MAX)
+				return -1;
+			ether_addr->addr_bytes[i] = (uint8_t)o[i];
+		}
+	/* Support the format XXXX:XXXX:XXXX */
+	} else if (i == ETHER_ADDR_LEN / 2) {
+		while (i-- != 0) {
+			if (o[i] > UINT16_MAX)
+				return -1;
+			ether_addr->addr_bytes[i * 2] =
+					(uint8_t)(o[i] >> 8);
+			ether_addr->addr_bytes[i * 2 + 1] =
+					(uint8_t)(o[i] & 0xff);
+		}
+	/* unknown format */
+	} else
+		return -1;
+
+	return 0;
+}
+
+static int
+set_policy_mac(struct channel_packet *pkt, int idx, char *mac)
+{
+	union PFID pfid;
+	int ret;
+
+	/* Use port MAC address as the vfid */
+	ret = str_to_ether_addr(mac, &pfid.addr);
+
+	if (ret != 0) {
+		RTE_LOG(ERR, CHANNEL_MONITOR,
+			"Invalid mac address received in JSON\n");
+		pkt->vfid[idx] = 0;
+		return -1;
+	}
+
+	printf("Received MAC Address: %02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 ":"
+			"%02" PRIx8 ":%02" PRIx8 ":%02" PRIx8 "\n",
+			pfid.addr.addr_bytes[0], pfid.addr.addr_bytes[1],
+			pfid.addr.addr_bytes[2], pfid.addr.addr_bytes[3],
+			pfid.addr.addr_bytes[4], pfid.addr.addr_bytes[5]);
+
+	pkt->vfid[idx] = pfid.pfid;
+	return 0;
+}
+
+
+static int
+parse_json_to_pkt(json_t *element, struct channel_packet *pkt)
+{
+	const char *key;
+	json_t *value;
+	int ret;
+
+	memset(pkt, 0, sizeof(struct channel_packet));
+
+	pkt->nb_mac_to_monitor = 0;
+	pkt->t_boost_status.tbEnabled = false;
+	pkt->workload = LOW;
+	pkt->policy_to_use = TIME;
+	pkt->command = PKT_POLICY;
+	pkt->core_type = CORE_TYPE_PHYSICAL;
+
+	json_object_foreach(element, key, value) {
+		if (!strcmp(key, "policy")) {
+			/* Recurse in to get the contents of profile */
+			ret = parse_json_to_pkt(value, pkt);
+			if (ret)
+				return ret;
+		} else if (!strcmp(key, "instruction")) {
+			/* Recurse in to get the contents of instruction */
+			ret = parse_json_to_pkt(value, pkt);
+			if (ret)
+				return ret;
+		} else if (!strcmp(key, "name")) {
+			strcpy(pkt->vm_name, json_string_value(value));
+		} else if (!strcmp(key, "command")) {
+			char command[32];
+			snprintf(command, 32, "%s", json_string_value(value));
+			if (!strcmp(command, "power")) {
+				pkt->command = CPU_POWER;
+			} else if (!strcmp(command, "create")) {
+				pkt->command = PKT_POLICY;
+			} else if (!strcmp(command, "destroy")) {
+				pkt->command = PKT_POLICY_REMOVE;
+			} else {
+				RTE_LOG(ERR, CHANNEL_MONITOR,
+					"Invalid command received in JSON\n");
+				return -1;
+			}
+		} else if (!strcmp(key, "policy_type")) {
+			char command[32];
+			snprintf(command, 32, "%s", json_string_value(value));
+			if (!strcmp(command, "TIME")) {
+				pkt->policy_to_use = TIME;
+			} else if (!strcmp(command, "TRAFFIC")) {
+				pkt->policy_to_use = TRAFFIC;
+			} else if (!strcmp(command, "WORKLOAD")) {
+				pkt->policy_to_use = WORKLOAD;
+			} else if (!strcmp(command, "BRANCH_RATIO")) {
+				pkt->policy_to_use = BRANCH_RATIO;
+			} else {
+				RTE_LOG(ERR, CHANNEL_MONITOR,
+					"Wrong policy_type received in JSON\n");
+				return -1;
+			}
+		} else if (!strcmp(key, "workload")) {
+			char command[32];
+			snprintf(command, 32, "%s", json_string_value(value));
+			if (!strcmp(command, "HIGH")) {
+				pkt->workload = HIGH;
+			} else if (!strcmp(command, "MEDIUM")) {
+				pkt->workload = MEDIUM;
+			} else if (!strcmp(command, "LOW")) {
+				pkt->workload = LOW;
+			} else {
+				RTE_LOG(ERR, CHANNEL_MONITOR,
+					"Wrong workload received in JSON\n");
+				return -1;
+			}
+		} else if (!strcmp(key, "busy_hours")) {
+			unsigned int i;
+			size_t size = json_array_size(value);
+
+			for (i = 0; i < size; i++) {
+				int hour = (int)json_integer_value(
+						json_array_get(value, i));
+				pkt->timer_policy.busy_hours[i] = hour;
+			}
+		} else if (!strcmp(key, "quiet_hours")) {
+			unsigned int i;
+			size_t size = json_array_size(value);
+
+			for (i = 0; i < size; i++) {
+				int hour = (int)json_integer_value(
+						json_array_get(value, i));
+				pkt->timer_policy.quiet_hours[i] = hour;
+			}
+		} else if (!strcmp(key, "core_list")) {
+			unsigned int i;
+			size_t size = json_array_size(value);
+
+			for (i = 0; i < size; i++) {
+				int core = (int)json_integer_value(
+						json_array_get(value, i));
+				pkt->vcpu_to_control[i] = core;
+			}
+			pkt->num_vcpu = size;
+		} else if (!strcmp(key, "mac_list")) {
+			unsigned int i;
+			size_t size = json_array_size(value);
+
+			for (i = 0; i < size; i++) {
+				char mac[32];
+				snprintf(mac, 32, "%s", json_string_value(
+						json_array_get(value, i)));
+				set_policy_mac(pkt, i, mac);
+			}
+			pkt->nb_mac_to_monitor = size;
+		} else if (!strcmp(key, "avg_packet_thresh")) {
+			pkt->traffic_policy.avg_max_packet_thresh =
+					(uint32_t)json_integer_value(value);
+		} else if (!strcmp(key, "max_packet_thresh")) {
+			pkt->traffic_policy.max_max_packet_thresh =
+					(uint32_t)json_integer_value(value);
+		} else if (!strcmp(key, "unit")) {
+			char unit[32];
+			snprintf(unit, 32, "%s", json_string_value(value));
+			if (!strcmp(unit, "SCALE_UP")) {
+				pkt->unit = CPU_POWER_SCALE_UP;
+			} else if (!strcmp(unit, "SCALE_DOWN")) {
+				pkt->unit = CPU_POWER_SCALE_DOWN;
+			} else if (!strcmp(unit, "SCALE_MAX")) {
+				pkt->unit = CPU_POWER_SCALE_MAX;
+			} else if (!strcmp(unit, "SCALE_MIN")) {
+				pkt->unit = CPU_POWER_SCALE_MIN;
+			} else if (!strcmp(unit, "ENABLE_TURBO")) {
+				pkt->unit = CPU_POWER_ENABLE_TURBO;
+			} else if (!strcmp(unit, "DISABLE_TURBO")) {
+				pkt->unit = CPU_POWER_DISABLE_TURBO;
+			} else {
+				RTE_LOG(ERR, CHANNEL_MONITOR,
+					"Invalid command received in JSON\n");
+				return -1;
+			}
+		} else if (!strcmp(key, "resource_id")) {
+			pkt->resource_id = (uint32_t)json_integer_value(value);
+		} else {
+			RTE_LOG(ERR, CHANNEL_MONITOR,
+				"Unknown key received in JSON string: %s\n",
+				key);
+		}
+	}
+	return 0;
+}
+#endif
 
 void channel_monitor_exit(void)
 {
@@ -66,7 +303,7 @@ static void
 core_share_status(int pNo)
 {
 
-	int noVms, noVcpus, z, x, t;
+	int noVms = 0, noVcpus = 0, z, x, t;
 
 	get_all_vm(&noVms, &noVcpus);
 
@@ -85,6 +322,33 @@ core_share_status(int pNo)
 	}
 }
 
+
+static int
+pcpu_monitor(struct policy *pol, struct core_info *ci, int pcpu, int count)
+{
+	int ret = 0;
+
+	if (pol->pkt.policy_to_use == BRANCH_RATIO) {
+		ci->cd[pcpu].oob_enabled = 1;
+		ret = add_core_to_monitor(pcpu);
+		if (ret == 0)
+			RTE_LOG(INFO, CHANNEL_MONITOR,
+					"Monitoring pcpu %d OOB for %s\n",
+					pcpu, pol->pkt.vm_name);
+		else
+			RTE_LOG(ERR, CHANNEL_MONITOR,
+					"Error monitoring pcpu %d OOB for %s\n",
+					pcpu, pol->pkt.vm_name);
+
+	} else {
+		pol->core_share[count].pcpu = pcpu;
+		RTE_LOG(INFO, CHANNEL_MONITOR,
+				"Monitoring pcpu %d for %s\n",
+				pcpu, pol->pkt.vm_name);
+	}
+	return ret;
+}
+
 static void
 get_pcpu_to_control(struct policy *pol)
 {
@@ -94,34 +358,42 @@ get_pcpu_to_control(struct policy *pol)
 	int pcpu, count;
 	uint64_t mask_u64b;
 	struct core_info *ci;
-	int ret;
 
 	ci = get_core_info();
 
-	RTE_LOG(INFO, CHANNEL_MONITOR, "Looking for pcpu for %s\n",
-			pol->pkt.vm_name);
-	get_info_vm(pol->pkt.vm_name, &info);
-
-	for (count = 0; count < pol->pkt.num_vcpu; count++) {
-		mask_u64b = info.pcpu_mask[pol->pkt.vcpu_to_control[count]];
-		for (pcpu = 0; mask_u64b; mask_u64b &= ~(1ULL << pcpu++)) {
-			if ((mask_u64b >> pcpu) & 1) {
-				if (pol->pkt.policy_to_use == BRANCH_RATIO) {
-					ci->cd[pcpu].oob_enabled = 1;
-					ret = add_core_to_monitor(pcpu);
-					if (ret == 0)
-						printf("Monitoring pcpu %d via Branch Ratio\n",
-								pcpu);
-					else
-						printf("Failed to start OOB Monitoring pcpu %d\n",
-								pcpu);
-
-				} else {
-					pol->core_share[count].pcpu = pcpu;
-					printf("Monitoring pcpu %d\n", pcpu);
-				}
+	RTE_LOG(DEBUG, CHANNEL_MONITOR,
+			"Looking for pcpu for %s\n", pol->pkt.vm_name);
+
+	/*
+	 * So now that we're handling virtual and physical cores, we need to
+	 * differenciate between them when adding them to the branch monitor.
+	 * Virtual cores need to be converted to physical cores.
+	 */
+	if (pol->pkt.core_type == CORE_TYPE_VIRTUAL) {
+		/*
+		 * If the cores in the policy are virtual, we need to map them
+		 * to physical core. We look up the vm info and use that for
+		 * the mapping.
+		 */
+		get_info_vm(pol->pkt.vm_name, &info);
+		for (count = 0; count < pol->pkt.num_vcpu; count++) {
+			mask_u64b =
+				info.pcpu_mask[pol->pkt.vcpu_to_control[count]];
+			for (pcpu = 0; mask_u64b;
+					mask_u64b &= ~(1ULL << pcpu++)) {
+				if ((mask_u64b >> pcpu) & 1)
+					pcpu_monitor(pol, ci, pcpu, count);
 			}
 		}
+	} else {
+		/*
+		 * If the cores in the policy are physical, we just use
+		 * those core id's directly.
+		 */
+		for (count = 0; count < pol->pkt.num_vcpu; count++) {
+			pcpu = pol->pkt.vcpu_to_control[count];
+			pcpu_monitor(pol, ci, pcpu, count);
+		}
 	}
 }
 
@@ -160,8 +432,13 @@ update_policy(struct channel_packet *pkt)
 	unsigned int updated = 0;
 	int i;
 
-	for (i = 0; i < MAX_VMS; i++) {
+
+	RTE_LOG(INFO, CHANNEL_MONITOR,
+			"Applying policy for %s\n", pkt->vm_name);
+
+	for (i = 0; i < MAX_CLIENTS; i++) {
 		if (strcmp(policies[i].pkt.vm_name, pkt->vm_name) == 0) {
+			/* Copy the contents of *pkt into the policy.pkt */
 			policies[i].pkt = *pkt;
 			get_pcpu_to_control(&policies[i]);
 			if (get_pfid(&policies[i]) == -1) {
@@ -174,7 +451,7 @@ update_policy(struct channel_packet *pkt)
 		}
 	}
 	if (!updated) {
-		for (i = 0; i < MAX_VMS; i++) {
+		for (i = 0; i < MAX_CLIENTS; i++) {
 			if (policies[i].enabled == 0) {
 				policies[i].pkt = *pkt;
 				get_pcpu_to_control(&policies[i]);
@@ -189,6 +466,24 @@ update_policy(struct channel_packet *pkt)
 	return 0;
 }
 
+static int
+remove_policy(struct channel_packet *pkt __rte_unused)
+{
+	int i;
+
+	/*
+	 * Disabling the policy is simply a case of setting
+	 * enabled to 0
+	 */
+	for (i = 0; i < MAX_CLIENTS; i++) {
+		if (strcmp(policies[i].pkt.vm_name, pkt->vm_name) == 0) {
+			policies[i].enabled = 0;
+			return 0;
+		}
+	}
+	return -1;
+}
+
 static uint64_t
 get_pkt_diff(struct policy *pol)
 {
@@ -233,8 +528,6 @@ apply_traffic_profile(struct policy *pol)
 
 	diff = get_pkt_diff(pol);
 
-	RTE_LOG(INFO, CHANNEL_MONITOR, "Applying traffic profile\n");
-
 	if (diff >= (pol->pkt.traffic_policy.max_max_packet_thresh)) {
 		for (count = 0; count < pol->pkt.num_vcpu; count++) {
 			if (pol->core_share[count].status != 1)
@@ -278,9 +571,6 @@ apply_time_profile(struct policy *pol)
 				if (pol->core_share[count].status != 1) {
 					power_manager_scale_core_max(
 						pol->core_share[count].pcpu);
-				RTE_LOG(INFO, CHANNEL_MONITOR,
-					"Scaling up core %d to max\n",
-					pol->core_share[count].pcpu);
 				}
 			}
 			break;
@@ -290,9 +580,6 @@ apply_time_profile(struct policy *pol)
 				if (pol->core_share[count].status != 1) {
 					power_manager_scale_core_min(
 						pol->core_share[count].pcpu);
-				RTE_LOG(INFO, CHANNEL_MONITOR,
-					"Scaling down core %d to min\n",
-					pol->core_share[count].pcpu);
 			}
 		}
 			break;
@@ -346,7 +633,6 @@ apply_policy(struct policy *pol)
 		apply_workload_profile(pol);
 }
 
-
 static int
 process_request(struct channel_packet *pkt, struct channel_info *chan_info)
 {
@@ -362,10 +648,12 @@ process_request(struct channel_packet *pkt, struct channel_info *chan_info)
 	if (pkt->command == CPU_POWER) {
 		core_mask = get_pcpus_mask(chan_info, pkt->resource_id);
 		if (core_mask == 0) {
-			RTE_LOG(ERR, CHANNEL_MONITOR, "Error get physical CPU mask for "
-				"channel '%s' using vCPU(%u)\n", chan_info->channel_path,
-				(unsigned)pkt->unit);
-			return -1;
+			/*
+			 * Core mask will be 0 in the case where
+			 * hypervisor is not available so we're working in
+			 * the host, so use the core as the mask.
+			 */
+			core_mask = 1ULL << pkt->resource_id;
 		}
 		if (__builtin_popcountll(core_mask) == 1) {
 
@@ -421,12 +709,20 @@ process_request(struct channel_packet *pkt, struct channel_info *chan_info)
 	}
 
 	if (pkt->command == PKT_POLICY) {
-		RTE_LOG(INFO, CHANNEL_MONITOR, "\nProcessing Policy request from Guest\n");
+		RTE_LOG(INFO, CHANNEL_MONITOR, "Processing policy request %s\n",
+				pkt->vm_name);
 		update_policy(pkt);
 		policy_is_set = 1;
 	}
 
-	/* Return is not checked as channel status may have been set to DISABLED
+	if (pkt->command == PKT_POLICY_REMOVE) {
+		RTE_LOG(INFO, CHANNEL_MONITOR,
+				 "Removing policy %s\n", pkt->vm_name);
+		remove_policy(pkt);
+	}
+
+	/*
+	 * Return is not checked as channel status may have been set to DISABLED
 	 * from management thread
 	 */
 	rte_atomic32_cmpset(&(chan_info->status), CHANNEL_MGR_CHANNEL_PROCESSING,
@@ -448,13 +744,16 @@ add_channel_to_monitor(struct channel_info **chan_info)
 				"to epoll\n", info->channel_path);
 		return -1;
 	}
+	RTE_LOG(ERR, CHANNEL_MONITOR, "Added channel '%s' "
+			"to monitor\n", info->channel_path);
 	return 0;
 }
 
 int
 remove_channel_from_monitor(struct channel_info *chan_info)
 {
-	if (epoll_ctl(global_event_fd, EPOLL_CTL_DEL, chan_info->fd, NULL) < 0) {
+	if (epoll_ctl(global_event_fd, EPOLL_CTL_DEL,
+			chan_info->fd, NULL) < 0) {
 		RTE_LOG(ERR, CHANNEL_MONITOR, "Unable to remove channel '%s' "
 				"from epoll\n", chan_info->channel_path);
 		return -1;
@@ -467,11 +766,13 @@ channel_monitor_init(void)
 {
 	global_event_fd = epoll_create1(0);
 	if (global_event_fd == 0) {
-		RTE_LOG(ERR, CHANNEL_MONITOR, "Error creating epoll context with "
-				"error %s\n", strerror(errno));
+		RTE_LOG(ERR, CHANNEL_MONITOR,
+				"Error creating epoll context with error %s\n",
+				strerror(errno));
 		return -1;
 	}
-	global_events_list = rte_malloc("epoll_events", sizeof(*global_events_list)
+	global_events_list = rte_malloc("epoll_events",
+			sizeof(*global_events_list)
 			* MAX_EVENTS, RTE_CACHE_LINE_SIZE);
 	if (global_events_list == NULL) {
 		RTE_LOG(ERR, CHANNEL_MONITOR, "Unable to rte_malloc for "
@@ -481,6 +782,103 @@ channel_monitor_init(void)
 	return 0;
 }
 
+static void
+read_binary_packet(struct channel_info *chan_info)
+{
+	struct channel_packet pkt;
+	void *buffer = &pkt;
+	int buffer_len = sizeof(pkt);
+	int n_bytes, err = 0;
+
+	while (buffer_len > 0) {
+		n_bytes = read(chan_info->fd,
+				buffer, buffer_len);
+		if (n_bytes == buffer_len)
+			break;
+		if (n_bytes == -1) {
+			err = errno;
+			RTE_LOG(DEBUG, CHANNEL_MONITOR,
+				"Received error on "
+				"channel '%s' read: %s\n",
+				chan_info->channel_path,
+				strerror(err));
+			remove_channel(&chan_info);
+			break;
+		}
+		buffer = (char *)buffer + n_bytes;
+		buffer_len -= n_bytes;
+	}
+	if (!err)
+		process_request(&pkt, chan_info);
+}
+
+#ifdef USE_JANSSON
+static void
+read_json_packet(struct channel_info *chan_info)
+{
+	struct channel_packet pkt;
+	int n_bytes, ret;
+	json_t *root;
+	json_error_t error;
+
+	/* read opening brace to closing brace */
+	do {
+		int idx = 0;
+		int indent = 0;
+		do {
+			n_bytes = read(chan_info->fd, &json_data[idx], 1);
+			if (n_bytes == 0)
+				break;
+			if (json_data[idx] == '{')
+				indent++;
+			if (json_data[idx] == '}')
+				indent--;
+			if ((indent > 0) || (idx > 0))
+				idx++;
+			if (indent == 0)
+				json_data[idx] = 0;
+			if (idx >= MAX_JSON_STRING_LEN-1)
+				break;
+		} while (indent > 0);
+
+		if (indent > 0)
+			/*
+			 * We've broken out of the read loop without getting
+			 * a closing brace, so throw away the data
+			 */
+			json_data[idx] = 0;
+
+		if (strlen(json_data) == 0)
+			continue;
+
+		printf("got [%s]\n", json_data);
+
+		root = json_loads(json_data, 0, &error);
+
+		if (root) {
+			/*
+			 * Because our data is now in the json
+			 * object, we can overwrite the pkt
+			 * with a channel_packet struct, using
+			 * parse_json_to_pkt()
+			 */
+			ret = parse_json_to_pkt(root, &pkt);
+			json_decref(root);
+			if (ret) {
+				RTE_LOG(ERR, CHANNEL_MONITOR,
+					"Error validating JSON profile data\n");
+				break;
+			}
+			process_request(&pkt, chan_info);
+		} else {
+			RTE_LOG(ERR, CHANNEL_MONITOR,
+					"JSON error on line %d: %s\n",
+					error.line, error.text);
+		}
+	} while (n_bytes > 0);
+}
+#endif
+
 void
 run_channel_monitor(void)
 {
@@ -496,7 +894,8 @@ run_channel_monitor(void)
 					global_events_list[i].data.ptr;
 			if ((global_events_list[i].events & EPOLLERR) ||
 				(global_events_list[i].events & EPOLLHUP)) {
-				RTE_LOG(DEBUG, CHANNEL_MONITOR, "Remote closed connection for "
+				RTE_LOG(INFO, CHANNEL_MONITOR,
+						"Remote closed connection for "
 						"channel '%s'\n",
 						chan_info->channel_path);
 				remove_channel(&chan_info);
@@ -504,38 +903,25 @@ run_channel_monitor(void)
 			}
 			if (global_events_list[i].events & EPOLLIN) {
 
-				int n_bytes, err = 0;
-				struct channel_packet pkt;
-				void *buffer = &pkt;
-				int buffer_len = sizeof(pkt);
-
-				while (buffer_len > 0) {
-					n_bytes = read(chan_info->fd,
-							buffer, buffer_len);
-					if (n_bytes == buffer_len)
-						break;
-					if (n_bytes == -1) {
-						err = errno;
-						RTE_LOG(DEBUG, CHANNEL_MONITOR,
-							"Received error on "
-							"channel '%s' read: %s\n",
-							chan_info->channel_path,
-							strerror(err));
-						remove_channel(&chan_info);
-						break;
-					}
-					buffer = (char *)buffer + n_bytes;
-					buffer_len -= n_bytes;
+				switch (chan_info->type) {
+				case CHANNEL_TYPE_BINARY:
+					read_binary_packet(chan_info);
+					break;
+#ifdef USE_JANSSON
+				case CHANNEL_TYPE_JSON:
+					read_json_packet(chan_info);
+					break;
+#endif
+				default:
+					break;
 				}
-				if (!err)
-					process_request(&pkt, chan_info);
 			}
 		}
 		rte_delay_us(time_period_ms*1000);
 		if (policy_is_set) {
 			int j;
 
-			for (j = 0; j < MAX_VMS; j++) {
+			for (j = 0; j < MAX_CLIENTS; j++) {
 				if (policies[j].enabled == 1)
 					apply_policy(&policies[j]);
 			}
diff --git a/examples/vm_power_manager/guest_cli/meson.build b/examples/vm_power_manager/guest_cli/meson.build
new file mode 100644
index 00000000..9e821ceb
--- /dev/null
+++ b/examples/vm_power_manager/guest_cli/meson.build
@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2018 Intel Corporation
+
+# meson file, for building this example as part of a main DPDK build.
+#
+# To build this example as a standalone application with an already-installed
+# DPDK instance, use 'make'
+
+# Setting the name here because the default name will conflict with the
+# vm_power_manager app because of the way the directories are parsed.
+name = 'guest_cli'
+
+deps += ['power']
+
+sources = files(
+	'main.c', 'parse.c', 'vm_power_cli_guest.c'
+)
+
+opt_dep = cc.find_library('virt', required : false)
+build = opt_dep.found()
+ext_deps += opt_dep
diff --git a/examples/vm_power_manager/guest_cli/vm_power_cli_guest.c b/examples/vm_power_manager/guest_cli/vm_power_cli_guest.c
index 0db1b804..2d9e7689 100644
--- a/examples/vm_power_manager/guest_cli/vm_power_cli_guest.c
+++ b/examples/vm_power_manager/guest_cli/vm_power_cli_guest.c
@@ -92,6 +92,7 @@ set_policy_defaults(struct channel_packet *pkt)
 	pkt->timer_policy.hours_to_use_traffic_profile[0] = 8;
 	pkt->timer_policy.hours_to_use_traffic_profile[1] = 10;
 
+	pkt->core_type = CORE_TYPE_VIRTUAL;
 	pkt->workload = LOW;
 	pkt->policy_to_use = TIME;
 	pkt->command = PKT_POLICY;
diff --git a/examples/vm_power_manager/main.c b/examples/vm_power_manager/main.c
index 58c5fa45..893bf4cd 100644
--- a/examples/vm_power_manager/main.c
+++ b/examples/vm_power_manager/main.c
@@ -421,6 +421,8 @@ main(int argc, char **argv)
 		return -1;
 	}
 
+	add_host_channel();
+
 	printf("Running core monitor on lcore id %d\n", lcore_id);
 	rte_eal_remote_launch(run_core_monitor, NULL, lcore_id);
 
diff --git a/examples/vm_power_manager/meson.build b/examples/vm_power_manager/meson.build
index c370d747..f98445bc 100644
--- a/examples/vm_power_manager/meson.build
+++ b/examples/vm_power_manager/meson.build
@@ -6,5 +6,38 @@
 # To build this example as a standalone application with an already-installed
 # DPDK instance, use 'make'
 
-# Example app currently unsupported by meson build
-build = false
+if dpdk_conf.has('RTE_LIBRTE_BNXT_PMD')
+	deps += ['pmd_bnxt']
+endif
+
+if dpdk_conf.has('RTE_LIBRTE_I40E_PMD')
+	deps += ['pmd_i40e']
+endif
+
+if dpdk_conf.has('RTE_LIBRTE_IXGBE_PMD')
+	deps += ['pmd_ixgbe']
+endif
+
+deps += ['power']
+
+
+sources = files(
+	'channel_manager.c', 'channel_monitor.c', 'main.c', 'parse.c', 'power_manager.c', 'vm_power_cli.c'
+)
+
+# If we're on X86, pull in the x86 code for the branch monitor algo.
+if dpdk_conf.has('RTE_ARCH_X86_64')
+	sources += files('oob_monitor_x86.c')
+else
+	sources += files('oob_monitor_nop.c')
+endif
+
+opt_dep = cc.find_library('virt', required : false)
+build = opt_dep.found()
+ext_deps += opt_dep
+
+opt_dep = dependency('jansson', required : false)
+if opt_dep.found()
+	ext_deps += opt_dep
+	cflags += '-DUSE_JANSSON'
+endif
diff --git a/examples/vm_power_manager/power_manager.c b/examples/vm_power_manager/power_manager.c
index b7769c3c..f9e8c0ab 100644
--- a/examples/vm_power_manager/power_manager.c
+++ b/examples/vm_power_manager/power_manager.c
@@ -95,6 +95,7 @@ power_manager_init(void)
 	unsigned int i, num_cpus = 0, num_freqs = 0;
 	int ret = 0;
 	struct core_info *ci;
+	unsigned int max_core_num;
 
 	rte_power_set_env(PM_ENV_ACPI_CPUFREQ);
 
@@ -105,7 +106,12 @@ power_manager_init(void)
 		return -1;
 	}
 
-	for (i = 0; i < ci->core_count; i++) {
+	if (ci->core_count > POWER_MGR_MAX_CPUS)
+		max_core_num = POWER_MGR_MAX_CPUS;
+	else
+		max_core_num = ci->core_count;
+
+	for (i = 0; i < max_core_num; i++) {
 		if (ci->cd[i].global_enabled_cpus) {
 			if (rte_power_init(i) < 0)
 				RTE_LOG(ERR, POWER_MANAGER,
@@ -165,6 +171,7 @@ power_manager_exit(void)
 	unsigned int i;
 	int ret = 0;
 	struct core_info *ci;
+	unsigned int max_core_num;
 
 	ci = get_core_info();
 	if (!ci) {
@@ -173,7 +180,12 @@ power_manager_exit(void)
 		return -1;
 	}
 
-	for (i = 0; i < ci->core_count; i++) {
+	if (ci->core_count > POWER_MGR_MAX_CPUS)
+		max_core_num = POWER_MGR_MAX_CPUS;
+	else
+		max_core_num = ci->core_count;
+
+	for (i = 0; i < max_core_num; i++) {
 		if (ci->cd[i].global_enabled_cpus) {
 			if (rte_power_exit(i) < 0) {
 				RTE_LOG(ERR, POWER_MANAGER, "Unable to shutdown power manager "