Imported Upstream version 16.07-rc1

Change-Id: If3f757dc95532706b04053286c6b54492169f1a3
Signed-off-by: Christian Ehrhardt <christian.ehrhardt@canonical.com>

17 files changed, 603 insertions, 585 deletions

diff --git a/app/test-pmd/Makefile b/app/test-pmd/Makefile
index 72426f31..2a0b5a5d 100644
--- a/app/test-pmd/Makefile
+++ b/app/test-pmd/Makefile
@@ -50,7 +50,6 @@ SRCS-$(CONFIG_RTE_LIBRTE_CMDLINE) += cmdline.c
 SRCS-y += config.c
 SRCS-y += iofwd.c
 SRCS-y += macfwd.c
-SRCS-y += macfwd-retry.c
 SRCS-y += macswap.c
 SRCS-y += flowgen.c
 SRCS-y += rxonly.c
@@ -58,11 +57,7 @@ SRCS-y += txonly.c
 SRCS-y += csumonly.c
 SRCS-y += icmpecho.c
 SRCS-$(CONFIG_RTE_LIBRTE_IEEE1588) += ieee1588fwd.c
-SRCS-y += mempool_anon.c
 
-ifeq ($(CONFIG_RTE_EXEC_ENV_LINUXAPP),y)
-CFLAGS_mempool_anon.o := -D_GNU_SOURCE
-endif
 CFLAGS_cmdline.o := -D_GNU_SOURCE
 
 # this application needs libraries first
diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c
index c5b94797..b6b61ad3 100644
--- a/app/test-pmd/cmdline.c
+++ b/app/test-pmd/cmdline.c
@@ -246,8 +246,8 @@ static void cmd_help_long_parsed(void *parsed_result,
 			"    Set number of packets per burst.\n\n"
 
 			"set burst tx delay (microseconds) retry (num)\n"
-			"    Set the transmit delay time and number of retries"
-			" in mac_retry forwarding mode.\n\n"
+			"    Set the transmit delay time and number of retries,"
+			" effective when retry is enabled.\n\n"
 
 			"set txpkts (x[,y]*)\n"
 			"    Set the length of each segment of TXONLY"
@@ -559,13 +559,13 @@ static void cmd_help_long_parsed(void *parsed_result,
 			"port config all max-pkt-len (value)\n"
 			"    Set the max packet length.\n\n"
 
-			"port config all (crc-strip|rx-cksum|hw-vlan|hw-vlan-filter|"
+			"port config all (crc-strip|scatter|rx-cksum|hw-vlan|hw-vlan-filter|"
 			"hw-vlan-strip|hw-vlan-extend|drop-en)"
 			" (on|off)\n"
-			"    Set crc-strip/rx-checksum/hardware-vlan/drop_en"
+			"    Set crc-strip/scatter/rx-checksum/hardware-vlan/drop_en"
 			" for ports.\n\n"
 
-			"port config all rss (all|ip|tcp|udp|sctp|ether|none)\n"
+			"port config all rss (all|ip|tcp|udp|sctp|ether|port|vxlan|geneve|nvgre|none)\n"
 			"    Set the RSS mode.\n\n"
 
 			"port config port-id rss reta (hash,queue)[,(hash,queue)]\n"
@@ -1223,6 +1223,8 @@ cmd_config_rx_tx_parsed(void *parsed_result,
 		return;
 	}
 
+	fwd_config_setup();
+
 	init_port_config();
 
 	cmd_reconfig_device_queue(RTE_PORT_ALL, 1, 1);
@@ -1410,6 +1412,15 @@ cmd_config_rx_mode_flag_parsed(void *parsed_result,
 			printf("Unknown parameter\n");
 			return;
 		}
+	} else if (!strcmp(res->name, "scatter")) {
+		if (!strcmp(res->value, "on"))
+			rx_mode.enable_scatter = 1;
+		else if (!strcmp(res->value, "off"))
+			rx_mode.enable_scatter = 0;
+		else {
+			printf("Unknown parameter\n");
+			return;
+		}
 	} else if (!strcmp(res->name, "rx-cksum")) {
 		if (!strcmp(res->value, "on"))
 			rx_mode.hw_ip_checksum = 1;
@@ -1487,7 +1498,7 @@ cmdline_parse_token_string_t cmd_config_rx_mode_flag_all =
 	TOKEN_STRING_INITIALIZER(struct cmd_config_rx_mode_flag, all, "all");
 cmdline_parse_token_string_t cmd_config_rx_mode_flag_name =
 	TOKEN_STRING_INITIALIZER(struct cmd_config_rx_mode_flag, name,
-					"crc-strip#rx-cksum#hw-vlan#"
+					"crc-strip#scatter#rx-cksum#hw-vlan#"
 					"hw-vlan-filter#hw-vlan-strip#hw-vlan-extend");
 cmdline_parse_token_string_t cmd_config_rx_mode_flag_value =
 	TOKEN_STRING_INITIALIZER(struct cmd_config_rx_mode_flag, value,
@@ -1496,7 +1507,7 @@ cmdline_parse_token_string_t cmd_config_rx_mode_flag_value =
 cmdline_parse_inst_t cmd_config_rx_mode_flag = {
 	.f = cmd_config_rx_mode_flag_parsed,
 	.data = NULL,
-	.help_str = "port config all crc-strip|rx-cksum|hw-vlan|"
+	.help_str = "port config all crc-strip|scatter|rx-cksum|hw-vlan|"
 		"hw-vlan-filter|hw-vlan-strip|hw-vlan-extend on|off",
 	.tokens = {
 		(void *)&cmd_config_rx_mode_flag_port,
@@ -1524,6 +1535,7 @@ cmd_config_rss_parsed(void *parsed_result,
 {
 	struct cmd_config_rss *res = parsed_result;
 	struct rte_eth_rss_conf rss_conf;
+	int diag;
 	uint8_t i;
 
 	if (!strcmp(res->value, "all"))
@@ -1540,6 +1552,14 @@ cmd_config_rss_parsed(void *parsed_result,
 		rss_conf.rss_hf = ETH_RSS_SCTP;
 	else if (!strcmp(res->value, "ether"))
 		rss_conf.rss_hf = ETH_RSS_L2_PAYLOAD;
+	else if (!strcmp(res->value, "port"))
+		rss_conf.rss_hf = ETH_RSS_PORT;
+	else if (!strcmp(res->value, "vxlan"))
+		rss_conf.rss_hf = ETH_RSS_VXLAN;
+	else if (!strcmp(res->value, "geneve"))
+		rss_conf.rss_hf = ETH_RSS_GENEVE;
+	else if (!strcmp(res->value, "nvgre"))
+		rss_conf.rss_hf = ETH_RSS_NVGRE;
 	else if (!strcmp(res->value, "none"))
 		rss_conf.rss_hf = 0;
 	else {
@@ -1547,8 +1567,13 @@ cmd_config_rss_parsed(void *parsed_result,
 		return;
 	}
 	rss_conf.rss_key = NULL;
-	for (i = 0; i < rte_eth_dev_count(); i++)
-		rte_eth_dev_rss_hash_update(i, &rss_conf);
+	for (i = 0; i < rte_eth_dev_count(); i++) {
+		diag = rte_eth_dev_rss_hash_update(i, &rss_conf);
+		if (diag < 0)
+			printf("Configuration of RSS hash at ethernet port %d "
+				"failed with error (%d): %s.\n",
+				i, -diag, strerror(-diag));
+	}
 }
 
 cmdline_parse_token_string_t cmd_config_rss_port =
@@ -1561,12 +1586,12 @@ cmdline_parse_token_string_t cmd_config_rss_name =
 	TOKEN_STRING_INITIALIZER(struct cmd_config_rss, name, "rss");
 cmdline_parse_token_string_t cmd_config_rss_value =
 	TOKEN_STRING_INITIALIZER(struct cmd_config_rss, value,
-		"all#ip#tcp#udp#sctp#ether#none");
+		"all#ip#tcp#udp#sctp#ether#port#vxlan#geneve#nvgre#none");
 
 cmdline_parse_inst_t cmd_config_rss = {
 	.f = cmd_config_rss_parsed,
 	.data = NULL,
-	.help_str = "port config all rss all|ip|tcp|udp|sctp|ether|none",
+	.help_str = "port config all rss all|ip|tcp|udp|sctp|ether|port|vxlan|geneve|nvgre|none",
 	.tokens = {
 		(void *)&cmd_config_rss_port,
 		(void *)&cmd_config_rss_keyword,
@@ -2511,16 +2536,20 @@ static void cmd_set_list_parsed(void *parsed_result,
 		nb_item = parse_item_list(res->list_of_items, "core",
 					  RTE_MAX_LCORE,
 					  parsed_items.lcorelist, 1);
-		if (nb_item > 0)
+		if (nb_item > 0) {
 			set_fwd_lcores_list(parsed_items.lcorelist, nb_item);
+			fwd_config_setup();
+		}
 		return;
 	}
 	if (!strcmp(res->list_name, "portlist")) {
 		nb_item = parse_item_list(res->list_of_items, "port",
 					  RTE_MAX_ETHPORTS,
 					  parsed_items.portlist, 1);
-		if (nb_item > 0)
+		if (nb_item > 0) {
 			set_fwd_ports_list(parsed_items.portlist, nb_item);
+			fwd_config_setup();
+		}
 	}
 }
 
@@ -2564,10 +2593,13 @@ static void cmd_set_mask_parsed(void *parsed_result,
 		printf("Please stop forwarding first\n");
 		return;
 	}
-	if (!strcmp(res->mask, "coremask"))
+	if (!strcmp(res->mask, "coremask")) {
 		set_fwd_lcores_mask(res->hexavalue);
-	else if (!strcmp(res->mask, "portmask"))
+		fwd_config_setup();
+	} else if (!strcmp(res->mask, "portmask")) {
 		set_fwd_ports_mask(res->hexavalue);
+		fwd_config_setup();
+	}
 }
 
 cmdline_parse_token_string_t cmd_setmask_set =
@@ -2604,11 +2636,13 @@ static void cmd_set_parsed(void *parsed_result,
 			   __attribute__((unused)) void *data)
 {
 	struct cmd_set_result *res = parsed_result;
-	if (!strcmp(res->what, "nbport"))
+	if (!strcmp(res->what, "nbport")) {
 		set_fwd_ports_number(res->value);
-	else if (!strcmp(res->what, "nbcore"))
+		fwd_config_setup();
+	} else if (!strcmp(res->what, "nbcore")) {
 		set_fwd_lcores_number(res->value);
-	else if (!strcmp(res->what, "burst"))
+		fwd_config_setup();
+	} else if (!strcmp(res->what, "burst"))
 		set_nb_pkt_per_burst(res->value);
 	else if (!strcmp(res->what, "verbose"))
 		set_verbose_level(res->value);
@@ -2721,6 +2755,74 @@ cmdline_parse_inst_t cmd_set_txsplit = {
 	},
 };
 
+/* *** CONFIG TX QUEUE FLAGS *** */
+
+struct cmd_config_txqflags_result {
+	cmdline_fixed_string_t port;
+	cmdline_fixed_string_t config;
+	cmdline_fixed_string_t all;
+	cmdline_fixed_string_t what;
+	int32_t hexvalue;
+};
+
+static void cmd_config_txqflags_parsed(void *parsed_result,
+				__attribute__((unused)) struct cmdline *cl,
+				__attribute__((unused)) void *data)
+{
+	struct cmd_config_txqflags_result *res = parsed_result;
+
+	if (!all_ports_stopped()) {
+		printf("Please stop all ports first\n");
+		return;
+	}
+
+	if (strcmp(res->what, "txqflags")) {
+		printf("Unknown parameter\n");
+		return;
+	}
+
+	if (res->hexvalue >= 0) {
+		txq_flags = res->hexvalue;
+	} else {
+		printf("txqflags must be >= 0\n");
+		return;
+	}
+
+	init_port_config();
+
+	cmd_reconfig_device_queue(RTE_PORT_ALL, 1, 1);
+}
+
+cmdline_parse_token_string_t cmd_config_txqflags_port =
+	TOKEN_STRING_INITIALIZER(struct cmd_config_txqflags_result, port,
+				 "port");
+cmdline_parse_token_string_t cmd_config_txqflags_config =
+	TOKEN_STRING_INITIALIZER(struct cmd_config_txqflags_result, config,
+				 "config");
+cmdline_parse_token_string_t cmd_config_txqflags_all =
+	TOKEN_STRING_INITIALIZER(struct cmd_config_txqflags_result, all,
+				 "all");
+cmdline_parse_token_string_t cmd_config_txqflags_what =
+	TOKEN_STRING_INITIALIZER(struct cmd_config_txqflags_result, what,
+				 "txqflags");
+cmdline_parse_token_num_t cmd_config_txqflags_value =
+	TOKEN_NUM_INITIALIZER(struct cmd_config_txqflags_result,
+				hexvalue, INT32);
+
+cmdline_parse_inst_t cmd_config_txqflags = {
+	.f = cmd_config_txqflags_parsed,
+	.data = NULL,
+	.help_str = "port config all txqflags value",
+	.tokens = {
+		(void *)&cmd_config_txqflags_port,
+		(void *)&cmd_config_txqflags_config,
+		(void *)&cmd_config_txqflags_all,
+		(void *)&cmd_config_txqflags_what,
+		(void *)&cmd_config_txqflags_value,
+		NULL,
+	},
+};
+
 /* *** ADD/REMOVE ALL VLAN IDENTIFIERS TO/FROM A PORT VLAN RX FILTER *** */
 struct cmd_rx_vlan_filter_all_result {
 	cmdline_fixed_string_t rx_vlan;
@@ -4480,6 +4582,7 @@ static void cmd_set_fwd_mode_parsed(void *parsed_result,
 {
 	struct cmd_set_fwd_mode_result *res = parsed_result;
 
+	retry_enabled = 0;
 	set_pkt_forwarding_mode(res->mode);
 }
 
@@ -4525,6 +4628,74 @@ static void cmd_set_fwd_mode_init(void)
 	token_struct->string_data.str = token;
 }
 
+/* *** SET RETRY FORWARDING MODE *** */
+struct cmd_set_fwd_retry_mode_result {
+	cmdline_fixed_string_t set;
+	cmdline_fixed_string_t fwd;
+	cmdline_fixed_string_t mode;
+	cmdline_fixed_string_t retry;
+};
+
+static void cmd_set_fwd_retry_mode_parsed(void *parsed_result,
+			    __attribute__((unused)) struct cmdline *cl,
+			    __attribute__((unused)) void *data)
+{
+	struct cmd_set_fwd_retry_mode_result *res = parsed_result;
+
+	retry_enabled = 1;
+	set_pkt_forwarding_mode(res->mode);
+}
+
+cmdline_parse_token_string_t cmd_setfwd_retry_set =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_retry_mode_result,
+			set, "set");
+cmdline_parse_token_string_t cmd_setfwd_retry_fwd =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_retry_mode_result,
+			fwd, "fwd");
+cmdline_parse_token_string_t cmd_setfwd_retry_mode =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_retry_mode_result,
+			mode,
+		"" /* defined at init */);
+cmdline_parse_token_string_t cmd_setfwd_retry_retry =
+	TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_retry_mode_result,
+			retry, "retry");
+
+cmdline_parse_inst_t cmd_set_fwd_retry_mode = {
+	.f = cmd_set_fwd_retry_mode_parsed,
+	.data = NULL,
+	.help_str = NULL, /* defined at init */
+	.tokens = {
+		(void *)&cmd_setfwd_retry_set,
+		(void *)&cmd_setfwd_retry_fwd,
+		(void *)&cmd_setfwd_retry_mode,
+		(void *)&cmd_setfwd_retry_retry,
+		NULL,
+	},
+};
+
+static void cmd_set_fwd_retry_mode_init(void)
+{
+	char *modes, *c;
+	static char token[128];
+	static char help[256];
+	cmdline_parse_token_string_t *token_struct;
+
+	modes = list_pkt_forwarding_retry_modes();
+	snprintf(help, sizeof(help), "set fwd %s retry - "
+		"set packet forwarding mode with retry", modes);
+	cmd_set_fwd_retry_mode.help_str = help;
+
+	/* string token separator is # */
+	for (c = token; *modes != '\0'; modes++)
+		if (*modes == '|')
+			*c++ = '#';
+		else
+			*c++ = *modes;
+	token_struct = (cmdline_parse_token_string_t *)
+		cmd_set_fwd_retry_mode.tokens[2];
+	token_struct->string_data.str = token;
+}
+
 /* *** SET BURST TX DELAY TIME RETRY NUMBER *** */
 struct cmd_set_burst_tx_retry_result {
 	cmdline_fixed_string_t set;
@@ -5240,6 +5411,46 @@ cmdline_parse_inst_t cmd_start_tx_first = {
 	},
 };
 
+/* *** START FORWARDING WITH N TX BURST FIRST *** */
+struct cmd_start_tx_first_n_result {
+	cmdline_fixed_string_t start;
+	cmdline_fixed_string_t tx_first;
+	uint32_t tx_num;
+};
+
+static void
+cmd_start_tx_first_n_parsed(void *parsed_result,
+			  __attribute__((unused)) struct cmdline *cl,
+			  __attribute__((unused)) void *data)
+{
+	struct cmd_start_tx_first_n_result *res = parsed_result;
+
+	start_packet_forwarding(res->tx_num);
+}
+
+cmdline_parse_token_string_t cmd_start_tx_first_n_start =
+	TOKEN_STRING_INITIALIZER(struct cmd_start_tx_first_n_result,
+			start, "start");
+cmdline_parse_token_string_t cmd_start_tx_first_n_tx_first =
+	TOKEN_STRING_INITIALIZER(struct cmd_start_tx_first_n_result,
+			tx_first, "tx_first");
+cmdline_parse_token_num_t cmd_start_tx_first_n_tx_num =
+	TOKEN_NUM_INITIALIZER(struct cmd_start_tx_first_n_result,
+			tx_num, UINT32);
+
+cmdline_parse_inst_t cmd_start_tx_first_n = {
+	.f = cmd_start_tx_first_n_parsed,
+	.data = NULL,
+	.help_str = "start packet forwarding, after sending <num> "
+		"bursts of packets",
+	.tokens = {
+		(void *)&cmd_start_tx_first_n_start,
+		(void *)&cmd_start_tx_first_n_tx_first,
+		(void *)&cmd_start_tx_first_n_tx_num,
+		NULL,
+	},
+};
+
 /* *** SET LINK UP *** */
 struct cmd_set_link_up_result {
 	cmdline_fixed_string_t set;
@@ -5336,7 +5547,7 @@ static void cmd_showcfg_parsed(void *parsed_result,
 	else if (!strcmp(res->what, "cores"))
 		fwd_lcores_config_display();
 	else if (!strcmp(res->what, "fwd"))
-		fwd_config_display();
+		pkt_fwd_config_display(&cur_fwd_config);
 	else if (!strcmp(res->what, "txpkts"))
 		show_tx_pkt_segments();
 }
@@ -7191,8 +7402,6 @@ static void cmd_dump_parsed(void *parsed_result,
 		rte_dump_physmem_layout(stdout);
 	else if (!strcmp(res->dump, "dump_memzone"))
 		rte_memzone_dump(stdout);
-	else if (!strcmp(res->dump, "dump_log_history"))
-		rte_log_dump_history(stdout);
 	else if (!strcmp(res->dump, "dump_struct_sizes"))
 		dump_struct_sizes();
 	else if (!strcmp(res->dump, "dump_ring"))
@@ -7207,7 +7416,6 @@ cmdline_parse_token_string_t cmd_dump_dump =
 	TOKEN_STRING_INITIALIZER(struct cmd_dump_result, dump,
 		"dump_physmem#"
 		"dump_memzone#"
-		"dump_log_history#"
 		"dump_struct_sizes#"
 		"dump_ring#"
 		"dump_mempool#"
@@ -9299,6 +9507,10 @@ flowtype_to_str(uint16_t ftype)
 		{"ipv6-sctp", RTE_ETH_FLOW_NONFRAG_IPV6_SCTP},
 		{"ipv6-other", RTE_ETH_FLOW_NONFRAG_IPV6_OTHER},
 		{"l2_payload", RTE_ETH_FLOW_L2_PAYLOAD},
+		{"port", RTE_ETH_FLOW_PORT},
+		{"vxlan", RTE_ETH_FLOW_VXLAN},
+		{"geneve", RTE_ETH_FLOW_GENEVE},
+		{"nvgre", RTE_ETH_FLOW_NVGRE},
 	};
 
 	for (i = 0; i < RTE_DIM(ftype_table); i++) {
@@ -10399,6 +10611,7 @@ cmdline_parse_ctx_t main_ctx[] = {
 	(cmdline_parse_inst_t *)&cmd_showcfg,
 	(cmdline_parse_inst_t *)&cmd_start,
 	(cmdline_parse_inst_t *)&cmd_start_tx_first,
+	(cmdline_parse_inst_t *)&cmd_start_tx_first_n,
 	(cmdline_parse_inst_t *)&cmd_set_link_up,
 	(cmdline_parse_inst_t *)&cmd_set_link_down,
 	(cmdline_parse_inst_t *)&cmd_reset,
@@ -10408,6 +10621,7 @@ cmdline_parse_ctx_t main_ctx[] = {
 	(cmdline_parse_inst_t *)&cmd_set_fwd_list,
 	(cmdline_parse_inst_t *)&cmd_set_fwd_mask,
 	(cmdline_parse_inst_t *)&cmd_set_fwd_mode,
+	(cmdline_parse_inst_t *)&cmd_set_fwd_retry_mode,
 	(cmdline_parse_inst_t *)&cmd_set_burst_tx_retry,
 	(cmdline_parse_inst_t *)&cmd_set_promisc_mode_one,
 	(cmdline_parse_inst_t *)&cmd_set_promisc_mode_all,
@@ -10478,6 +10692,7 @@ cmdline_parse_ctx_t main_ctx[] = {
 	(cmdline_parse_inst_t *)&cmd_config_rx_mode_flag,
 	(cmdline_parse_inst_t *)&cmd_config_rss,
 	(cmdline_parse_inst_t *)&cmd_config_rxtx_queue,
+	(cmdline_parse_inst_t *)&cmd_config_txqflags,
 	(cmdline_parse_inst_t *)&cmd_config_rss_reta,
 	(cmdline_parse_inst_t *)&cmd_showport_reta,
 	(cmdline_parse_inst_t *)&cmd_config_burst,
@@ -10546,6 +10761,7 @@ prompt(void)
 {
 	/* initialize non-constant commands */
 	cmd_set_fwd_mode_init();
+	cmd_set_fwd_retry_mode_init();
 
 	testpmd_cl = cmdline_stdin_new(main_ctx, "testpmd> ");
 	if (testpmd_cl == NULL)
diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c
index b1bbec6d..c5865f95 100644
--- a/app/test-pmd/config.c
+++ b/app/test-pmd/config.c
@@ -1,7 +1,7 @@
 /*-
  *   BSD LICENSE
  *
- *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
  *   All rights reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
@@ -92,6 +92,7 @@
 #include <rte_ether.h>
 #include <rte_ethdev.h>
 #include <rte_string_fns.h>
+#include <rte_cycles.h>
 
 #include "testpmd.h"
 
@@ -137,6 +138,11 @@ static const struct rss_type_info rss_type_table[] = {
 	{ "ipv6-ex", ETH_RSS_IPV6_EX },
 	{ "ipv6-tcp-ex", ETH_RSS_IPV6_TCP_EX },
 	{ "ipv6-udp-ex", ETH_RSS_IPV6_UDP_EX },
+	{ "port", ETH_RSS_PORT },
+	{ "vxlan", ETH_RSS_VXLAN },
+	{ "geneve", ETH_RSS_GENEVE },
+	{ "nvgre", ETH_RSS_NVGRE },
+
 };
 
 static void
@@ -150,6 +156,11 @@ print_ethaddr(const char *name, struct ether_addr *eth_addr)
 void
 nic_stats_display(portid_t port_id)
 {
+	static uint64_t prev_pkts_rx[RTE_MAX_ETHPORTS];
+	static uint64_t prev_pkts_tx[RTE_MAX_ETHPORTS];
+	static uint64_t prev_cycles[RTE_MAX_ETHPORTS];
+	uint64_t diff_pkts_rx, diff_pkts_tx, diff_cycles;
+	uint64_t mpps_rx, mpps_tx;
 	struct rte_eth_stats stats;
 	struct rte_port *port = &ports[port_id];
 	uint8_t i;
@@ -209,6 +220,23 @@ nic_stats_display(portid_t port_id)
 		}
 	}
 
+	diff_cycles = prev_cycles[port_id];
+	prev_cycles[port_id] = rte_rdtsc();
+	if (diff_cycles > 0)
+		diff_cycles = prev_cycles[port_id] - diff_cycles;
+
+	diff_pkts_rx = stats.ipackets - prev_pkts_rx[port_id];
+	diff_pkts_tx = stats.opackets - prev_pkts_tx[port_id];
+	prev_pkts_rx[port_id] = stats.ipackets;
+	prev_pkts_tx[port_id] = stats.opackets;
+	mpps_rx = diff_cycles > 0 ?
+		diff_pkts_rx * rte_get_tsc_hz() / diff_cycles : 0;
+	mpps_tx = diff_cycles > 0 ?
+		diff_pkts_tx * rte_get_tsc_hz() / diff_cycles : 0;
+	printf("\n  Throughput (since last show)\n");
+	printf("  Rx-pps: %12"PRIu64"\n  Tx-pps: %12"PRIu64"\n",
+			mpps_rx, mpps_tx);
+
 	printf("  %s############################%s\n",
 	       nic_stats_border, nic_stats_border);
 }
@@ -232,29 +260,56 @@ nic_stats_clear(portid_t port_id)
 void
 nic_xstats_display(portid_t port_id)
 {
-	struct rte_eth_xstats *xstats;
-	int len, ret, i;
+	struct rte_eth_xstat *xstats;
+	int cnt_xstats, idx_xstat;
+	struct rte_eth_xstat_name *xstats_names;
 
 	printf("###### NIC extended statistics for port %-2d\n", port_id);
+	if (!rte_eth_dev_is_valid_port(port_id)) {
+		printf("Error: Invalid port number %i\n", port_id);
+		return;
+	}
 
-	len = rte_eth_xstats_get(port_id, NULL, 0);
-	if (len < 0) {
-		printf("Cannot get xstats count\n");
+	/* Get count */
+	cnt_xstats = rte_eth_xstats_get_names(port_id, NULL, 0);
+	if (cnt_xstats  < 0) {
+		printf("Error: Cannot get count of xstats\n");
 		return;
 	}
-	xstats = malloc(sizeof(xstats[0]) * len);
+
+	/* Get id-name lookup table */
+	xstats_names = malloc(sizeof(struct rte_eth_xstat_name) * cnt_xstats);
+	if (xstats_names == NULL) {
+		printf("Cannot allocate memory for xstats lookup\n");
+		return;
+	}
+	if (cnt_xstats != rte_eth_xstats_get_names(
+			port_id, xstats_names, cnt_xstats)) {
+		printf("Error: Cannot get xstats lookup\n");
+		free(xstats_names);
+		return;
+	}
+
+	/* Get stats themselves */
+	xstats = malloc(sizeof(struct rte_eth_xstat) * cnt_xstats);
 	if (xstats == NULL) {
 		printf("Cannot allocate memory for xstats\n");
+		free(xstats_names);
 		return;
 	}
-	ret = rte_eth_xstats_get(port_id, xstats, len);
-	if (ret < 0 || ret > len) {
-		printf("Cannot get xstats\n");
+	if (cnt_xstats != rte_eth_xstats_get(port_id, xstats, cnt_xstats)) {
+		printf("Error: Unable to get xstats\n");
+		free(xstats_names);
 		free(xstats);
 		return;
 	}
-	for (i = 0; i < len; i++)
-		printf("%s: %"PRIu64"\n", xstats[i].name, xstats[i].value);
+
+	/* Display xstats */
+	for (idx_xstat = 0; idx_xstat < cnt_xstats; idx_xstat++)
+		printf("%s: %"PRIu64"\n",
+			xstats_names[idx_xstat].name,
+			xstats[idx_xstat].value);
+	free(xstats_names);
 	free(xstats);
 }
 
@@ -893,8 +948,9 @@ fwd_lcores_config_display(void)
 void
 rxtx_config_display(void)
 {
-	printf("  %s packet forwarding - CRC stripping %s - "
+	printf("  %s packet forwarding%s - CRC stripping %s - "
 	       "packets/burst=%d\n", cur_fwd_eng->fwd_mode_name,
+	       retry_enabled == 0 ? "" : " with retry",
 	       rx_mode.hw_strip_crc ? "enabled" : "disabled",
 	       nb_pkt_per_burst);
 
@@ -1131,6 +1187,7 @@ simple_fwd_config_setup(void)
 		fwd_streams[i]->tx_port   = fwd_ports_ids[j];
 		fwd_streams[i]->tx_queue  = 0;
 		fwd_streams[i]->peer_addr = j;
+		fwd_streams[i]->retry_enabled = retry_enabled;
 
 		if (port_topology == PORT_TOPOLOGY_PAIRED) {
 			fwd_streams[j]->rx_port   = fwd_ports_ids[j];
@@ -1138,6 +1195,7 @@ simple_fwd_config_setup(void)
 			fwd_streams[j]->tx_port   = fwd_ports_ids[i];
 			fwd_streams[j]->tx_queue  = 0;
 			fwd_streams[j]->peer_addr = i;
+			fwd_streams[j]->retry_enabled = retry_enabled;
 		}
 	}
 }
@@ -1173,10 +1231,8 @@ rss_fwd_config_setup(void)
 	cur_fwd_config.nb_fwd_ports = nb_fwd_ports;
 	cur_fwd_config.nb_fwd_streams =
 		(streamid_t) (nb_q * cur_fwd_config.nb_fwd_ports);
-	if (cur_fwd_config.nb_fwd_streams > cur_fwd_config.nb_fwd_lcores)
-		cur_fwd_config.nb_fwd_streams =
-			(streamid_t)cur_fwd_config.nb_fwd_lcores;
-	else
+
+	if (cur_fwd_config.nb_fwd_streams < cur_fwd_config.nb_fwd_lcores)
 		cur_fwd_config.nb_fwd_lcores =
 			(lcoreid_t)cur_fwd_config.nb_fwd_streams;
 
@@ -1185,7 +1241,7 @@ rss_fwd_config_setup(void)
 
 	setup_fwd_config_of_each_lcore(&cur_fwd_config);
 	rxp = 0; rxq = 0;
-	for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) {
+	for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_streams; lc_id++) {
 		struct fwd_stream *fs;
 
 		fs = fwd_streams[lc_id];
@@ -1206,6 +1262,7 @@ rss_fwd_config_setup(void)
 		fs->tx_port = fwd_ports_ids[txp];
 		fs->tx_queue = rxq;
 		fs->peer_addr = fs->tx_port;
+		fs->retry_enabled = retry_enabled;
 		rxq = (queueid_t) (rxq + 1);
 		if (rxq < nb_q)
 			continue;
@@ -1280,6 +1337,7 @@ dcb_fwd_config_setup(void)
 				fs->tx_port = fwd_ports_ids[txp];
 				fs->tx_queue = txq + j % nb_tx_queue;
 				fs->peer_addr = fs->tx_port;
+				fs->retry_enabled = retry_enabled;
 			}
 			fwd_lcores[lc_id]->stream_nb +=
 				rxp_dcb_info.tc_queue.tc_rxq[i][tc].nb_queue;
@@ -1350,6 +1408,7 @@ icmp_echo_config_setup(void)
 			fs->tx_port = fs->rx_port;
 			fs->tx_queue = rxq;
 			fs->peer_addr = fs->tx_port;
+			fs->retry_enabled = retry_enabled;
 			if (verbose_level > 0)
 				printf("  stream=%d port=%d rxq=%d txq=%d\n",
 				       sm_id, fs->rx_port, fs->rx_queue,
@@ -1381,21 +1440,22 @@ fwd_config_setup(void)
 		simple_fwd_config_setup();
 }
 
-static void
+void
 pkt_fwd_config_display(struct fwd_config *cfg)
 {
 	struct fwd_stream *fs;
 	lcoreid_t  lc_id;
 	streamid_t sm_id;
 
-	printf("%s packet forwarding - ports=%d - cores=%d - streams=%d - "
+	printf("%s packet forwarding%s - ports=%d - cores=%d - streams=%d - "
 		"NUMA support %s, MP over anonymous pages %s\n",
 		cfg->fwd_eng->fwd_mode_name,
+		retry_enabled == 0 ? "" : " with retry",
 		cfg->nb_fwd_ports, cfg->nb_fwd_lcores, cfg->nb_fwd_streams,
 		numa_support == 1 ? "enabled" : "disabled",
 		mp_anon != 0 ? "enabled" : "disabled");
 
-	if (strcmp(cfg->fwd_eng->fwd_mode_name, "mac_retry") == 0)
+	if (retry_enabled)
 		printf("TX retry num: %u, delay between TX retries: %uus\n",
 			burst_tx_retry_num, burst_tx_delay_time);
 	for (lc_id = 0; lc_id < cfg->nb_fwd_lcores; lc_id++) {
@@ -1420,14 +1480,6 @@ pkt_fwd_config_display(struct fwd_config *cfg)
 	printf("\n");
 }
 
-
-void
-fwd_config_display(void)
-{
-	fwd_config_setup();
-	pkt_fwd_config_display(&cur_fwd_config);
-}
-
 int
 set_fwd_lcores_list(unsigned int *lcorelist, unsigned int nb_lc)
 {
@@ -1565,6 +1617,22 @@ set_fwd_ports_number(uint16_t nb_pt)
 	       (unsigned int) nb_fwd_ports);
 }
 
+int
+port_is_forwarding(portid_t port_id)
+{
+	unsigned int i;
+
+	if (port_id_is_invalid(port_id, ENABLED_WARN))
+		return -1;
+
+	for (i = 0; i < nb_fwd_ports; i++) {
+		if (fwd_ports_ids[i] == port_id)
+			return 1;
+	}
+
+	return 0;
+}
+
 void
 set_nb_pkt_per_burst(uint16_t nb)
 {
@@ -1673,8 +1741,35 @@ list_pkt_forwarding_modes(void)
 
 	if (strlen (fwd_modes) == 0) {
 		while ((fwd_eng = fwd_engines[i++]) != NULL) {
-			strcat(fwd_modes, fwd_eng->fwd_mode_name);
-			strcat(fwd_modes, separator);
+			strncat(fwd_modes, fwd_eng->fwd_mode_name,
+					sizeof(fwd_modes) - strlen(fwd_modes) - 1);
+			strncat(fwd_modes, separator,
+					sizeof(fwd_modes) - strlen(fwd_modes) - 1);
+		}
+		fwd_modes[strlen(fwd_modes) - strlen(separator)] = '\0';
+	}
+
+	return fwd_modes;
+}
+
+char*
+list_pkt_forwarding_retry_modes(void)
+{
+	static char fwd_modes[128] = "";
+	const char *separator = "|";
+	struct fwd_engine *fwd_eng;
+	unsigned i = 0;
+
+	if (strlen(fwd_modes) == 0) {
+		while ((fwd_eng = fwd_engines[i++]) != NULL) {
+			if (fwd_eng == &rx_only_engine)
+				continue;
+			strncat(fwd_modes, fwd_eng->fwd_mode_name,
+					sizeof(fwd_modes) -
+					strlen(fwd_modes) - 1);
+			strncat(fwd_modes, separator,
+					sizeof(fwd_modes) -
+					strlen(fwd_modes) - 1);
 		}
 		fwd_modes[strlen(fwd_modes) - strlen(separator)] = '\0';
 	}
@@ -1691,8 +1786,9 @@ set_pkt_forwarding_mode(const char *fwd_mode_name)
 	i = 0;
 	while ((fwd_eng = fwd_engines[i]) != NULL) {
 		if (! strcmp(fwd_eng->fwd_mode_name, fwd_mode_name)) {
-			printf("Set %s packet forwarding mode\n",
-			       fwd_mode_name);
+			printf("Set %s packet forwarding mode%s\n",
+			       fwd_mode_name,
+			       retry_enabled == 0 ? "" : " with retry");
 			cur_fwd_eng = fwd_eng;
 			return;
 		}
@@ -2028,6 +2124,10 @@ flowtype_to_str(uint16_t flow_type)
 		{"ipv6-sctp", RTE_ETH_FLOW_NONFRAG_IPV6_SCTP},
 		{"ipv6-other", RTE_ETH_FLOW_NONFRAG_IPV6_OTHER},
 		{"l2_payload", RTE_ETH_FLOW_L2_PAYLOAD},
+		{"port", RTE_ETH_FLOW_PORT},
+		{"vxlan", RTE_ETH_FLOW_VXLAN},
+		{"geneve", RTE_ETH_FLOW_GENEVE},
+		{"nvgre", RTE_ETH_FLOW_NVGRE},
 	};
 
 	for (i = 0; i < RTE_DIM(flowtype_str_table); i++) {
diff --git a/app/test-pmd/csumonly.c b/app/test-pmd/csumonly.c
index 7e4f6620..ac4bd8f4 100644
--- a/app/test-pmd/csumonly.c
+++ b/app/test-pmd/csumonly.c
@@ -643,6 +643,7 @@ pkt_burst_checksum_forward(struct fwd_stream *fs)
 	uint16_t i;
 	uint64_t ol_flags;
 	uint16_t testpmd_ol_flags;
+	uint32_t retry;
 	uint32_t rx_bad_ip_csum;
 	uint32_t rx_bad_l4_csum;
 	struct testpmd_offload_info info;
@@ -676,6 +677,9 @@ pkt_burst_checksum_forward(struct fwd_stream *fs)
 	info.tso_segsz = txp->tso_segsz;
 
 	for (i = 0; i < nb_rx; i++) {
+		if (likely(i < nb_rx - 1))
+			rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[i + 1],
+						       void *));
 
 		ol_flags = 0;
 		info.is_tunnel = 0;
@@ -845,6 +849,17 @@ pkt_burst_checksum_forward(struct fwd_stream *fs)
 		}
 	}
 	nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue, pkts_burst, nb_rx);
+	/*
+	 * Retry if necessary
+	 */
+	if (unlikely(nb_tx < nb_rx) && fs->retry_enabled) {
+		retry = 0;
+		while (nb_tx < nb_rx && retry++ < burst_tx_retry_num) {
+			rte_delay_us(burst_tx_delay_time);
+			nb_tx += rte_eth_tx_burst(fs->tx_port, fs->tx_queue,
+					&pkts_burst[nb_tx], nb_rx - nb_tx);
+		}
+	}
 	fs->tx_packets += nb_tx;
 	fs->rx_bad_ip_csum += rx_bad_ip_csum;
 	fs->rx_bad_l4_csum += rx_bad_l4_csum;
diff --git a/app/test-pmd/flowgen.c b/app/test-pmd/flowgen.c
index 0f307e85..a6abe91e 100644
--- a/app/test-pmd/flowgen.c
+++ b/app/test-pmd/flowgen.c
@@ -89,17 +89,6 @@ static struct ether_addr cfg_ether_dst	=
 #define IP_HDRLEN  0x05 /* default IP header length == five 32-bits words. */
 #define IP_VHL_DEF (IP_VERSION | IP_HDRLEN)
 
-static inline struct rte_mbuf *
-tx_mbuf_alloc(struct rte_mempool *mp)
-{
-	struct rte_mbuf *m;
-
-	m = __rte_mbuf_raw_alloc(mp);
-	__rte_mbuf_sanity_check_raw(m, 0);
-	return m;
-}
-
-
 static inline uint16_t
 ip_sum(const unaligned_uint16_t *hdr, int hdr_len)
 {
@@ -142,6 +131,7 @@ pkt_burst_flow_gen(struct fwd_stream *fs)
 	uint16_t nb_tx;
 	uint16_t nb_pkt;
 	uint16_t i;
+	uint32_t retry;
 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
 	uint64_t start_tsc;
 	uint64_t end_tsc;
@@ -167,7 +157,7 @@ pkt_burst_flow_gen(struct fwd_stream *fs)
 	ol_flags = ports[fs->tx_port].tx_ol_flags;
 
 	for (nb_pkt = 0; nb_pkt < nb_pkt_per_burst; nb_pkt++) {
-		pkt = tx_mbuf_alloc(mbp);
+		pkt = rte_mbuf_raw_alloc(mbp);
 		if (!pkt)
 			break;
 
@@ -218,6 +208,17 @@ pkt_burst_flow_gen(struct fwd_stream *fs)
 	}
 
 	nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue, pkts_burst, nb_pkt);
+	/*
+	 * Retry if necessary
+	 */
+	if (unlikely(nb_tx < nb_rx) && fs->retry_enabled) {
+		retry = 0;
+		while (nb_tx < nb_rx && retry++ < burst_tx_retry_num) {
+			rte_delay_us(burst_tx_delay_time);
+			nb_tx += rte_eth_tx_burst(fs->tx_port, fs->tx_queue,
+					&pkts_burst[nb_tx], nb_rx - nb_tx);
+		}
+	}
 	fs->tx_packets += nb_tx;
 
 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
diff --git a/app/test-pmd/icmpecho.c b/app/test-pmd/icmpecho.c
index e510f9bf..be308c9f 100644
--- a/app/test-pmd/icmpecho.c
+++ b/app/test-pmd/icmpecho.c
@@ -311,6 +311,7 @@ reply_to_icmp_echo_rqsts(struct fwd_stream *fs)
 	struct ipv4_hdr *ip_h;
 	struct icmp_hdr *icmp_h;
 	struct ether_addr eth_addr;
+	uint32_t retry;
 	uint32_t ip_addr;
 	uint16_t nb_rx;
 	uint16_t nb_tx;
@@ -346,6 +347,9 @@ reply_to_icmp_echo_rqsts(struct fwd_stream *fs)
 	fs->rx_packets += nb_rx;
 	nb_replies = 0;
 	for (i = 0; i < nb_rx; i++) {
+		if (likely(i < nb_rx - 1))
+			rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[i + 1],
+						       void *));
 		pkt = pkts_burst[i];
 		eth_h = rte_pktmbuf_mtod(pkt, struct ether_hdr *);
 		eth_type = RTE_BE_TO_CPU_16(eth_h->ether_type);
@@ -515,6 +519,20 @@ reply_to_icmp_echo_rqsts(struct fwd_stream *fs)
 	if (nb_replies > 0) {
 		nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue, pkts_burst,
 					 nb_replies);
+		/*
+		 * Retry if necessary
+		 */
+		if (unlikely(nb_tx < nb_replies) && fs->retry_enabled) {
+			retry = 0;
+			while (nb_tx < nb_replies &&
+					retry++ < burst_tx_retry_num) {
+				rte_delay_us(burst_tx_delay_time);
+				nb_tx += rte_eth_tx_burst(fs->tx_port,
+						fs->tx_queue,
+						&pkts_burst[nb_tx],
+						nb_replies - nb_tx);
+			}
+		}
 		fs->tx_packets += nb_tx;
 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 		fs->tx_burst_stats.pkt_burst_spread[nb_tx]++;
diff --git a/app/test-pmd/iofwd.c b/app/test-pmd/iofwd.c
index 8840d868..7b6033a5 100644
--- a/app/test-pmd/iofwd.c
+++ b/app/test-pmd/iofwd.c
@@ -80,6 +80,8 @@ pkt_burst_io_forward(struct fwd_stream *fs)
 	struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
 	uint16_t nb_rx;
 	uint16_t nb_tx;
+	uint32_t retry;
+
 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
 	uint64_t start_tsc;
 	uint64_t end_tsc;
@@ -93,16 +95,28 @@ pkt_burst_io_forward(struct fwd_stream *fs)
 	/*
 	 * Receive a burst of packets and forward them.
 	 */
-	nb_rx = rte_eth_rx_burst(fs->rx_port, fs->rx_queue, pkts_burst,
-				 nb_pkt_per_burst);
+	nb_rx = rte_eth_rx_burst(fs->rx_port, fs->rx_queue,
+			pkts_burst, nb_pkt_per_burst);
 	if (unlikely(nb_rx == 0))
 		return;
+	fs->rx_packets += nb_rx;
 
 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 	fs->rx_burst_stats.pkt_burst_spread[nb_rx]++;
 #endif
-	fs->rx_packets += nb_rx;
-	nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue, pkts_burst, nb_rx);
+	nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue,
+			pkts_burst, nb_rx);
+	/*
+	 * Retry if necessary
+	 */
+	if (unlikely(nb_tx < nb_rx) && fs->retry_enabled) {
+		retry = 0;
+		while (nb_tx < nb_rx && retry++ < burst_tx_retry_num) {
+			rte_delay_us(burst_tx_delay_time);
+			nb_tx += rte_eth_tx_burst(fs->tx_port, fs->tx_queue,
+					&pkts_burst[nb_tx], nb_rx - nb_tx);
+		}
+	}
 	fs->tx_packets += nb_tx;
 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 	fs->tx_burst_stats.pkt_burst_spread[nb_tx]++;
diff --git a/app/test-pmd/macfwd-retry.c b/app/test-pmd/macfwd-retry.c
deleted file mode 100644
index 3a96b3df..00000000
--- a/app/test-pmd/macfwd-retry.c
+++ /dev/null
@@ -1,164 +0,0 @@
-/*-
- *   BSD LICENSE
- *
- *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
- *   All rights reserved.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of Intel Corporation nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <stdarg.h>
-#include <string.h>
-#include <stdio.h>
-#include <errno.h>
-#include <stdint.h>
-#include <unistd.h>
-#include <inttypes.h>
-
-#include <sys/queue.h>
-#include <sys/stat.h>
-
-#include <rte_common.h>
-#include <rte_byteorder.h>
-#include <rte_log.h>
-#include <rte_debug.h>
-#include <rte_cycles.h>
-#include <rte_memory.h>
-#include <rte_memcpy.h>
-#include <rte_memzone.h>
-#include <rte_launch.h>
-#include <rte_eal.h>
-#include <rte_per_lcore.h>
-#include <rte_lcore.h>
-#include <rte_atomic.h>
-#include <rte_branch_prediction.h>
-#include <rte_ring.h>
-#include <rte_memory.h>
-#include <rte_mempool.h>
-#include <rte_mbuf.h>
-#include <rte_interrupts.h>
-#include <rte_pci.h>
-#include <rte_ether.h>
-#include <rte_ethdev.h>
-#include <rte_ip.h>
-#include <rte_string_fns.h>
-
-#include "testpmd.h"
-
-#define BURST_TX_WAIT_US 10
-#define BURST_TX_RETRIES 5
-
-/*
- * Global variables that control number of retires and
- * timeout (in us) between retires.
- */
-uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
-uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
-
-/*
- * Forwarding of packets in MAC mode with a wait and retry on TX to reduce packet loss.
- * Change the source and the destination Ethernet addressed of packets
- * before forwarding them.
- */
-static void
-pkt_burst_mac_retry_forward(struct fwd_stream *fs)
-{
-	struct rte_mbuf  *pkts_burst[MAX_PKT_BURST];
-	struct rte_mbuf  *mb;
-	struct ether_hdr *eth_hdr;
-	uint32_t retry;
-	uint16_t nb_rx;
-	uint16_t nb_tx;
-	uint16_t i;
-#ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
-	uint64_t start_tsc;
-	uint64_t end_tsc;
-	uint64_t core_cycles;
-#endif
-
-#ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
-	start_tsc = rte_rdtsc();
-#endif
-
-	/*
-	 * Receive a burst of packets and forward them.
-	 */
-	nb_rx = rte_eth_rx_burst(fs->rx_port, fs->rx_queue, pkts_burst,
-				 nb_pkt_per_burst);
-	if (unlikely(nb_rx == 0))
-		return;
-
-#ifdef RTE_TEST_PMD_RECORD_BURST_STATS
-	fs->rx_burst_stats.pkt_burst_spread[nb_rx]++;
-#endif
-	fs->rx_packets += nb_rx;
-	for (i = 0; i < nb_rx; i++) {
-		mb = pkts_burst[i];
-		eth_hdr = rte_pktmbuf_mtod(mb, struct ether_hdr *);
-		ether_addr_copy(&peer_eth_addrs[fs->peer_addr],
-				&eth_hdr->d_addr);
-		ether_addr_copy(&ports[fs->tx_port].eth_addr,
-				&eth_hdr->s_addr);
-	}
-	nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue, pkts_burst, nb_rx);
-
-	/*
-	 * If not all packets have been TX'd then wait and retry.
-	 */
-	if (unlikely(nb_tx < nb_rx)) {
-		for (retry = 0; retry < burst_tx_retry_num; retry++) {
-			rte_delay_us(burst_tx_delay_time);
-			nb_tx += rte_eth_tx_burst(fs->tx_port, fs->tx_queue,
-				&pkts_burst[nb_tx], nb_rx - nb_tx);
-			if (nb_tx == nb_rx)
-				break;
-		}
-	}
-
-	fs->tx_packets += nb_tx;
-#ifdef RTE_TEST_PMD_RECORD_BURST_STATS
-	fs->tx_burst_stats.pkt_burst_spread[nb_tx]++;
-#endif
-	if (unlikely(nb_tx < nb_rx)) {
-		fs->fwd_dropped += (nb_rx - nb_tx);
-		do {
-			rte_pktmbuf_free(pkts_burst[nb_tx]);
-		} while (++nb_tx < nb_rx);
-	}
-#ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
-	end_tsc = rte_rdtsc();
-	core_cycles = (end_tsc - start_tsc);
-	fs->core_cycles = (uint64_t) (fs->core_cycles + core_cycles);
-#endif
-}
-
-struct fwd_engine mac_retry_fwd_engine = {
-	.fwd_mode_name  = "mac_retry",
-	.port_fwd_begin = NULL,
-	.port_fwd_end   = NULL,
-	.packet_fwd     = pkt_burst_mac_retry_forward,
-};
diff --git a/app/test-pmd/macfwd.c b/app/test-pmd/macfwd.c
index 3b7fffb7..5d1c1617 100644
--- a/app/test-pmd/macfwd.c
+++ b/app/test-pmd/macfwd.c
@@ -81,6 +81,7 @@ pkt_burst_mac_forward(struct fwd_stream *fs)
 	struct rte_port  *txp;
 	struct rte_mbuf  *mb;
 	struct ether_hdr *eth_hdr;
+	uint32_t retry;
 	uint16_t nb_rx;
 	uint16_t nb_tx;
 	uint16_t i;
@@ -113,6 +114,9 @@ pkt_burst_mac_forward(struct fwd_stream *fs)
 	if (txp->tx_ol_flags & TESTPMD_TX_OFFLOAD_INSERT_QINQ)
 		ol_flags |= PKT_TX_QINQ_PKT;
 	for (i = 0; i < nb_rx; i++) {
+		if (likely(i < nb_rx - 1))
+			rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[i + 1],
+						       void *));
 		mb = pkts_burst[i];
 		eth_hdr = rte_pktmbuf_mtod(mb, struct ether_hdr *);
 		ether_addr_copy(&peer_eth_addrs[fs->peer_addr],
@@ -126,6 +130,18 @@ pkt_burst_mac_forward(struct fwd_stream *fs)
 		mb->vlan_tci_outer = txp->tx_vlan_id_outer;
 	}
 	nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue, pkts_burst, nb_rx);
+	/*
+	 * Retry if necessary
+	 */
+	if (unlikely(nb_tx < nb_rx) && fs->retry_enabled) {
+		retry = 0;
+		while (nb_tx < nb_rx && retry++ < burst_tx_retry_num) {
+			rte_delay_us(burst_tx_delay_time);
+			nb_tx += rte_eth_tx_burst(fs->tx_port, fs->tx_queue,
+					&pkts_burst[nb_tx], nb_rx - nb_tx);
+		}
+	}
+
 	fs->tx_packets += nb_tx;
 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 	fs->tx_burst_stats.pkt_burst_spread[nb_tx]++;
diff --git a/app/test-pmd/macswap.c b/app/test-pmd/macswap.c
index 154889d1..4b0dbeb5 100644
--- a/app/test-pmd/macswap.c
+++ b/app/test-pmd/macswap.c
@@ -84,6 +84,7 @@ pkt_burst_mac_swap(struct fwd_stream *fs)
 	uint16_t nb_rx;
 	uint16_t nb_tx;
 	uint16_t i;
+	uint32_t retry;
 	uint64_t ol_flags = 0;
 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
 	uint64_t start_tsc;
@@ -113,6 +114,9 @@ pkt_burst_mac_swap(struct fwd_stream *fs)
 	if (txp->tx_ol_flags & TESTPMD_TX_OFFLOAD_INSERT_QINQ)
 		ol_flags |= PKT_TX_QINQ_PKT;
 	for (i = 0; i < nb_rx; i++) {
+		if (likely(i < nb_rx - 1))
+			rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[i + 1],
+						       void *));
 		mb = pkts_burst[i];
 		eth_hdr = rte_pktmbuf_mtod(mb, struct ether_hdr *);
 
@@ -128,6 +132,17 @@ pkt_burst_mac_swap(struct fwd_stream *fs)
 		mb->vlan_tci_outer = txp->tx_vlan_id_outer;
 	}
 	nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue, pkts_burst, nb_rx);
+	/*
+	 * Retry if necessary
+	 */
+	if (unlikely(nb_tx < nb_rx) && fs->retry_enabled) {
+		retry = 0;
+		while (nb_tx < nb_rx && retry++ < burst_tx_retry_num) {
+			rte_delay_us(burst_tx_delay_time);
+			nb_tx += rte_eth_tx_burst(fs->tx_port, fs->tx_queue,
+					&pkts_burst[nb_tx], nb_rx - nb_tx);
+		}
+	}
 	fs->tx_packets += nb_tx;
 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS
 	fs->tx_burst_stats.pkt_burst_spread[nb_tx]++;
diff --git a/app/test-pmd/mempool_anon.c b/app/test-pmd/mempool_anon.c
deleted file mode 100644
index 47304329..00000000
--- a/app/test-pmd/mempool_anon.c
+++ /dev/null
@@ -1,201 +0,0 @@
-/*-
- *   BSD LICENSE
- *
- *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
- *   All rights reserved.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of Intel Corporation nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include "mempool_osdep.h"
-#include <rte_errno.h>
-
-#ifdef RTE_EXEC_ENV_LINUXAPP
-
-#include <fcntl.h>
-#include <unistd.h>
-#include <sys/mman.h>
-
-
-#define	PAGEMAP_FNAME		"/proc/self/pagemap"
-
-/*
- * the pfn (page frame number) are bits 0-54 (see pagemap.txt in linux
- * Documentation).
- */
-#define	PAGEMAP_PFN_BITS	54
-#define	PAGEMAP_PFN_MASK	RTE_LEN2MASK(PAGEMAP_PFN_BITS, phys_addr_t)
-
-
-static int
-get_phys_map(void *va, phys_addr_t pa[], uint32_t pg_num, uint32_t pg_sz)
-{
-	int32_t fd, rc;
-	uint32_t i, nb;
-	off_t ofs;
-
-	ofs = (uintptr_t)va / pg_sz * sizeof(*pa);
-	nb = pg_num * sizeof(*pa);
-
-	if ((fd = open(PAGEMAP_FNAME, O_RDONLY)) < 0)
-		return ENOENT;
-
-	if ((rc = pread(fd, pa, nb, ofs)) < 0 || (rc -= nb) != 0) {
-
-		RTE_LOG(ERR, USER1, "failed read of %u bytes from \'%s\' "
-			"at offset %zu, error code: %d\n",
-			nb, PAGEMAP_FNAME, (size_t)ofs, errno);
-		rc = ENOENT;
-	}
-
-	close(fd);
-
-	for (i = 0; i != pg_num; i++)
-		pa[i] = (pa[i] & PAGEMAP_PFN_MASK) * pg_sz;
-
-	return rc;
-}
-
-struct rte_mempool *
-mempool_anon_create(const char *name, unsigned elt_num, unsigned elt_size,
-		   unsigned cache_size, unsigned private_data_size,
-		   rte_mempool_ctor_t *mp_init, void *mp_init_arg,
-		   rte_mempool_obj_ctor_t *obj_init, void *obj_init_arg,
-		   int socket_id, unsigned flags)
-{
-	struct rte_mempool *mp;
-	phys_addr_t *pa;
-	char *va, *uv;
-	uint32_t n, pg_num, pg_shift, pg_sz, total_size;
-	size_t sz;
-	ssize_t usz;
-	int32_t rc;
-
-	rc = ENOMEM;
-	mp = NULL;
-
-	pg_sz = getpagesize();
-	if (rte_is_power_of_2(pg_sz) == 0) {
-		rte_errno = EINVAL;
-		return mp;
-	}
-
-	pg_shift = rte_bsf32(pg_sz);
-
-	total_size = rte_mempool_calc_obj_size(elt_size, flags, NULL);
-
-	/* calc max memory size and max number of pages needed. */
-	sz = rte_mempool_xmem_size(elt_num, total_size, pg_shift);
-	pg_num = sz >> pg_shift;
-
-	/* get chunk of virtually continuos memory.*/
-	if ((va = mmap(NULL, sz, PROT_READ | PROT_WRITE,
-			MAP_SHARED | MAP_ANONYMOUS | MAP_LOCKED,
-			-1, 0)) == MAP_FAILED) {
-		RTE_LOG(ERR, USER1, "%s(%s) failed mmap of %zu bytes, "
-			"error code: %d\n",
-			__func__, name, sz, errno);
-		rte_errno = rc;
-		return mp;
-	}
-
-	/* extract physical mappings of the allocated memory. */
-	if ((pa = calloc(pg_num, sizeof (*pa))) != NULL &&
-			(rc = get_phys_map(va, pa, pg_num, pg_sz)) == 0) {
-
-		/*
-		 * Check that allocated size is big enough to hold elt_num
-		 * objects and a calcualte how many bytes are actually required.
-		 */
-
-		if ((usz = rte_mempool_xmem_usage(va, elt_num, total_size, pa,
-				pg_num, pg_shift)) < 0) {
-
-			n = -usz;
-			rc = ENOENT;
-			RTE_LOG(ERR, USER1, "%s(%s) only %u objects from %u "
-				"requested can  be created over "
-				"mmaped region %p of %zu bytes\n",
-				__func__, name, n, elt_num, va, sz);
-		} else {
-
-			/* unmap unused pages if any */
-			if ((size_t)usz < sz) {
-
-				uv = va + usz;
-				usz = sz - usz;
-
-				RTE_LOG(INFO, USER1,
-					"%s(%s): unmap unused %zu of %zu "
-					"mmaped bytes @%p\n",
-					__func__, name, (size_t)usz, sz, uv);
-				munmap(uv, usz);
-				sz -= usz;
-				pg_num = sz >> pg_shift;
-			}
-
-			if ((mp = rte_mempool_xmem_create(name, elt_num,
-					elt_size, cache_size, private_data_size,
-					mp_init, mp_init_arg,
-					obj_init, obj_init_arg,
-					socket_id, flags, va, pa, pg_num,
-					pg_shift)) != NULL)
-
-				RTE_VERIFY(elt_num == mp->size);
-		}
-	}
-
-	if (mp == NULL) {
-		munmap(va, sz);
-		rte_errno = rc;
-	}
-
-	free(pa);
-	return mp;
-}
-
-#else /* RTE_EXEC_ENV_LINUXAPP */
-
-
-struct rte_mempool *
-mempool_anon_create(__rte_unused const char *name,
-	__rte_unused unsigned elt_num, __rte_unused unsigned elt_size,
-	__rte_unused unsigned cache_size,
-	__rte_unused unsigned private_data_size,
-	__rte_unused rte_mempool_ctor_t *mp_init,
-	__rte_unused void *mp_init_arg,
-	__rte_unused rte_mempool_obj_ctor_t *obj_init,
-	__rte_unused void *obj_init_arg,
-	__rte_unused int socket_id, __rte_unused unsigned flags)
-{
-	rte_errno = ENOTSUP;
-	return NULL;
-}
-
-#endif /* RTE_EXEC_ENV_LINUXAPP */
diff --git a/app/test-pmd/mempool_osdep.h b/app/test-pmd/mempool_osdep.h
deleted file mode 100644
index 6b8df68a..00000000
--- a/app/test-pmd/mempool_osdep.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/*-
- *   BSD LICENSE
- *
- *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
- *   All rights reserved.
- *
- *   Redistribution and use in source and binary forms, with or without
- *   modification, are permitted provided that the following conditions
- *   are met:
- *
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in
- *       the documentation and/or other materials provided with the
- *       distribution.
- *     * Neither the name of Intel Corporation nor the names of its
- *       contributors may be used to endorse or promote products derived
- *       from this software without specific prior written permission.
- *
- *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _MEMPOOL_OSDEP_H_
-#define _MEMPOOL_OSDEP_H_
-
-#include <rte_mempool.h>
-
-/**
- * @file
- * mempool OS specific header.
- */
-
-/*
- * Create mempool over objects from mmap(..., MAP_ANONYMOUS, ...).
- */
-struct rte_mempool *
-mempool_anon_create(const char *name, unsigned n, unsigned elt_size,
-	unsigned cache_size, unsigned private_data_size,
-	rte_mempool_ctor_t *mp_init, void *mp_init_arg,
-	rte_mempool_obj_ctor_t *obj_init, void *obj_init_arg,
-	int socket_id, unsigned flags);
-
-#endif /*_RTE_MEMPOOL_OSDEP_H_ */
diff --git a/app/test-pmd/parameters.c b/app/test-pmd/parameters.c
index 55572ebe..8792c2c6 100644
--- a/app/test-pmd/parameters.c
+++ b/app/test-pmd/parameters.c
@@ -526,6 +526,7 @@ launch_args_parse(int argc, char** argv)
 		{ "pkt-filter-drop-queue",      1, 0, 0 },
 		{ "crc-strip",                  0, 0, 0 },
 		{ "enable-rx-cksum",            0, 0, 0 },
+		{ "enable-scatter",             0, 0, 0 },
 		{ "disable-hw-vlan",            0, 0, 0 },
 		{ "disable-hw-vlan-filter",     0, 0, 0 },
 		{ "disable-hw-vlan-strip",      0, 0, 0 },
@@ -764,6 +765,8 @@ launch_args_parse(int argc, char** argv)
 			}
 			if (!strcmp(lgopts[opt_idx].name, "crc-strip"))
 				rx_mode.hw_strip_crc = 1;
+			if (!strcmp(lgopts[opt_idx].name, "enable-scatter"))
+				rx_mode.enable_scatter = 1;
 			if (!strcmp(lgopts[opt_idx].name, "enable-rx-cksum"))
 				rx_mode.hw_ip_checksum = 1;
 
diff --git a/app/test-pmd/rxonly.c b/app/test-pmd/rxonly.c
index 14555abc..fbf287dc 100644
--- a/app/test-pmd/rxonly.c
+++ b/app/test-pmd/rxonly.c
@@ -156,9 +156,9 @@ pkt_burst_receive(struct fwd_stream *fs)
 				printf("hash=0x%x ID=0x%x ",
 				       mb->hash.fdir.hash, mb->hash.fdir.id);
 		}
-		if (ol_flags & PKT_RX_VLAN_PKT)
+		if (ol_flags & PKT_RX_VLAN_STRIPPED)
 			printf(" - VLAN tci=0x%x", mb->vlan_tci);
-		if (ol_flags & PKT_RX_QINQ_PKT)
+		if (ol_flags & PKT_RX_QINQ_STRIPPED)
 			printf(" - QinQ VLAN tci=0x%x, VLAN tci outer=0x%x",
 					mb->vlan_tci, mb->vlan_tci_outer);
 		if (mb->packet_type) {
@@ -179,6 +179,9 @@ pkt_burst_receive(struct fwd_stream *fs)
 			case RTE_PTYPE_L2_ETHER_LLDP:
 				printf(" - (outer) L2 type: ETHER_LLDP");
 				break;
+			case RTE_PTYPE_L2_ETHER_NSH:
+				printf(" - (outer) L2 type: ETHER_NSH");
+				break;
 			default:
 				printf(" - (outer) L2 type: Unknown");
 				break;
diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c
index 26a174c1..06885ceb 100644
--- a/app/test-pmd/testpmd.c
+++ b/app/test-pmd/testpmd.c
@@ -49,6 +49,7 @@
 #include <inttypes.h>
 
 #include <rte_common.h>
+#include <rte_errno.h>
 #include <rte_byteorder.h>
 #include <rte_log.h>
 #include <rte_debug.h>
@@ -75,9 +76,11 @@
 #ifdef RTE_LIBRTE_PMD_XENVIRT
 #include <rte_eth_xenvirt.h>
 #endif
+#ifdef RTE_LIBRTE_PDUMP
+#include <rte_pdump.h>
+#endif
 
 #include "testpmd.h"
-#include "mempool_osdep.h"
 
 uint16_t verbose_level = 0; /**< Silent by default. */
 
@@ -144,7 +147,6 @@ streamid_t nb_fwd_streams;       /**< Is equal to (nb_ports * nb_rxq). */
 struct fwd_engine * fwd_engines[] = {
 	&io_fwd_engine,
 	&mac_fwd_engine,
-	&mac_retry_fwd_engine,
 	&mac_swap_engine,
 	&flow_gen_engine,
 	&rx_only_engine,
@@ -159,6 +161,9 @@ struct fwd_engine * fwd_engines[] = {
 
 struct fwd_config cur_fwd_config;
 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */
+uint32_t retry_enabled;
+uint32_t burst_tx_delay_time = BURST_TX_WAIT_US;
+uint32_t burst_tx_retry_num = BURST_TX_RETRIES;
 
 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */
 uint32_t param_total_num_mbufs = 0;  /**< number of mbufs in all pools - if
@@ -416,6 +421,10 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 	mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
 	mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name));
 
+	RTE_LOG(INFO, USER1,
+		"create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
+		pool_name, nb_mbuf, mbuf_seg_size, socket_id);
+
 #ifdef RTE_LIBRTE_PMD_XENVIRT
 	rte_mp = rte_mempool_gntalloc_create(pool_name, nb_mbuf, mb_size,
 		(unsigned) mb_mempool_cache,
@@ -427,22 +436,29 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
 
 	/* if the former XEN allocation failed fall back to normal allocation */
 	if (rte_mp == NULL) {
-		if (mp_anon != 0)
-			rte_mp = mempool_anon_create(pool_name, nb_mbuf,
-					mb_size, (unsigned) mb_mempool_cache,
-					sizeof(struct rte_pktmbuf_pool_private),
-					rte_pktmbuf_pool_init, NULL,
-					rte_pktmbuf_init, NULL,
-					socket_id, 0);
-		else
+		if (mp_anon != 0) {
+			rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
+				mb_size, (unsigned) mb_mempool_cache,
+				sizeof(struct rte_pktmbuf_pool_private),
+				socket_id, 0);
+
+			if (rte_mempool_populate_anon(rte_mp) == 0) {
+				rte_mempool_free(rte_mp);
+				rte_mp = NULL;
+			}
+			rte_pktmbuf_pool_init(rte_mp, NULL);
+			rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
+		} else {
 			/* wrapper to rte_mempool_create() */
 			rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
 				mb_mempool_cache, 0, mbuf_seg_size, socket_id);
+		}
 	}
 
 	if (rte_mp == NULL) {
-		rte_exit(EXIT_FAILURE, "Creation of mbuf pool for socket %u "
-						"failed\n", socket_id);
+		rte_exit(EXIT_FAILURE,
+			"Creation of mbuf pool for socket %u failed: %s\n",
+			socket_id, rte_strerror(rte_errno));
 	} else if (verbose_level > 0) {
 		rte_mempool_dump(stdout, rte_mp);
 	}
@@ -580,6 +596,8 @@ init_config(void)
 	/* Configuration of packet forwarding streams. */
 	if (init_fwd_streams() < 0)
 		rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n");
+
+	fwd_config_setup();
 }
 
 
@@ -981,6 +999,12 @@ start_packet_forwarding(int with_tx_first)
 		printf("Packet forwarding already started\n");
 		return;
 	}
+
+	if (init_fwd_streams() < 0) {
+		printf("Fail from init_fwd_streams()\n");
+		return;
+	}
+
 	if(dcb_test) {
 		for (i = 0; i < nb_fwd_ports; i++) {
 			pt_id = fwd_ports_ids[i];
@@ -1003,6 +1027,7 @@ start_packet_forwarding(int with_tx_first)
 		flush_fwd_rx_queues();
 
 	fwd_config_setup();
+	pkt_fwd_config_display(&cur_fwd_config);
 	rxtx_config_display();
 
 	for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) {
@@ -1036,8 +1061,11 @@ start_packet_forwarding(int with_tx_first)
 			for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
 				(*port_fwd_begin)(fwd_ports_ids[i]);
 		}
-		launch_packet_forwarding(run_one_txonly_burst_on_core);
-		rte_eal_mp_wait_lcore();
+		while (with_tx_first--) {
+			launch_packet_forwarding(
+					run_one_txonly_burst_on_core);
+			rte_eal_mp_wait_lcore();
+		}
 		port_fwd_end = tx_only_engine.port_fwd_end;
 		if (port_fwd_end != NULL) {
 			for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++)
@@ -1070,10 +1098,6 @@ stop_packet_forwarding(void)
 #endif
 	static const char *acc_stats_border = "+++++++++++++++";
 
-	if (all_ports_started() == 0) {
-		printf("Not all ports were started\n");
-		return;
-	}
 	if (test_done) {
 		printf("Packet forwarding not started\n");
 		return;
@@ -1268,19 +1292,9 @@ start_port(portid_t pid)
 	struct rte_port *port;
 	struct ether_addr mac_addr;
 
-	if (test_done == 0) {
-		printf("Please stop forwarding first\n");
-		return -1;
-	}
-
 	if (port_id_is_invalid(pid, ENABLED_WARN))
 		return 0;
 
-	if (init_fwd_streams() < 0) {
-		printf("Fail from init_fwd_streams()\n");
-		return -1;
-	}
-
 	if(dcb_config)
 		dcb_test = 1;
 	FOREACH_PORT(pi, ports) {
@@ -1351,7 +1365,7 @@ start_port(portid_t pid)
 					if (mp == NULL) {
 						printf("Failed to setup RX queue:"
 							"No mempool allocation"
-							"on the socket %d\n",
+							" on the socket %d\n",
 							rxring_numa[pi]);
 						return -1;
 					}
@@ -1359,17 +1373,23 @@ start_port(portid_t pid)
 					diag = rte_eth_rx_queue_setup(pi, qi,
 					     nb_rxd,rxring_numa[pi],
 					     &(port->rx_conf),mp);
-				}
-				else
+				} else {
+					struct rte_mempool *mp =
+						mbuf_pool_find(port->socket_id);
+					if (mp == NULL) {
+						printf("Failed to setup RX queue:"
+							"No mempool allocation"
+							" on the socket %d\n",
+							port->socket_id);
+						return -1;
+					}
 					diag = rte_eth_rx_queue_setup(pi, qi,
 					     nb_rxd,port->socket_id,
-					     &(port->rx_conf),
-				             mbuf_pool_find(port->socket_id));
-
+					     &(port->rx_conf), mp);
+				}
 				if (diag == 0)
 					continue;
 
-
 				/* Fail to setup rx queue, return */
 				if (rte_atomic16_cmpset(&(port->port_status),
 							RTE_PORT_HANDLING,
@@ -1424,10 +1444,6 @@ stop_port(portid_t pid)
 	struct rte_port *port;
 	int need_check_link_status = 0;
 
-	if (test_done == 0) {
-		printf("Please stop forwarding first\n");
-		return;
-	}
 	if (dcb_test) {
 		dcb_test = 0;
 		dcb_config = 0;
@@ -1442,6 +1458,16 @@ stop_port(portid_t pid)
 		if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
 			continue;
 
+		if (port_is_forwarding(pi) != 0 && test_done == 0) {
+			printf("Please remove port %d from forwarding configuration.\n", pi);
+			continue;
+		}
+
+		if (port_is_bonding_slave(pi)) {
+			printf("Please remove port %d from bonded device.\n", pi);
+			continue;
+		}
+
 		port = &ports[pi];
 		if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED,
 						RTE_PORT_HANDLING) == 0)
@@ -1466,11 +1492,6 @@ close_port(portid_t pid)
 	portid_t pi;
 	struct rte_port *port;
 
-	if (test_done == 0) {
-		printf("Please stop forwarding first\n");
-		return;
-	}
-
 	if (port_id_is_invalid(pid, ENABLED_WARN))
 		return;
 
@@ -1480,6 +1501,16 @@ close_port(portid_t pid)
 		if (pid != pi && pid != (portid_t)RTE_PORT_ALL)
 			continue;
 
+		if (port_is_forwarding(pi) != 0 && test_done == 0) {
+			printf("Please remove port %d from forwarding configuration.\n", pi);
+			continue;
+		}
+
+		if (port_is_bonding_slave(pi)) {
+			printf("Please remove port %d from bonded device.\n", pi);
+			continue;
+		}
+
 		port = &ports[pi];
 		if (rte_atomic16_cmpset(&(port->port_status),
 			RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) {
@@ -1497,7 +1528,7 @@ close_port(portid_t pid)
 
 		if (rte_atomic16_cmpset(&(port->port_status),
 			RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0)
-			printf("Port %d can not be set into stopped\n", pi);
+			printf("Port %d cannot be set to closed\n", pi);
 	}
 
 	printf("Done\n");
@@ -1506,7 +1537,8 @@ close_port(portid_t pid)
 void
 attach_port(char *identifier)
 {
-	portid_t i, j, pi = 0;
+	portid_t pi = 0;
+	unsigned int socket_id;
 
 	printf("Attaching a new port...\n");
 
@@ -1515,30 +1547,19 @@ attach_port(char *identifier)
 		return;
 	}
 
-	if (test_done == 0) {
-		printf("Please stop forwarding first\n");
-		return;
-	}
-
 	if (rte_eth_dev_attach(identifier, &pi))
 		return;
 
 	ports[pi].enabled = 1;
-	reconfig(pi, rte_eth_dev_socket_id(pi));
+	socket_id = (unsigned)rte_eth_dev_socket_id(pi);
+	/* if socket_id is invalid, set to 0 */
+	if (check_socket_id(socket_id) < 0)
+		socket_id = 0;
+	reconfig(pi, socket_id);
 	rte_eth_promiscuous_enable(pi);
 
 	nb_ports = rte_eth_dev_count();
 
-	/* set_default_fwd_ports_config(); */
-	memset(fwd_ports_ids, 0, sizeof(fwd_ports_ids));
-	i = 0;
-	FOREACH_PORT(j, ports) {
-		fwd_ports_ids[i] = j;
-		i++;
-	}
-	nb_cfg_ports = nb_ports;
-	nb_fwd_ports++;
-
 	ports[pi].port_status = RTE_PORT_STOPPED;
 
 	printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports);
@@ -1548,7 +1569,6 @@ attach_port(char *identifier)
 void
 detach_port(uint8_t port_id)
 {
-	portid_t i, pi = 0;
 	char name[RTE_ETH_NAME_MAX_LEN];
 
 	printf("Detaching a port...\n");
@@ -1564,16 +1584,6 @@ detach_port(uint8_t port_id)
 	ports[port_id].enabled = 0;
 	nb_ports = rte_eth_dev_count();
 
-	/* set_default_fwd_ports_config(); */
-	memset(fwd_ports_ids, 0, sizeof(fwd_ports_ids));
-	i = 0;
-	FOREACH_PORT(pi, ports) {
-		fwd_ports_ids[i] = pi;
-		i++;
-	}
-	nb_cfg_ports = nb_ports;
-	nb_fwd_ports--;
-
 	printf("Port '%s' is detached. Now total ports is %d\n",
 			name, nb_ports);
 	printf("Done\n");
@@ -1843,6 +1853,14 @@ void clear_port_slave_flag(portid_t slave_pid)
 	port->slave_flag = 0;
 }
 
+uint8_t port_is_bonding_slave(portid_t slave_pid)
+{
+	struct rte_port *port;
+
+	port = &ports[slave_pid];
+	return port->slave_flag;
+}
+
 const uint16_t vlan_tags[] = {
 		0,  1,  2,  3,  4,  5,  6,  7,
 		8,  9, 10, 11,  12, 13, 14, 15,
@@ -2018,6 +2036,10 @@ signal_handler(int signum)
 	if (signum == SIGINT || signum == SIGTERM) {
 		printf("\nSignal %d received, preparing to exit...\n",
 				signum);
+#ifdef RTE_LIBRTE_PDUMP
+		/* uninitialize packet capture framework */
+		rte_pdump_uninit();
+#endif
 		force_quit();
 		/* exit with the expected status */
 		signal(signum, SIG_DFL);
@@ -2038,6 +2060,11 @@ main(int argc, char** argv)
 	if (diag < 0)
 		rte_panic("Cannot init EAL\n");
 
+#ifdef RTE_LIBRTE_PDUMP
+	/* initialize packet capture framework */
+	rte_pdump_init(NULL);
+#endif
+
 	nb_ports = (portid_t) rte_eth_dev_count();
 	if (nb_ports == 0)
 		RTE_LOG(WARNING, EAL, "No probed ethernet devices\n");
diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h
index 0f72ca1f..2b281cc4 100644
--- a/app/test-pmd/testpmd.h
+++ b/app/test-pmd/testpmd.h
@@ -1,7 +1,7 @@
 /*-
  *   BSD LICENSE
  *
- *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
+ *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
  *   All rights reserved.
  *
  *   Redistribution and use in source and binary forms, with or without
@@ -103,6 +103,8 @@ struct fwd_stream {
 	queueid_t  tx_queue;  /**< TX queue to send forwarded packets */
 	streamid_t peer_addr; /**< index of peer ethernet address of packets */
 
+	unsigned int retry_enabled;
+
 	/* "read-write" results */
 	unsigned int rx_packets;  /**< received packets */
 	unsigned int tx_packets;  /**< received packets transmitted */
@@ -220,9 +222,14 @@ struct fwd_engine {
 	packet_fwd_t     packet_fwd;     /**< Mandatory. */
 };
 
+#define BURST_TX_WAIT_US 1
+#define BURST_TX_RETRIES 64
+
+extern uint32_t burst_tx_delay_time;
+extern uint32_t burst_tx_retry_num;
+
 extern struct fwd_engine io_fwd_engine;
 extern struct fwd_engine mac_fwd_engine;
-extern struct fwd_engine mac_retry_fwd_engine;
 extern struct fwd_engine mac_swap_engine;
 extern struct fwd_engine flow_gen_engine;
 extern struct fwd_engine rx_only_engine;
@@ -380,6 +387,7 @@ extern int8_t tx_wthresh;
 
 extern struct fwd_config cur_fwd_config;
 extern struct fwd_engine *cur_fwd_eng;
+extern uint32_t retry_enabled;
 extern struct fwd_lcore  **fwd_lcores;
 extern struct fwd_stream **fwd_streams;
 
@@ -472,7 +480,7 @@ void port_infos_display(portid_t port_id);
 void rx_queue_infos_display(portid_t port_idi, uint16_t queue_id);
 void tx_queue_infos_display(portid_t port_idi, uint16_t queue_id);
 void fwd_lcores_config_display(void);
-void fwd_config_display(void);
+void pkt_fwd_config_display(struct fwd_config *cfg);
 void rxtx_config_display(void);
 void fwd_config_setup(void);
 void set_def_fwd_config(void);
@@ -500,6 +508,7 @@ void set_fwd_lcores_number(uint16_t nb_lc);
 void set_fwd_ports_list(unsigned int *portlist, unsigned int nb_pt);
 void set_fwd_ports_mask(uint64_t portmask);
 void set_fwd_ports_number(uint16_t nb_pt);
+int port_is_forwarding(portid_t port_id);
 
 void rx_vlan_strip_set(portid_t port_id, int on);
 void rx_vlan_strip_set_on_queue(portid_t port_id, uint16_t queue_id, int on);
@@ -523,6 +532,7 @@ void show_tx_pkt_segments(void);
 void set_tx_pkt_split(const char *name);
 void set_nb_pkt_per_burst(uint16_t pkt_burst);
 char *list_pkt_forwarding_modes(void);
+char *list_pkt_forwarding_retry_modes(void);
 void set_pkt_forwarding_mode(const char *fwd_mode);
 void start_packet_forwarding(int with_tx_first);
 void stop_packet_forwarding(void);
@@ -531,6 +541,8 @@ void dev_set_link_down(portid_t pid);
 void init_port_config(void);
 void set_port_slave_flag(portid_t slave_pid);
 void clear_port_slave_flag(portid_t slave_pid);
+uint8_t port_is_bonding_slave(portid_t slave_pid);
+
 int init_port_dcb_config(portid_t pid, enum dcb_mode_enable dcb_mode,
 		     enum rte_eth_nb_tcs num_tcs,
 		     uint8_t pfc_en);
diff --git a/app/test-pmd/txonly.c b/app/test-pmd/txonly.c
index b37cae57..11fd681d 100644
--- a/app/test-pmd/txonly.c
+++ b/app/test-pmd/txonly.c
@@ -86,16 +86,6 @@
 static struct ipv4_hdr  pkt_ip_hdr;  /**< IP header of transmitted packets. */
 static struct udp_hdr pkt_udp_hdr; /**< UDP header of transmitted packets. */
 
-static inline struct rte_mbuf *
-tx_mbuf_alloc(struct rte_mempool *mp)
-{
-	struct rte_mbuf *m;
-
-	m = __rte_mbuf_raw_alloc(mp);
-	__rte_mbuf_sanity_check_raw(m, 0);
-	return m;
-}
-
 static void
 copy_buf_to_pkt_segs(void* buf, unsigned len, struct rte_mbuf *pkt,
 		     unsigned offset)
@@ -203,6 +193,7 @@ pkt_burst_transmit(struct fwd_stream *fs)
 	uint16_t nb_tx;
 	uint16_t nb_pkt;
 	uint16_t vlan_tci, vlan_tci_outer;
+	uint32_t retry;
 	uint64_t ol_flags = 0;
 	uint8_t  i;
 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES
@@ -225,7 +216,7 @@ pkt_burst_transmit(struct fwd_stream *fs)
 	if (txp->tx_ol_flags & TESTPMD_TX_OFFLOAD_INSERT_QINQ)
 		ol_flags |= PKT_TX_QINQ_PKT;
 	for (nb_pkt = 0; nb_pkt < nb_pkt_per_burst; nb_pkt++) {
-		pkt = tx_mbuf_alloc(mbp);
+		pkt = rte_mbuf_raw_alloc(mbp);
 		if (pkt == NULL) {
 		nomore_mbuf:
 			if (nb_pkt == 0)
@@ -240,7 +231,7 @@ pkt_burst_transmit(struct fwd_stream *fs)
 			nb_segs = tx_pkt_nb_segs;
 		pkt_len = pkt->data_len;
 		for (i = 1; i < nb_segs; i++) {
-			pkt_seg->next = tx_mbuf_alloc(mbp);
+			pkt_seg->next = rte_mbuf_raw_alloc(mbp);
 			if (pkt_seg->next == NULL) {
 				pkt->nb_segs = i;
 				rte_pktmbuf_free(pkt);
@@ -283,6 +274,17 @@ pkt_burst_transmit(struct fwd_stream *fs)
 		pkts_burst[nb_pkt] = pkt;
 	}
 	nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue, pkts_burst, nb_pkt);
+	/*
+	 * Retry if necessary
+	 */
+	if (unlikely(nb_tx < nb_pkt) && fs->retry_enabled) {
+		retry = 0;
+		while (nb_tx < nb_pkt && retry++ < burst_tx_retry_num) {
+			rte_delay_us(burst_tx_delay_time);
+			nb_tx += rte_eth_tx_burst(fs->tx_port, fs->tx_queue,
+					&pkts_burst[nb_tx], nb_pkt - nb_tx);
+		}
+	}
 	fs->tx_packets += nb_tx;
 
 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS