Imported Upstream version 17.05

Change-Id: Id1e419c5a214e4a18739663b91f0f9a549f1fdc6
Signed-off-by: Christian Ehrhardt <christian.ehrhardt@canonical.com>

4 files changed, 329 insertions, 301 deletions

diff --git a/examples/quota_watermark/qw/args.c b/examples/quota_watermark/qw/args.c
index 408b54d1..6ba77bc0 100644
--- a/examples/quota_watermark/qw/args.c
+++ b/examples/quota_watermark/qw/args.c
@@ -47,9 +47,9 @@ unsigned int portmask = 0;
 static void
 usage(const char *prgname)
 {
-    fprintf(stderr, "Usage: %s [EAL args] -- -p <portmask>\n"
-                    "-p PORTMASK: hexadecimal bitmask of NIC ports to configure\n",
-            prgname);
+	fprintf(stderr, "Usage: %s [EAL args] -- -p <portmask>\n"
+			"-p PORTMASK: hexadecimal bitmask of NIC ports to configure\n",
+			prgname);
 }
 
 static unsigned long
@@ -61,44 +61,47 @@ parse_portmask(const char *portmask_str)
 static void
 check_core_count(void)
 {
-    if (rte_lcore_count() < 3)
-        rte_exit(EXIT_FAILURE, "At least 3 cores need to be passed in the coremask\n");
+	if (rte_lcore_count() < 3)
+		rte_exit(EXIT_FAILURE,
+				"At least 3 cores need to be passed in the coremask\n");
 }
 
 static void
 check_portmask_value(unsigned int portmask)
 {
-    unsigned int port_nb = 0;
+	unsigned int port_nb = 0;
 
-    port_nb = __builtin_popcount(portmask);
+	port_nb = __builtin_popcount(portmask);
 
-    if (port_nb == 0)
-        rte_exit(EXIT_FAILURE, "At least 2 ports need to be passed in the portmask\n");
+	if (port_nb == 0)
+		rte_exit(EXIT_FAILURE,
+				"At least 2 ports need to be passed in the portmask\n");
 
-    if (port_nb % 2 != 0)
-        rte_exit(EXIT_FAILURE, "An even number of ports is required in the portmask\n");
+	if (port_nb % 2 != 0)
+		rte_exit(EXIT_FAILURE,
+				"An even number of ports is required in the portmask\n");
 }
 
 int
 parse_qw_args(int argc, char **argv)
 {
-    int opt;
-
-    while ((opt = getopt(argc, argv, "h:p:")) != -1) {
-        switch (opt) {
-        case 'h':
-                usage(argv[0]);
-                break;
-        case 'p':
-                portmask = parse_portmask(optarg);
-                break;
-        default:
-                usage(argv[0]);
-        }
-    }
-
-    check_core_count();
-    check_portmask_value(portmask);
-
-    return 0;
+	int opt;
+
+	while ((opt = getopt(argc, argv, "h:p:")) != -1) {
+		switch (opt) {
+		case 'h':
+			usage(argv[0]);
+			break;
+		case 'p':
+			portmask = parse_portmask(optarg);
+			break;
+		default:
+			usage(argv[0]);
+		}
+	}
+
+	check_core_count();
+	check_portmask_value(portmask);
+
+	return 0;
 }
diff --git a/examples/quota_watermark/qw/init.c b/examples/quota_watermark/qw/init.c
index c2087218..b6264fcf 100644
--- a/examples/quota_watermark/qw/init.c
+++ b/examples/quota_watermark/qw/init.c
@@ -51,124 +51,128 @@
 
 
 static const struct rte_eth_conf port_conf = {
-	.rxmode = {
-		.split_hdr_size = 0,
-		.header_split   = 0, /**< Header Split disabled */
-		.hw_ip_checksum = 0, /**< IP checksum offload disabled */
-		.hw_vlan_filter = 0, /**< VLAN filtering disabled */
-		.jumbo_frame    = 0, /**< Jumbo Frame Support disabled */
-		.hw_strip_crc   = 0, /**< CRC stripped by hardware */
-	},
-	.txmode = {
-		.mq_mode = ETH_DCB_NONE,
-	},
+		.rxmode = {
+			.split_hdr_size = 0,
+			.header_split   = 0, /**< Header Split disabled */
+			.hw_ip_checksum = 0, /**< IP csum offload disabled */
+			.hw_vlan_filter = 0, /**< VLAN filtering disabled */
+			.jumbo_frame    = 0, /**< Jumbo Frame disabled */
+			.hw_strip_crc   = 1, /**< CRC stripped by hardware */
+		},
+		.txmode = {
+			.mq_mode = ETH_DCB_NONE,
+		},
 };
 
 static struct rte_eth_fc_conf fc_conf = {
-    .mode       = RTE_FC_TX_PAUSE,
-    .high_water = 80 * 510 / 100,
-    .low_water  = 60 * 510 / 100,
-    .pause_time = 1337,
-    .send_xon   = 0,
+		.mode       = RTE_FC_TX_PAUSE,
+		.high_water = 80 * 510 / 100,
+		.low_water  = 60 * 510 / 100,
+		.pause_time = 1337,
+		.send_xon   = 0,
 };
 
 
 void configure_eth_port(uint8_t port_id)
 {
-    int ret;
-
-    rte_eth_dev_stop(port_id);
-
-    ret = rte_eth_dev_configure(port_id, 1, 1, &port_conf);
-    if (ret < 0)
-        rte_exit(EXIT_FAILURE, "Cannot configure port %u (error %d)\n",
-                               (unsigned) port_id, ret);
-
-    /* Initialize the port's RX queue */
-    ret = rte_eth_rx_queue_setup(port_id, 0, RX_DESC_PER_QUEUE,
-				rte_eth_dev_socket_id(port_id),
-				NULL,
-				mbuf_pool);
-    if (ret < 0)
-        rte_exit(EXIT_FAILURE, "Failed to setup RX queue on "
-                               "port %u (error %d)\n", (unsigned) port_id, ret);
-
-    /* Initialize the port's TX queue */
-    ret = rte_eth_tx_queue_setup(port_id, 0, TX_DESC_PER_QUEUE,
-				rte_eth_dev_socket_id(port_id),
-				NULL);
-    if (ret < 0)
-        rte_exit(EXIT_FAILURE, "Failed to setup TX queue on "
-                               "port %u (error %d)\n", (unsigned) port_id, ret);
-
-    /* Initialize the port's flow control */
-    ret = rte_eth_dev_flow_ctrl_set(port_id, &fc_conf);
-    if (ret < 0)
-        rte_exit(EXIT_FAILURE, "Failed to setup hardware flow control on "
-                               "port %u (error %d)\n", (unsigned) port_id, ret);
-
-    /* Start the port */
-    ret = rte_eth_dev_start(port_id);
-    if (ret < 0)
-        rte_exit(EXIT_FAILURE, "Failed to start port %u (error %d)\n",
-                               (unsigned) port_id, ret);
-
-    /* Put it in promiscuous mode */
-    rte_eth_promiscuous_enable(port_id);
+	int ret;
+
+	rte_eth_dev_stop(port_id);
+
+	ret = rte_eth_dev_configure(port_id, 1, 1, &port_conf);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE, "Cannot configure port %u (error %d)\n",
+				(unsigned int) port_id, ret);
+
+	/* Initialize the port's RX queue */
+	ret = rte_eth_rx_queue_setup(port_id, 0, RX_DESC_PER_QUEUE,
+			rte_eth_dev_socket_id(port_id),
+			NULL,
+			mbuf_pool);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE,
+				"Failed to setup RX queue on port %u (error %d)\n",
+				(unsigned int) port_id, ret);
+
+	/* Initialize the port's TX queue */
+	ret = rte_eth_tx_queue_setup(port_id, 0, TX_DESC_PER_QUEUE,
+			rte_eth_dev_socket_id(port_id),
+			NULL);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE,
+				"Failed to setup TX queue on port %u (error %d)\n",
+				(unsigned int) port_id, ret);
+
+	/* Initialize the port's flow control */
+	ret = rte_eth_dev_flow_ctrl_set(port_id, &fc_conf);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE,
+				"Failed to setup hardware flow control on port %u (error %d)\n",
+				(unsigned int) port_id, ret);
+
+	/* Start the port */
+	ret = rte_eth_dev_start(port_id);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE, "Failed to start port %u (error %d)\n",
+				(unsigned int) port_id, ret);
+
+	/* Put it in promiscuous mode */
+	rte_eth_promiscuous_enable(port_id);
 }
 
 void
 init_dpdk(void)
 {
-    if (rte_eth_dev_count() < 2)
-        rte_exit(EXIT_FAILURE, "Not enough ethernet port available\n");
+	if (rte_eth_dev_count() < 2)
+		rte_exit(EXIT_FAILURE, "Not enough ethernet port available\n");
 }
 
 void init_ring(int lcore_id, uint8_t port_id)
 {
-    struct rte_ring *ring;
-    char ring_name[RTE_RING_NAMESIZE];
+	struct rte_ring *ring;
+	char ring_name[RTE_RING_NAMESIZE];
 
-    snprintf(ring_name, RTE_RING_NAMESIZE,
-		"core%d_port%d", lcore_id, port_id);
-    ring = rte_ring_create(ring_name, RING_SIZE, rte_socket_id(),
-                           RING_F_SP_ENQ | RING_F_SC_DEQ);
+	snprintf(ring_name, RTE_RING_NAMESIZE,
+			"core%d_port%d", lcore_id, port_id);
+	ring = rte_ring_create(ring_name, RING_SIZE, rte_socket_id(),
+			RING_F_SP_ENQ | RING_F_SC_DEQ);
 
-    if (ring == NULL)
-        rte_exit(EXIT_FAILURE, "%s\n", rte_strerror(rte_errno));
+	if (ring == NULL)
+		rte_exit(EXIT_FAILURE, "%s\n", rte_strerror(rte_errno));
 
-    rte_ring_set_water_mark(ring, 80 * RING_SIZE / 100);
+	*high_watermark = 80 * RING_SIZE / 100;
 
-    rings[lcore_id][port_id] = ring;
+	rings[lcore_id][port_id] = ring;
 }
 
 void
 pair_ports(void)
 {
-    uint8_t i, j;
-
-    /* Pair ports with their "closest neighbour" in the portmask */
-    for (i = 0; i < RTE_MAX_ETHPORTS; i++)
-        if (is_bit_set(i, portmask))
-            for (j = (uint8_t) (i + 1); j < RTE_MAX_ETHPORTS; j++)
-                if (is_bit_set(j, portmask)) {
-                    port_pairs[i] = j;
-                    port_pairs[j] = i;
-                    i = j;
-                    break;
-                }
+	uint8_t i, j;
+
+	/* Pair ports with their "closest neighbour" in the portmask */
+	for (i = 0; i < RTE_MAX_ETHPORTS; i++)
+		if (is_bit_set(i, portmask))
+			for (j = (uint8_t) (i + 1); j < RTE_MAX_ETHPORTS; j++)
+				if (is_bit_set(j, portmask)) {
+					port_pairs[i] = j;
+					port_pairs[j] = i;
+					i = j;
+					break;
+				}
 }
 
 void
 setup_shared_variables(void)
 {
-    const struct rte_memzone *qw_memzone;
+	const struct rte_memzone *qw_memzone;
 
-    qw_memzone = rte_memzone_reserve(QUOTA_WATERMARK_MEMZONE_NAME, 2 * sizeof(int),
-                                     rte_socket_id(), RTE_MEMZONE_2MB);
-    if (qw_memzone == NULL)
-        rte_exit(EXIT_FAILURE, "%s\n", rte_strerror(rte_errno));
+	qw_memzone = rte_memzone_reserve(QUOTA_WATERMARK_MEMZONE_NAME,
+			3 * sizeof(int), rte_socket_id(), 0);
+	if (qw_memzone == NULL)
+		rte_exit(EXIT_FAILURE, "%s\n", rte_strerror(rte_errno));
 
-    quota = qw_memzone->addr;
-    low_watermark = (unsigned int *) qw_memzone->addr + 1;
+	quota = qw_memzone->addr;
+	low_watermark = (unsigned int *) qw_memzone->addr + 1;
+	high_watermark = (unsigned int *) qw_memzone->addr + 2;
 }
diff --git a/examples/quota_watermark/qw/main.c b/examples/quota_watermark/qw/main.c
index 8ed02148..d4fcfde4 100644
--- a/examples/quota_watermark/qw/main.c
+++ b/examples/quota_watermark/qw/main.c
@@ -60,13 +60,14 @@
 #define ETHER_TYPE_FLOW_CONTROL 0x8808
 
 struct ether_fc_frame {
-    uint16_t opcode;
-    uint16_t param;
+	uint16_t opcode;
+	uint16_t param;
 } __attribute__((__packed__));
 
 
 int *quota;
 unsigned int *low_watermark;
+unsigned int *high_watermark;
 
 uint8_t port_pairs[RTE_MAX_ETHPORTS];
 
@@ -76,38 +77,39 @@ struct rte_mempool *mbuf_pool;
 
 static void send_pause_frame(uint8_t port_id, uint16_t duration)
 {
-    struct rte_mbuf *mbuf;
-    struct ether_fc_frame *pause_frame;
-    struct ether_hdr *hdr;
-    struct ether_addr mac_addr;
+	struct rte_mbuf *mbuf;
+	struct ether_fc_frame *pause_frame;
+	struct ether_hdr *hdr;
+	struct ether_addr mac_addr;
 
-    RTE_LOG(DEBUG, USER1, "Sending PAUSE frame (duration=%d) on port %d\n",
-            duration, port_id);
+	RTE_LOG_DP(DEBUG, USER1,
+			"Sending PAUSE frame (duration=%d) on port %d\n",
+			duration, port_id);
 
-    /* Get a mbuf from the pool */
-    mbuf = rte_pktmbuf_alloc(mbuf_pool);
-    if (unlikely(mbuf == NULL))
-        return;
+	/* Get a mbuf from the pool */
+	mbuf = rte_pktmbuf_alloc(mbuf_pool);
+	if (unlikely(mbuf == NULL))
+		return;
 
-    /* Prepare a PAUSE frame */
-    hdr = rte_pktmbuf_mtod(mbuf, struct ether_hdr *);
-    pause_frame = (struct ether_fc_frame *) &hdr[1];
+	/* Prepare a PAUSE frame */
+	hdr = rte_pktmbuf_mtod(mbuf, struct ether_hdr *);
+	pause_frame = (struct ether_fc_frame *) &hdr[1];
 
-    rte_eth_macaddr_get(port_id, &mac_addr);
-    ether_addr_copy(&mac_addr, &hdr->s_addr);
+	rte_eth_macaddr_get(port_id, &mac_addr);
+	ether_addr_copy(&mac_addr, &hdr->s_addr);
 
-    void *tmp = &hdr->d_addr.addr_bytes[0];
-    *((uint64_t *)tmp) = 0x010000C28001ULL;
+	void *tmp = &hdr->d_addr.addr_bytes[0];
+	*((uint64_t *)tmp) = 0x010000C28001ULL;
 
-    hdr->ether_type = rte_cpu_to_be_16(ETHER_TYPE_FLOW_CONTROL);
+	hdr->ether_type = rte_cpu_to_be_16(ETHER_TYPE_FLOW_CONTROL);
 
-    pause_frame->opcode = rte_cpu_to_be_16(0x0001);
-    pause_frame->param  = rte_cpu_to_be_16(duration);
+	pause_frame->opcode = rte_cpu_to_be_16(0x0001);
+	pause_frame->param  = rte_cpu_to_be_16(duration);
 
-    mbuf->pkt_len  = 60;
-    mbuf->data_len = 60;
+	mbuf->pkt_len  = 60;
+	mbuf->data_len = 60;
 
-    rte_eth_tx_burst(port_id, 0, &mbuf, 1);
+	rte_eth_tx_burst(port_id, 0, &mbuf, 1);
 }
 
 /**
@@ -121,13 +123,13 @@ static void send_pause_frame(uint8_t port_id, uint16_t duration)
 static unsigned int
 get_previous_lcore_id(unsigned int lcore_id)
 {
-    int i;
+	int i;
 
-    for (i = lcore_id - 1; i >= 0; i--)
-        if (rte_lcore_is_enabled(i))
-            return i;
+	for (i = lcore_id - 1; i >= 0; i--)
+		if (rte_lcore_is_enabled(i))
+			return i;
 
-    return -1;
+	return -1;
 }
 
 /**
@@ -139,125 +141,137 @@ get_previous_lcore_id(unsigned int lcore_id)
 static unsigned int
 get_last_lcore_id(void)
 {
-    int i;
+	int i;
 
-    for (i = RTE_MAX_LCORE; i >= 0; i--)
-        if (rte_lcore_is_enabled(i))
-            return i;
+	for (i = RTE_MAX_LCORE; i >= 0; i--)
+		if (rte_lcore_is_enabled(i))
+			return i;
 
-    return 0;
+	return 0;
 }
 
 static void
 receive_stage(__attribute__((unused)) void *args)
 {
-    int i, ret;
+	int i, ret;
 
-    uint8_t port_id;
-    uint16_t nb_rx_pkts;
+	uint8_t port_id;
+	uint16_t nb_rx_pkts;
 
-    unsigned int lcore_id;
+	unsigned int lcore_id;
+	unsigned int free;
 
-    struct rte_mbuf *pkts[MAX_PKT_QUOTA];
-    struct rte_ring *ring;
-    enum ring_state ring_state[RTE_MAX_ETHPORTS] = { RING_READY };
+	struct rte_mbuf *pkts[MAX_PKT_QUOTA];
+	struct rte_ring *ring;
+	enum ring_state ring_state[RTE_MAX_ETHPORTS] = { RING_READY };
 
-    lcore_id = rte_lcore_id();
+	lcore_id = rte_lcore_id();
 
-    RTE_LOG(INFO, USER1,
-            "%s() started on core %u\n", __func__, lcore_id);
+	RTE_LOG(INFO, USER1,
+			"%s() started on core %u\n", __func__, lcore_id);
 
-    while (1) {
+	while (1) {
 
-        /* Process each port round robin style */
-        for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) {
+		/* Process each port round robin style */
+		for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) {
 
-            if (!is_bit_set(port_id, portmask))
-                continue;
+			if (!is_bit_set(port_id, portmask))
+				continue;
 
-            ring = rings[lcore_id][port_id];
+			ring = rings[lcore_id][port_id];
 
-            if (ring_state[port_id] != RING_READY) {
-                if (rte_ring_count(ring) > *low_watermark)
-                    continue;
-                else
-                    ring_state[port_id] = RING_READY;
-            }
+			if (ring_state[port_id] != RING_READY) {
+				if (rte_ring_count(ring) > *low_watermark)
+					continue;
+				else
+					ring_state[port_id] = RING_READY;
+			}
 
-            /* Enqueue received packets on the RX ring */
-            nb_rx_pkts = rte_eth_rx_burst(port_id, 0, pkts, (uint16_t) *quota);
-            ret = rte_ring_enqueue_bulk(ring, (void *) pkts, nb_rx_pkts);
-            if (ret == -EDQUOT) {
-                ring_state[port_id] = RING_OVERLOADED;
-                send_pause_frame(port_id, 1337);
-            }
+			/* Enqueue received packets on the RX ring */
+			nb_rx_pkts = rte_eth_rx_burst(port_id, 0, pkts,
+					(uint16_t) *quota);
+			ret = rte_ring_enqueue_bulk(ring, (void *) pkts,
+					nb_rx_pkts, &free);
+			if (RING_SIZE - free > *high_watermark) {
+				ring_state[port_id] = RING_OVERLOADED;
+				send_pause_frame(port_id, 1337);
+			}
 
-            else if (ret == -ENOBUFS) {
+			if (ret == 0) {
 
-                /* Return  mbufs to the pool, effectively dropping packets */
-                for (i = 0; i < nb_rx_pkts; i++)
-                    rte_pktmbuf_free(pkts[i]);
-            }
-        }
-    }
+				/*
+				 * Return  mbufs to the pool,
+				 * effectively dropping packets
+				 */
+				for (i = 0; i < nb_rx_pkts; i++)
+					rte_pktmbuf_free(pkts[i]);
+			}
+		}
+	}
 }
 
 static void
 pipeline_stage(__attribute__((unused)) void *args)
 {
-    int i, ret;
-    int nb_dq_pkts;
+	int i, ret;
+	int nb_dq_pkts;
 
-    uint8_t port_id;
+	uint8_t port_id;
 
-    unsigned int lcore_id, previous_lcore_id;
+	unsigned int lcore_id, previous_lcore_id;
+	unsigned int free;
 
-    void *pkts[MAX_PKT_QUOTA];
-    struct rte_ring *rx, *tx;
-    enum ring_state ring_state[RTE_MAX_ETHPORTS] = { RING_READY };
+	void *pkts[MAX_PKT_QUOTA];
+	struct rte_ring *rx, *tx;
+	enum ring_state ring_state[RTE_MAX_ETHPORTS] = { RING_READY };
 
-    lcore_id = rte_lcore_id();
-    previous_lcore_id = get_previous_lcore_id(lcore_id);
+	lcore_id = rte_lcore_id();
+	previous_lcore_id = get_previous_lcore_id(lcore_id);
 
-    RTE_LOG(INFO, USER1,
-            "%s() started on core %u - processing packets from core %u\n",
-            __func__, lcore_id, previous_lcore_id);
+	RTE_LOG(INFO, USER1,
+			"%s() started on core %u - processing packets from core %u\n",
+			__func__, lcore_id, previous_lcore_id);
 
-    while (1) {
+	while (1) {
 
-        for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) {
+		for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) {
 
-            if (!is_bit_set(port_id, portmask))
-                continue;
+			if (!is_bit_set(port_id, portmask))
+				continue;
 
-            tx = rings[lcore_id][port_id];
-            rx = rings[previous_lcore_id][port_id];
+			tx = rings[lcore_id][port_id];
+			rx = rings[previous_lcore_id][port_id];
 
-            if (ring_state[port_id] != RING_READY) {
-                if (rte_ring_count(tx) > *low_watermark)
-                    continue;
-                else
-                    ring_state[port_id] = RING_READY;
-            }
+			if (ring_state[port_id] != RING_READY) {
+				if (rte_ring_count(tx) > *low_watermark)
+					continue;
+				else
+					ring_state[port_id] = RING_READY;
+			}
 
-            /* Dequeue up to quota mbuf from rx */
-            nb_dq_pkts = rte_ring_dequeue_burst(rx, pkts, *quota);
-            if (unlikely(nb_dq_pkts < 0))
-                continue;
+			/* Dequeue up to quota mbuf from rx */
+			nb_dq_pkts = rte_ring_dequeue_burst(rx, pkts,
+					*quota, NULL);
+			if (unlikely(nb_dq_pkts < 0))
+				continue;
 
-            /* Enqueue them on tx */
-            ret = rte_ring_enqueue_bulk(tx, pkts, nb_dq_pkts);
-            if (ret == -EDQUOT)
-                ring_state[port_id] = RING_OVERLOADED;
+			/* Enqueue them on tx */
+			ret = rte_ring_enqueue_bulk(tx, pkts,
+					nb_dq_pkts, &free);
+			if (RING_SIZE - free > *high_watermark)
+				ring_state[port_id] = RING_OVERLOADED;
 
-            else if (ret == -ENOBUFS) {
+			if (ret == 0) {
 
-                /* Return  mbufs to the pool, effectively dropping packets */
-                for (i = 0; i < nb_dq_pkts; i++)
-                    rte_pktmbuf_free(pkts[i]);
-            }
-        }
-    }
+				/*
+				 * Return  mbufs to the pool,
+				 * effectively dropping packets
+				 */
+				for (i = 0; i < nb_dq_pkts; i++)
+					rte_pktmbuf_free(pkts[i]);
+			}
+		}
+	}
 }
 
 static void
@@ -265,108 +279,114 @@ send_stage(__attribute__((unused)) void *args)
 {
 	uint16_t nb_dq_pkts;
 
-    uint8_t port_id;
-    uint8_t dest_port_id;
+	uint8_t port_id;
+	uint8_t dest_port_id;
 
-    unsigned int lcore_id, previous_lcore_id;
+	unsigned int lcore_id, previous_lcore_id;
 
-    struct rte_ring *tx;
-    struct rte_mbuf *tx_pkts[MAX_PKT_QUOTA];
+	struct rte_ring *tx;
+	struct rte_mbuf *tx_pkts[MAX_PKT_QUOTA];
 
-    lcore_id = rte_lcore_id();
-    previous_lcore_id = get_previous_lcore_id(lcore_id);
+	lcore_id = rte_lcore_id();
+	previous_lcore_id = get_previous_lcore_id(lcore_id);
 
-    RTE_LOG(INFO, USER1,
-            "%s() started on core %u - processing packets from core %u\n",
-            __func__, lcore_id, previous_lcore_id);
+	RTE_LOG(INFO, USER1,
+			"%s() started on core %u - processing packets from core %u\n",
+			__func__, lcore_id, previous_lcore_id);
 
-    while (1) {
+	while (1) {
 
-        /* Process each ring round robin style */
-        for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) {
+		/* Process each ring round robin style */
+		for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) {
 
-            if (!is_bit_set(port_id, portmask))
-                continue;
+			if (!is_bit_set(port_id, portmask))
+				continue;
 
-            dest_port_id = port_pairs[port_id];
-            tx = rings[previous_lcore_id][port_id];
+			dest_port_id = port_pairs[port_id];
+			tx = rings[previous_lcore_id][port_id];
 
-            if (rte_ring_empty(tx))
-                continue;
+			if (rte_ring_empty(tx))
+				continue;
 
-            /* Dequeue packets from tx and send them */
-            nb_dq_pkts = (uint16_t) rte_ring_dequeue_burst(tx, (void *) tx_pkts, *quota);
-            rte_eth_tx_burst(dest_port_id, 0, tx_pkts, nb_dq_pkts);
+			/* Dequeue packets from tx and send them */
+			nb_dq_pkts = (uint16_t) rte_ring_dequeue_burst(tx,
+					(void *) tx_pkts, *quota, NULL);
+			rte_eth_tx_burst(dest_port_id, 0, tx_pkts, nb_dq_pkts);
 
-            /* TODO: Check if nb_dq_pkts == nb_tx_pkts? */
-        }
-    }
+			/* TODO: Check if nb_dq_pkts == nb_tx_pkts? */
+		}
+	}
 }
 
 int
 main(int argc, char **argv)
 {
-    int ret;
-    unsigned int lcore_id, master_lcore_id, last_lcore_id;
+	int ret;
+	unsigned int lcore_id, master_lcore_id, last_lcore_id;
 
-    uint8_t port_id;
+	uint8_t port_id;
 
-    rte_set_log_level(RTE_LOG_INFO);
+	rte_log_set_global_level(RTE_LOG_INFO);
 
-    ret = rte_eal_init(argc, argv);
-    if (ret < 0)
-        rte_exit(EXIT_FAILURE, "Cannot initialize EAL\n");
+	ret = rte_eal_init(argc, argv);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE, "Cannot initialize EAL\n");
 
-    argc -= ret;
-    argv += ret;
+	argc -= ret;
+	argv += ret;
 
-    init_dpdk();
-    setup_shared_variables();
+	init_dpdk();
+	setup_shared_variables();
 
-    *quota = 32;
-    *low_watermark = 60 * RING_SIZE / 100;
+	*quota = 32;
+	*low_watermark = 60 * RING_SIZE / 100;
 
-    last_lcore_id   = get_last_lcore_id();
-    master_lcore_id = rte_get_master_lcore();
+	last_lcore_id   = get_last_lcore_id();
+	master_lcore_id = rte_get_master_lcore();
 
-    /* Parse the application's arguments */
-    ret = parse_qw_args(argc, argv);
-    if (ret < 0)
-        rte_exit(EXIT_FAILURE, "Invalid quota/watermark argument(s)\n");
+	/* Parse the application's arguments */
+	ret = parse_qw_args(argc, argv);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE, "Invalid quota/watermark argument(s)\n");
 
-    /* Create a pool of mbuf to store packets */
-    mbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", MBUF_PER_POOL, 32, 0,
-	    MBUF_DATA_SIZE, rte_socket_id());
-    if (mbuf_pool == NULL)
-        rte_panic("%s\n", rte_strerror(rte_errno));
+	/* Create a pool of mbuf to store packets */
+	mbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", MBUF_PER_POOL, 32, 0,
+			MBUF_DATA_SIZE, rte_socket_id());
+	if (mbuf_pool == NULL)
+		rte_panic("%s\n", rte_strerror(rte_errno));
 
-    for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++)
-        if (is_bit_set(port_id, portmask)) {
-            configure_eth_port(port_id);
-            init_ring(master_lcore_id, port_id);
-        }
+	for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++)
+		if (is_bit_set(port_id, portmask)) {
+			configure_eth_port(port_id);
+			init_ring(master_lcore_id, port_id);
+		}
 
-    pair_ports();
+	pair_ports();
 
-    /* Start pipeline_connect() on all the available slave lcore but the last */
-    for (lcore_id = 0 ; lcore_id < last_lcore_id; lcore_id++) {
-        if (rte_lcore_is_enabled(lcore_id) && lcore_id != master_lcore_id) {
+	/*
+	 * Start pipeline_connect() on all the available slave lcores
+	 * but the last
+	 */
+	for (lcore_id = 0 ; lcore_id < last_lcore_id; lcore_id++) {
+		if (rte_lcore_is_enabled(lcore_id) &&
+				lcore_id != master_lcore_id) {
 
-            for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++)
-                if (is_bit_set(port_id, portmask))
-                    init_ring(lcore_id, port_id);
+			for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++)
+				if (is_bit_set(port_id, portmask))
+					init_ring(lcore_id, port_id);
 
-            /* typecast is a workaround for GCC 4.3 bug */
-            rte_eal_remote_launch((int (*)(void *))pipeline_stage, NULL, lcore_id);
-        }
-    }
+			/* typecast is a workaround for GCC 4.3 bug */
+			rte_eal_remote_launch((int (*)(void *))pipeline_stage,
+					NULL, lcore_id);
+		}
+	}
 
-    /* Start send_stage() on the last slave core */
-    /* typecast is a workaround for GCC 4.3 bug */
-    rte_eal_remote_launch((int (*)(void *))send_stage, NULL, last_lcore_id);
+	/* Start send_stage() on the last slave core */
+	/* typecast is a workaround for GCC 4.3 bug */
+	rte_eal_remote_launch((int (*)(void *))send_stage, NULL, last_lcore_id);
 
-    /* Start receive_stage() on the master core */
-    receive_stage(NULL);
+	/* Start receive_stage() on the master core */
+	receive_stage(NULL);
 
-    return 0;
+	return 0;
 }
diff --git a/examples/quota_watermark/qw/main.h b/examples/quota_watermark/qw/main.h
index 6b364898..8c8e3116 100644
--- a/examples/quota_watermark/qw/main.h
+++ b/examples/quota_watermark/qw/main.h
@@ -37,12 +37,13 @@
 #include "../include/conf.h"
 
 enum ring_state {
-    RING_READY,
-    RING_OVERLOADED,
+	RING_READY,
+	RING_OVERLOADED,
 };
 
 extern int *quota;
 extern unsigned int *low_watermark;
+extern unsigned int *high_watermark;
 
 extern uint8_t port_pairs[RTE_MAX_ETHPORTS];
 
@@ -53,7 +54,7 @@ extern struct rte_mempool *mbuf_pool;
 static inline int
 is_bit_set(int i, unsigned int mask)
 {
-    return (1 << i) & mask;
+	return (1 << i) & mask;
 }
 
 #endif /* _MAIN_H_ */