libmemif: zero-copy-slave mode + header space

Slave is now able to dequeue buffers from rx queue and enqueue them to tx queue
(zero-copy operation). Slave can produce buffers with headroom, which will allow adding
 encap without copy.

Change-Id: Ia189f8de1a68be787545ed46cf78d36403e7e9bf
Signed-off-by: Jakub Grajciar <jgrajcia@cisco.com>

10 files changed, 1696 insertions, 72 deletions

diff --git a/extras/libmemif/examples/icmp_responder-epoll/main.c b/extras/libmemif/examples/icmp_responder-epoll/main.c
index 1dab51f4f23..fd354a38aaf 100644
--- a/extras/libmemif/examples/icmp_responder-epoll/main.c
+++ b/extras/libmemif/examples/icmp_responder-epoll/main.c
@@ -282,6 +282,7 @@ int
 on_connect (memif_conn_handle_t conn, void *private_ctx)
 {
   INFO ("memif connected!");
+  memif_refill_queue (conn, 0, -1, 0);
   enable_log = 1;
   return 0;
 }
@@ -384,6 +385,7 @@ on_interrupt (memif_conn_handle_t conn, void *private_ctx, uint16_t qid)
 	}
 
       i = 0;
+      memset (c->tx_bufs, 0, sizeof (memif_buffer_t) * rx);
       err = memif_buffer_alloc (c->conn, qid, c->tx_bufs, rx, &tx, 128);
       if ((err != MEMIF_ERR_SUCCESS) && (err != MEMIF_ERR_NOBUF_RING))
 	{
@@ -404,7 +406,7 @@ on_interrupt (memif_conn_handle_t conn, void *private_ctx, uint16_t qid)
 	  tx--;
 	}
 
-      err = memif_refill_queue (c->conn, qid, rx);
+      err = memif_refill_queue (c->conn, qid, rx, 0);
       if (err != MEMIF_ERR_SUCCESS)
 	INFO ("memif_buffer_free: %s", memif_strerror (err));
       rx -= rx;
@@ -426,7 +428,7 @@ on_interrupt (memif_conn_handle_t conn, void *private_ctx, uint16_t qid)
   return 0;
 
 error:
-  err = memif_refill_queue (c->conn, qid, rx);
+  err = memif_refill_queue (c->conn, qid, rx, 0);
   if (err != MEMIF_ERR_SUCCESS)
     INFO ("memif_buffer_free: %s", memif_strerror (err));
   c->rx_buf_num -= rx;
@@ -512,7 +514,7 @@ on_interrupt0 (memif_conn_handle_t conn, void *private_ctx, uint16_t qid)
 	    }
 	  /* mark memif buffers and shared memory buffers as free */
 	  /* free processed buffers */
-	  err = memif_refill_queue (c->conn, qid, j);
+	  err = memif_refill_queue (c->conn, qid, j, 0);
 	  if (err != MEMIF_ERR_SUCCESS)
 	    INFO ("memif_buffer_free: %s", memif_strerror (err));
 	  rx -= j;
@@ -538,7 +540,7 @@ on_interrupt0 (memif_conn_handle_t conn, void *private_ctx, uint16_t qid)
   return 0;
 
 error:
-  err = memif_refill_queue (c->conn, qid, rx);
+  err = memif_refill_queue (c->conn, qid, rx, 0);
   if (err != MEMIF_ERR_SUCCESS)
     INFO ("memif_buffer_free: %s", memif_strerror (err));
   c->rx_buf_num -= rx;
@@ -595,7 +597,7 @@ on_interrupt1 (memif_conn_handle_t conn, void *private_ctx, uint16_t qid)
 	    }
 	}
 
-      err = memif_refill_queue (c->conn, qid, rx);
+      err = memif_refill_queue (c->conn, qid, rx, 0);
       if (err != MEMIF_ERR_SUCCESS)
 	INFO ("memif_buffer_free: %s", memif_strerror (err));
       c->rx_buf_num -= rx;
@@ -607,7 +609,7 @@ on_interrupt1 (memif_conn_handle_t conn, void *private_ctx, uint16_t qid)
   return 0;
 
 error:
-  err = memif_refill_queue (c->conn, qid, rx);
+  err = memif_refill_queue (c->conn, qid, rx, 0);
   if (err != MEMIF_ERR_SUCCESS)
     INFO ("memif_buffer_free: %s", memif_strerror (err));
   c->rx_buf_num -= rx;
diff --git a/extras/libmemif/examples/icmp_responder-mt/main.c b/extras/libmemif/examples/icmp_responder-mt/main.c
index 4f7733b8072..c4ac907cb53 100644
--- a/extras/libmemif/examples/icmp_responder-mt/main.c
+++ b/extras/libmemif/examples/icmp_responder-mt/main.c
@@ -299,10 +299,6 @@ memif_rx_poll (void *ptr)
       data->rx_buf_num += rx;
       if (rx == 0)
 	{
-	  /* if queue is in polling mode, it's not refilled */
-	  err = memif_refill_queue (c->conn, data->qid, -1);
-	  if (err != MEMIF_ERR_SUCCESS)
-	    INFO ("memif_buffer_free: %s", memif_strerror (err));
 	  continue;
 	}
 
@@ -313,7 +309,7 @@ memif_rx_poll (void *ptr)
 
       err =
 	memif_buffer_alloc (c->conn, data->qid, data->tx_bufs,
-			    data->rx_buf_num, &tx, 128);
+			    data->rx_buf_num, &tx, 0);
       if (err != MEMIF_ERR_SUCCESS)
 	{
 	  INFO ("memif_buffer_alloc: %s", memif_strerror (err));
@@ -334,10 +330,10 @@ memif_rx_poll (void *ptr)
 	}
 
       /* mark memif buffers and shared memory buffers as free */
-      err = memif_refill_queue (c->conn, data->qid, -1);
+      err = memif_refill_queue (c->conn, data->qid, rx, 0);
       if (err != MEMIF_ERR_SUCCESS)
 	INFO ("memif_buffer_free: %s", memif_strerror (err));
-      data->rx_buf_num -= rx;
+      data->rx_buf_num -= fb;
 
       DBG ("freed %d buffers. %u/%u alloc/free buffers",
 	   fb, data->rx_buf_num, MAX_MEMIF_BUFS - data->rx_buf_num);
@@ -359,10 +355,10 @@ error:
   goto close;
 
 close:
-  err = memif_refill_queue (c->conn, data->qid, -1);
+  err = memif_refill_queue (c->conn, data->qid, rx, 0);
   if (err != MEMIF_ERR_SUCCESS)
     INFO ("memif_buffer_free: %s", memif_strerror (err));
-  data->rx_buf_num -= rx;
+  data->rx_buf_num -= fb;
   DBG ("freed %d buffers. %u/%u alloc/free buffers",
        fb, data->rx_buf_num, MAX_MEMIF_BUFS - data->rx_buf_num);
   free (data->rx_bufs);
@@ -443,7 +439,7 @@ memif_rx_interrupt (void *ptr)
 
 	  err =
 	    memif_buffer_alloc (c->conn, data->qid, data->tx_bufs,
-				data->rx_buf_num, &tx, 128);
+				data->rx_buf_num, &tx, 0);
 	  if (err != MEMIF_ERR_SUCCESS)
 	    {
 	      INFO ("memif_buffer_alloc: %s", memif_strerror (err));
@@ -464,10 +460,10 @@ memif_rx_interrupt (void *ptr)
 	    }
 
 	  /* mark memif buffers and shared memory buffers as free */
-	  err = memif_refill_queue (c->conn, data->qid, rx);
+	  err = memif_refill_queue (c->conn, data->qid, rx, 0);
 	  if (err != MEMIF_ERR_SUCCESS)
 	    INFO ("memif_buffer_free: %s", memif_strerror (err));
-	  data->rx_buf_num -= rx;
+	  data->rx_buf_num -= fb;
 
 	  DBG ("freed %d buffers. %u/%u alloc/free buffers",
 	       fb, data->rx_buf_num, MAX_MEMIF_BUFS - data->rx_buf_num);
@@ -490,10 +486,10 @@ error:
   goto close;
 
 close:
-  err = memif_refill_queue (c->conn, data->qid, rx);
+  err = memif_refill_queue (c->conn, data->qid, rx, 0);
   if (err != MEMIF_ERR_SUCCESS)
     INFO ("memif_buffer_free: %s", memif_strerror (err));
-  data->rx_buf_num -= rx;
+  data->rx_buf_num -= fb;
   DBG ("freed %d buffers. %u/%u alloc/free buffers",
        fb, data->rx_buf_num, MAX_MEMIF_BUFS - data->rx_buf_num);
   free (data->rx_bufs);
diff --git a/extras/libmemif/examples/icmp_responder-zero-copy-slave/main.c b/extras/libmemif/examples/icmp_responder-zero-copy-slave/main.c
new file mode 100644
index 00000000000..47f3ce5b01d
--- /dev/null
+++ b/extras/libmemif/examples/icmp_responder-zero-copy-slave/main.c
@@ -0,0 +1,1279 @@
+/*
+ *------------------------------------------------------------------
+ * Copyright (c) 2017 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *------------------------------------------------------------------
+ */
+
+#include <stdlib.h>
+#include <stdint.h>
+#include <net/if.h>
+#include <sys/types.h>
+#include <fcntl.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <sys/uio.h>
+#include <sys/mman.h>
+#include <sys/prctl.h>
+#include <inttypes.h>
+#include <string.h>
+#include <stdio.h>
+#include <netdb.h>
+#include <linux/ip.h>
+#include <linux/icmp.h>
+#include <arpa/inet.h>
+#include <stdlib.h>
+#include <netinet/if_ether.h>
+#include <net/if_arp.h>
+#include <asm/byteorder.h>
+#include <byteswap.h>
+#include <string.h>
+#include <sys/epoll.h>
+#include <errno.h>
+#include <unistd.h>
+#include <signal.h>
+
+#include <time.h>
+
+#include <libmemif.h>
+#include <icmp_proto.h>
+
+#define APP_NAME "ICMP_Responder"
+#define IF_NAME  "memif_connection"
+
+#define HEADROOM 0x80		/* 128b */
+#define ENCAP 0x60
+
+#ifdef ICMP_DBG
+#define DBG(...) do {                                               \
+                    printf (APP_NAME":%s:%d: ", __func__, __LINE__);         \
+                    printf (__VA_ARGS__);                           \
+                    printf ("\n");                                  \
+                } while (0)
+#define LOG(...) do {                                               \
+                    if (enable_log) {                               \
+                        dprintf (out_fd, __VA_ARGS__);              \
+                        dprintf (out_fd, "\n");                     \
+                    }                                               \
+                } while (0)
+#define LOG_FILE "/tmp/memif_time_test.txt"
+#else
+#define DBG(...)
+#define LOG(...)
+#endif
+
+#define INFO(...) do {                                              \
+                    printf ("INFO: "__VA_ARGS__);                   \
+                    printf ("\n");                                  \
+                } while (0)
+
+
+/* maximum tx/rx memif buffers */
+#define MAX_MEMIF_BUFS  256
+#define MAX_CONNS       50
+
+int epfd;
+int out_fd;
+uint8_t enable_log;
+
+typedef struct
+{
+  uint16_t index;
+  /* memif conenction handle */
+  memif_conn_handle_t conn;
+  /* buffers */
+  memif_buffer_t *bufs;
+  /* allocated tx buffers counter */
+  /* number of tx buffers pointing to shared memory */
+  uint16_t tx_buf_num;
+  /* allcoated rx buffers counter */
+  /* number of rx buffers pointing to shared memory */
+  uint16_t rx_buf_num;
+  /* interface ip address */
+  uint8_t ip_addr[4];
+  uint64_t tx_counter, rx_counter, tx_err_counter;
+  uint64_t t_sec, t_nsec;
+} memif_connection_t;
+
+typedef struct
+{
+  uint16_t index;
+  icmpr_flow_mode_t mode;
+  uint64_t packet_count;
+  uint16_t sequence;
+  uint64_t tx;
+  uint8_t ip_daddr[4];
+  uint8_t hw_daddr[6];
+  struct timespec *start, end;
+} icmpr_flow_t;
+
+memif_connection_t memif_connection[MAX_CONNS];
+long ctx[MAX_CONNS];
+icmpr_flow_t *flow;
+
+/* print details for all memif connections */
+static void
+print_memif_details ()
+{
+  memif_details_t md;
+  ssize_t buflen;
+  char *buf;
+  int err, i, e;
+  buflen = 2048;
+  buf = malloc (buflen);
+  printf ("MEMIF DETAILS\n");
+  printf ("==============================\n");
+  for (i = 0; i < MAX_CONNS; i++)
+    {
+      memif_connection_t *c = &memif_connection[i];
+
+      memset (&md, 0, sizeof (md));
+      memset (buf, 0, buflen);
+
+      err = memif_get_details (c->conn, &md, buf, buflen);
+      if (err != MEMIF_ERR_SUCCESS)
+	{
+	  if (err != MEMIF_ERR_NOCONN)
+	    INFO ("%s", memif_strerror (err));
+	  continue;
+	}
+
+      printf ("interface index: %d\n", i);
+
+      printf ("\tinterface ip: %u.%u.%u.%u\n",
+	      c->ip_addr[0], c->ip_addr[1], c->ip_addr[2], c->ip_addr[3]);
+      printf ("\tinterface name: %s\n", (char *) md.if_name);
+      printf ("\tapp name: %s\n", (char *) md.inst_name);
+      printf ("\tremote interface name: %s\n", (char *) md.remote_if_name);
+      printf ("\tremote app name: %s\n", (char *) md.remote_inst_name);
+      printf ("\tid: %u\n", md.id);
+      printf ("\tsecret: %s\n", (char *) md.secret);
+      printf ("\trole: ");
+      if (md.role)
+	printf ("slave\n");
+      else
+	printf ("master\n");
+      printf ("\tmode: ");
+      switch (md.mode)
+	{
+	case 0:
+	  printf ("ethernet\n");
+	  break;
+	case 1:
+	  printf ("ip\n");
+	  break;
+	case 2:
+	  printf ("punt/inject\n");
+	  break;
+	default:
+	  printf ("unknown\n");
+	  break;
+	}
+      printf ("\tsocket filename: %s\n", (char *) md.socket_filename);
+      printf ("\trx queues:\n");
+      for (e = 0; e < md.rx_queues_num; e++)
+	{
+	  printf ("\t\tqueue id: %u\n", md.rx_queues[e].qid);
+	  printf ("\t\tring size: %u\n", md.rx_queues[e].ring_size);
+	  printf ("\t\tring rx mode: %s\n",
+		  md.rx_queues[e].flags ? "polling" : "interrupt");
+	  printf ("\t\tring head: %u\n", md.rx_queues[e].head);
+	  printf ("\t\tring tail: %u\n", md.rx_queues[e].tail);
+	  printf ("\t\tbuffer size: %u\n", md.rx_queues[e].buffer_size);
+	}
+      printf ("\ttx queues:\n");
+      for (e = 0; e < md.tx_queues_num; e++)
+	{
+	  printf ("\t\tqueue id: %u\n", md.tx_queues[e].qid);
+	  printf ("\t\tring size: %u\n", md.tx_queues[e].ring_size);
+	  printf ("\t\tring rx mode: %s\n",
+		  md.tx_queues[e].flags ? "polling" : "interrupt");
+	  printf ("\t\tring head: %u\n", md.tx_queues[e].head);
+	  printf ("\t\tring tail: %u\n", md.tx_queues[e].tail);
+	  printf ("\t\tbuffer size: %u\n", md.tx_queues[e].buffer_size);
+	}
+      printf ("\tlink: ");
+      if (md.link_up_down)
+	printf ("up\n");
+      else
+	printf ("down\n");
+    }
+  free (buf);
+}
+
+int
+add_epoll_fd (int fd, uint32_t events)
+{
+  if (fd < 0)
+    {
+      DBG ("invalid fd %d", fd);
+      return -1;
+    }
+  struct epoll_event evt;
+  memset (&evt, 0, sizeof (evt));
+  evt.events = events;
+  evt.data.fd = fd;
+  if (epoll_ctl (epfd, EPOLL_CTL_ADD, fd, &evt) < 0)
+    {
+      DBG ("epoll_ctl: %s fd %d", strerror (errno), fd);
+      return -1;
+    }
+  DBG ("fd %d added to epoll", fd);
+  return 0;
+}
+
+int
+mod_epoll_fd (int fd, uint32_t events)
+{
+  if (fd < 0)
+    {
+      DBG ("invalid fd %d", fd);
+      return -1;
+    }
+  struct epoll_event evt;
+  memset (&evt, 0, sizeof (evt));
+  evt.events = events;
+  evt.data.fd = fd;
+  if (epoll_ctl (epfd, EPOLL_CTL_MOD, fd, &evt) < 0)
+    {
+      DBG ("epoll_ctl: %s fd %d", strerror (errno), fd);
+      return -1;
+    }
+  DBG ("fd %d moddified on epoll", fd);
+  return 0;
+}
+
+int
+del_epoll_fd (int fd)
+{
+  if (fd < 0)
+    {
+      DBG ("invalid fd %d", fd);
+      return -1;
+    }
+  struct epoll_event evt;
+  memset (&evt, 0, sizeof (evt));
+  if (epoll_ctl (epfd, EPOLL_CTL_DEL, fd, &evt) < 0)
+    {
+      DBG ("epoll_ctl: %s fd %d", strerror (errno), fd);
+      return -1;
+    }
+  DBG ("fd %d removed from epoll", fd);
+  return 0;
+}
+
+/* informs user about connected status. private_ctx is used by user to identify connection
+    (multiple connections WIP) */
+int
+on_connect (memif_conn_handle_t conn, void *private_ctx)
+{
+  INFO ("memif connected!");
+  memif_refill_queue (conn, 0, -1, HEADROOM);
+  enable_log = 1;
+  return 0;
+}
+
+/* informs user about disconnected status. private_ctx is used by user to identify connection
+    (multiple connections WIP) */
+int
+on_disconnect (memif_conn_handle_t conn, void *private_ctx)
+{
+  INFO ("memif disconnected!");
+  return 0;
+}
+
+/* user needs to watch new fd or stop watching fd that is about to be closed.
+    control fd will be modified during connection establishment to minimize CPU usage */
+int
+control_fd_update (int fd, uint8_t events)
+{
+  /* convert memif event definitions to epoll events */
+  if (events & MEMIF_FD_EVENT_DEL)
+    return del_epoll_fd (fd);
+
+  uint32_t evt = 0;
+  if (events & MEMIF_FD_EVENT_READ)
+    evt |= EPOLLIN;
+  if (events & MEMIF_FD_EVENT_WRITE)
+    evt |= EPOLLOUT;
+
+  if (events & MEMIF_FD_EVENT_MOD)
+    return mod_epoll_fd (fd, evt);
+
+  return add_epoll_fd (fd, evt);
+}
+
+/* called when event is polled on interrupt file descriptor.
+    there are packets in shared memory ready to be received */
+/* handle packet processing in rx buffer then enqueue this buffer to tx and transmit */
+int
+on_interrupt (memif_conn_handle_t conn, void *private_ctx, uint16_t qid)
+{
+  long index = *((long *) private_ctx);
+  memif_connection_t *c = &memif_connection[index];
+  if (c->index != index)
+    {
+      INFO ("invalid context: %ld/%u", index, c->index);
+      return 0;
+    }
+
+  int err = MEMIF_ERR_SUCCESS, ret_val;
+  uint16_t rx = 0, tx = 0;
+  uint16_t fb = 0;
+  int i = 0;			/* rx buffer iterator */
+  int j = 0;			/* tx bufferiterator */
+
+  /* loop while there are packets in shm */
+  do
+    {
+      /* receive data from shared memory buffers (dequeue rx buffers) */
+      err = memif_rx_burst (c->conn, qid, c->bufs, MAX_MEMIF_BUFS, &rx);
+      ret_val = err;
+      c->rx_counter += rx;
+      c->rx_buf_num += rx;
+      if ((err != MEMIF_ERR_SUCCESS) && (err != MEMIF_ERR_NOBUF))
+	{
+	  INFO ("memif_rx_burst: %s", memif_strerror (err));
+	  goto error;
+	}
+
+      /* process bufers in place */
+      for (i = 0; i < rx; i++)
+	{
+	  resolve_packet2 ((void *) (c->bufs + i)->data,
+			   &(c->bufs + i)->len, c->ip_addr);
+	}
+
+      /* enque processed buffers to tx ring */
+      err = memif_buffer_enq_tx (c->conn, qid, c->bufs, i, &tx);
+      if ((err != MEMIF_ERR_SUCCESS) && (err != MEMIF_ERR_NOBUF_RING))
+	{
+	  INFO ("memif_buffer_alloc: %s", memif_strerror (err));
+	  goto error;
+	}
+      c->rx_buf_num -= tx;
+      c->tx_buf_num += tx;
+      c->tx_err_counter += i - tx;
+
+      /* mark memif buffers and shared memory buffers as free */
+      err = memif_refill_queue (c->conn, qid, rx, HEADROOM);
+      if (err != MEMIF_ERR_SUCCESS)
+	INFO ("memif_buffer_free: %s", memif_strerror (err));
+      c->rx_buf_num -= rx;
+
+      DBG ("freed %d buffers. %u/%u alloc/free buffers",
+	   fb, rx, MAX_MEMIF_BUFS - rx);
+
+      /* transmit allocated buffers */
+      err = memif_tx_burst (c->conn, qid, c->bufs, rx, &tx);
+      if (err != MEMIF_ERR_SUCCESS)
+	{
+	  INFO ("memif_tx_burst: %s", memif_strerror (err));
+	  goto error;
+	}
+      c->tx_counter += tx;
+
+    }
+  while (ret_val == MEMIF_ERR_NOBUF);
+
+  return 0;
+
+error:
+  err = memif_refill_queue (c->conn, qid, -1, HEADROOM);
+  if (err != MEMIF_ERR_SUCCESS)
+    INFO ("memif_buffer_free: %s", memif_strerror (err));
+  c->rx_buf_num = 0;
+  DBG ("freed %d buffers. %u/%u alloc/free buffers",
+       fb, c->rx_buf_num, MAX_MEMIF_BUFS - c->rx_buf_num);
+  return 0;
+}
+
+/* add ethernet encap to packet in rx buffer then enqueue this buffer to tx and tranmit */
+int
+on_interrupt0 (memif_conn_handle_t conn, void *private_ctx, uint16_t qid)
+{
+  long index = *((long *) private_ctx);
+  memif_connection_t *c = &memif_connection[index];
+  if (c->index != index)
+    {
+      INFO ("invalid context: %ld/%u", index, c->index);
+      return 0;
+    }
+
+  int err = MEMIF_ERR_SUCCESS, ret_val;
+  uint16_t rx = 0, tx = 0;
+  uint16_t fb = 0;
+  int i = 0;			/* rx buffer iterator */
+  int j = 0;			/* tx bufferiterator */
+
+  /* loop while there are packets in shm */
+  do
+    {
+      /* receive data from shared memory buffers (dequeue rx buffers) */
+      err = memif_rx_burst (c->conn, qid, c->bufs, MAX_MEMIF_BUFS, &rx);
+      ret_val = err;
+      c->rx_counter += rx;
+      c->rx_buf_num += rx;
+      if ((err != MEMIF_ERR_SUCCESS) && (err != MEMIF_ERR_NOBUF))
+	{
+	  INFO ("memif_rx_burst: %s", memif_strerror (err));
+	  goto error;
+	}
+
+      /* process bufers in place */
+      for (i = 0; i < rx; i++)
+	{
+	  resolve_packet3 (&c->bufs[i].data, &c->bufs[i].len, c->ip_addr);
+	}
+      /* enque processed buffers to tx ring */
+      err = memif_buffer_enq_tx (c->conn, qid, c->bufs, rx, &tx);
+      if ((err != MEMIF_ERR_SUCCESS) && (err != MEMIF_ERR_NOBUF_RING))
+	{
+	  INFO ("memif_buffer_alloc: %s", memif_strerror (err));
+	  goto error;
+	}
+      c->rx_buf_num -= tx;
+      c->tx_buf_num += tx;
+      c->tx_err_counter += i - tx;
+
+      /* mark memif buffers and shared memory buffers as free */
+      err = memif_refill_queue (c->conn, qid, rx, HEADROOM);
+      if (err != MEMIF_ERR_SUCCESS)
+	INFO ("memif_buffer_free: %s", memif_strerror (err));
+      c->rx_buf_num -= rx;
+
+      DBG ("freed %d buffers. %u/%u alloc/free buffers",
+	   fb, rx, MAX_MEMIF_BUFS - rx);
+
+      /* transmit allocated buffers */
+      err = memif_tx_burst (c->conn, qid, c->bufs, i, &tx);
+      if (err != MEMIF_ERR_SUCCESS)
+	{
+	  INFO ("memif_tx_burst: %s", memif_strerror (err));
+	  goto error;
+	}
+      c->tx_counter += tx;
+
+    }
+  while (ret_val == MEMIF_ERR_NOBUF);
+
+  return 0;
+
+error:
+  err = memif_refill_queue (c->conn, qid, -1, HEADROOM);
+  if (err != MEMIF_ERR_SUCCESS)
+    INFO ("memif_buffer_free: %s", memif_strerror (err));
+  c->rx_buf_num = 0;
+  DBG ("freed %d buffers. %u/%u alloc/free buffers",
+       fb, c->rx_buf_num, MAX_MEMIF_BUFS - c->rx_buf_num);
+  return 0;
+}
+
+/* called when event is polled on interrupt file descriptor.
+    there are packets in shared memory ready to be received */
+/* dev test modification: handle only ARP requests */
+int
+on_interrupt1 (memif_conn_handle_t conn, void *private_ctx, uint16_t qid)
+{
+  long index = *((long *) private_ctx);
+  memif_connection_t *c = &memif_connection[index];
+  if (c->index != index)
+    {
+      INFO ("invalid context: %ld/%u", index, c->index);
+      return 0;
+    }
+
+  int err = MEMIF_ERR_SUCCESS, ret_val;
+  int i;
+  uint16_t rx, tx;
+  uint16_t fb;
+  uint16_t pck_seq;
+
+  do
+    {
+      /* receive data from shared memory buffers */
+      err = memif_rx_burst (c->conn, qid, c->bufs, MAX_MEMIF_BUFS, &rx);
+      ret_val = err;
+      c->rx_buf_num += rx;
+      c->rx_counter += rx;
+      if ((err != MEMIF_ERR_SUCCESS) && (err != MEMIF_ERR_NOBUF))
+	{
+	  INFO ("memif_rx_burst: %s", memif_strerror (err));
+	  goto error;
+	}
+
+      for (i = 0; i < rx; i++)
+	{
+	  if (((struct ether_header *) (c->bufs + i)->data)->ether_type ==
+	      0x0608)
+	    {
+	      /* process data in place */
+	      resolve_packet2 ((void *) (c->bufs + i)->data,
+			       &(c->bufs + i)->len, c->ip_addr);
+	      /* enque buffer to tx ring */
+	      memif_buffer_enq_tx (c->conn, qid, c->bufs, 1, &tx);
+	      c->rx_buf_num -= tx;
+	      memif_tx_burst (c->conn, qid, c->bufs, 1, &tx);
+	    }
+	}
+
+      err = memif_refill_queue (c->conn, qid, -1, HEADROOM);
+      if (err != MEMIF_ERR_SUCCESS)
+	INFO ("memif_buffer_free: %s", memif_strerror (err));
+      c->rx_buf_num -= rx;
+
+    }
+  while (ret_val == MEMIF_ERR_NOBUF);
+
+  return 0;
+
+error:
+  err = memif_refill_queue (c->conn, qid, -1, HEADROOM);
+  if (err != MEMIF_ERR_SUCCESS)
+    INFO ("memif_buffer_free: %s", memif_strerror (err));
+  c->rx_buf_num = 0;
+  DBG ("freed %d buffers. %u/%u alloc/free buffers",
+       fb, c->rx_buf_num, MAX_MEMIF_BUFS - c->rx_buf_num);
+  return 0;
+}
+
+int
+icmpr_memif_create (long index, long mode, char *s)
+{
+  if (index >= MAX_CONNS)
+    {
+      INFO ("connection array overflow");
+      return 0;
+    }
+  if (index < 0)
+    {
+      INFO ("don't even try...");
+      return 0;
+    }
+  memif_connection_t *c = &memif_connection[index];
+
+  /* setting memif connection arguments */
+  memif_conn_args_t args;
+  int fd = -1;
+  memset (&args, 0, sizeof (args));
+  args.is_master = mode;
+  args.log2_ring_size = 11;
+  args.buffer_size = 2048;
+  args.num_s2m_rings = 1;
+  args.num_m2s_rings = 1;
+  strncpy ((char *) args.interface_name, IF_NAME, strlen (IF_NAME));
+  args.mode = 0;
+  /* socket filename is not specified, because this app is supposed to
+     connect to VPP over memif. so default socket filename will be used */
+  /* default socketfile = /run/vpp/memif.sock */
+
+  args.interface_id = index;
+  /* last argument for memif_create (void * private_ctx) is used by user
+     to identify connection. this context is returned with callbacks */
+  int err;
+  /* default interrupt */
+  if (s == NULL)
+    {
+      err = memif_create (&c->conn,
+			  &args, on_connect, on_disconnect, on_interrupt,
+			  &ctx[index]);
+      if (err != MEMIF_ERR_SUCCESS)
+	{
+	  INFO ("memif_create: %s", memif_strerror (err));
+	  return 0;
+	}
+    }
+  else
+    {
+      if (strncmp (s, "0", 1) == 0)
+	{
+	  err = memif_create (&c->conn,
+			      &args, on_connect, on_disconnect, on_interrupt0,
+			      &ctx[index]);
+	  if (err != MEMIF_ERR_SUCCESS)
+	    {
+	      INFO ("memif_create: %s", memif_strerror (err));
+	      return 0;
+	    }
+	}
+      else if (strncmp (s, "1", 1) == 0)
+	{
+	  err = memif_create (&c->conn,
+			      &args, on_connect, on_disconnect, on_interrupt1,
+			      &ctx[index]);
+	  if (err != MEMIF_ERR_SUCCESS)
+	    {
+	      INFO ("memif_create: %s", memif_strerror (err));
+	      return 0;
+	    }
+	}
+      else
+	{
+	  INFO ("Unknown interrupt descriptor");
+	  goto done;
+	}
+    }
+
+  c->index = index;
+  /* alloc memif buffers */
+  c->rx_buf_num = 0;
+  c->tx_buf_num = 0;
+  c->bufs =
+    (memif_buffer_t *) malloc (sizeof (memif_buffer_t) * MAX_MEMIF_BUFS);
+
+  c->ip_addr[0] = 192;
+  c->ip_addr[1] = 168;
+  c->ip_addr[2] = c->index + 1;
+  c->ip_addr[3] = 2;
+
+  c->tx_err_counter = c->tx_counter = c->rx_counter = 0;
+
+done:
+  return 0;
+}
+
+int
+icmpr_memif_delete (long index)
+{
+  if (index >= MAX_CONNS)
+    {
+      INFO ("connection array overflow");
+      return 0;
+    }
+  if (index < 0)
+    {
+      INFO ("don't even try...");
+      return 0;
+    }
+  memif_connection_t *c = &memif_connection[index];
+
+  if (c->bufs)
+    free (c->bufs);
+  c->bufs = NULL;
+  c->tx_buf_num = 0;
+  c->rx_buf_num = 0;
+
+  int err;
+  /* disconenct then delete memif connection */
+  err = memif_delete (&c->conn);
+  if (err != MEMIF_ERR_SUCCESS)
+    INFO ("memif_delete: %s", memif_strerror (err));
+  if (c->conn != NULL)
+    INFO ("memif delete fail");
+  return 0;
+}
+
+void
+print_help ()
+{
+  printf ("LIBMEMIF EXAMPLE APP: %s", APP_NAME);
+#ifdef ICMP_DBG
+  printf (" (debug)");
+#endif
+  printf ("\n");
+  printf ("==============================\n");
+  printf ("libmemif version: %s", LIBMEMIF_VERSION);
+#ifdef MEMIF_DBG
+  printf (" (debug)");
+#endif
+  printf ("\n");
+  printf ("memif version: %d\n", memif_get_version ());
+  printf ("commands:\n");
+  printf ("\thelp - prints this help\n");
+  printf ("\texit - exit app\n");
+  printf
+    ("\tconn <index> <mode> [<interrupt-desc>] - create memif. index is also used as interface id, mode 0 = slave 1 = master, interrupt-desc none = default 0 = if ring is full wait 1 = handle only ARP requests\n");
+  printf ("\tdel  <index> - delete memif\n");
+  printf ("\tshow - show connection details\n");
+  printf ("\tip-set <index> <ip-addr> - set interface ip address\n");
+  printf
+    ("\trx-mode <index> <qid> <polling|interrupt> - set queue rx mode\n");
+  printf ("\tsh-count - print counters\n");
+  printf ("\tcl-count - clear counters\n");
+  printf
+    ("\tsend <index> <tx> <ip> <mac> - send icmp, ommit mac to transmit on ip layer\n");
+}
+
+int
+icmpr_free ()
+{
+  /* application cleanup */
+  int err;
+  long i;
+  if (out_fd > 0)
+    close (out_fd);
+  out_fd = -1;
+  for (i = 0; i < MAX_CONNS; i++)
+    {
+      memif_connection_t *c = &memif_connection[i];
+      if (c->conn)
+	icmpr_memif_delete (i);
+    }
+
+  err = memif_cleanup ();
+  if (err != MEMIF_ERR_SUCCESS)
+    INFO ("memif_delete: %s", memif_strerror (err));
+
+  return 0;
+}
+
+int
+icmpr_set_ip (long index, char *ip)
+{
+  if (index >= MAX_CONNS)
+    {
+      INFO ("connection array overflow");
+      return 0;
+    }
+  if (index < 0)
+    {
+      INFO ("don't even try...");
+      return 0;
+    }
+  memif_connection_t *c = &memif_connection[index];
+  if (c->conn == NULL)
+    {
+      INFO ("no connection at index %ld", index);
+      return 0;
+    }
+
+  char *end;
+  char *ui;
+  uint8_t tmp[4];
+  ui = strtok (ip, ".");
+  if (ui == NULL)
+    goto error;
+  tmp[0] = strtol (ui, &end, 10);
+
+  ui = strtok (NULL, ".");
+  if (ui == NULL)
+    goto error;
+  tmp[1] = strtol (ui, &end, 10);
+
+  ui = strtok (NULL, ".");
+  if (ui == NULL)
+    goto error;
+  tmp[2] = strtol (ui, &end, 10);
+
+  ui = strtok (NULL, ".");
+  if (ui == NULL)
+    goto error;
+  tmp[3] = strtol (ui, &end, 10);
+
+  c->ip_addr[0] = tmp[0];
+  c->ip_addr[1] = tmp[1];
+  c->ip_addr[2] = tmp[2];
+  c->ip_addr[3] = tmp[3];
+
+  INFO ("memif %ld ip address set to %u.%u.%u.%u",
+	index, c->ip_addr[0], c->ip_addr[1], c->ip_addr[2], c->ip_addr[3]);
+
+  return 0;
+
+error:
+  INFO ("invalid ip address");
+  return 0;
+}
+
+int
+icmpr_set_rx_mode (long index, long qid, char *mode)
+{
+  if (index >= MAX_CONNS)
+    {
+      INFO ("connection array overflow");
+      return 0;
+    }
+  if (index < 0)
+    {
+      INFO ("don't even try...");
+      return 0;
+    }
+  memif_connection_t *c = &memif_connection[index];
+
+  if (c->conn == NULL)
+    {
+      INFO ("no connection at index %ld", index);
+      return 0;
+    }
+
+  if (strncmp (mode, "interrupt", 9) == 0)
+    {
+      memif_set_rx_mode (c->conn, MEMIF_RX_MODE_INTERRUPT, qid);
+    }
+
+  else if (strncmp (mode, "polling", 7) == 0)
+    {
+      memif_set_rx_mode (c->conn, MEMIF_RX_MODE_POLLING, qid);
+    }
+  else
+    INFO ("expected rx mode <interrupt|polling>");
+  return 0;
+}
+
+void
+icmpr_print_counters ()
+{
+  int i;
+  for (i = 0; i < MAX_CONNS; i++)
+    {
+      memif_connection_t *c = &memif_connection[i];
+      if (c->conn == NULL)
+	continue;
+      printf ("===============================\n");
+      printf ("interface index: %d\n", c->index);
+      printf ("\trx: %lu\n", c->rx_counter);
+      printf ("\ttx: %lu\n", c->tx_counter);
+      printf ("\ttx_err: %lu\n", c->tx_err_counter);
+      printf ("\tts: %lus %luns\n", c->t_sec, c->t_nsec);
+    }
+}
+
+void
+icmpr_reset_counters ()
+{
+  int i;
+  for (i = 0; i < MAX_CONNS; i++)
+    {
+      memif_connection_t *c = &memif_connection[i];
+      if (c->conn == NULL)
+	continue;
+      c->t_sec = c->t_nsec = c->tx_err_counter = c->tx_counter =
+	c->rx_counter = 0;
+    }
+}
+
+void
+icmpr_send_proc ()
+{
+  memif_connection_t *c = &memif_connection[flow->index];
+  if (c->conn == NULL)
+    {
+      INFO ("No connection at index %d. Stopping flow...\n", flow->index);
+      goto stop_flow;
+    }
+  uint16_t tx, i;
+  int err = MEMIF_ERR_SUCCESS;
+
+  if (!flow->start)
+    {
+      flow->start = malloc (sizeof (struct timespec));
+      memset (flow->start, 0, sizeof (struct timespec));
+      timespec_get (flow->start, TIME_UTC);
+    }
+
+  i = 0;
+  err = memif_buffer_alloc (c->conn, 0, c->bufs,
+			    MAX_MEMIF_BUFS >
+			    flow->packet_count ? flow->packet_count :
+			    MAX_MEMIF_BUFS, &tx, 64);
+  if ((err != MEMIF_ERR_SUCCESS) && (err != MEMIF_ERR_NOBUF_RING))
+    {
+      INFO ("memif_buffer_alloc: %s Stopping flow...\n",
+	    memif_strerror (err));
+      goto stop_flow;
+    }
+  c->tx_buf_num += tx;
+
+  while (tx)
+    {
+      generate_packet2 ((void *) c->bufs[i].data,
+			&c->bufs[i].len, c->ip_addr,
+			flow->ip_daddr, flow->hw_daddr, (flow->sequence)++,
+			flow->mode);
+      i++;
+      tx--;
+    }
+  err = memif_tx_burst (c->conn, 0, c->bufs, i, &tx);
+  if (err != MEMIF_ERR_SUCCESS)
+    {
+      INFO ("memif_tx_burst: %s Stopping flow...\n", memif_strerror (err));
+      goto stop_flow;
+    }
+  c->tx_buf_num -= tx;
+  c->tx_counter += tx;
+  flow->tx += tx;
+  flow->packet_count -= tx;
+
+  if (flow->packet_count == 0)
+    {
+      timespec_get (&flow->end, TIME_UTC);
+      INFO ("Flow finished!");
+      INFO ("Flow length: %lu", flow->tx);
+      uint64_t t1 = flow->end.tv_sec - flow->start->tv_sec;
+      uint64_t t2;
+      if (flow->end.tv_nsec > flow->start->tv_nsec)
+	{
+	  t2 = flow->end.tv_nsec - flow->start->tv_nsec;
+	}
+      else
+	{
+	  t2 = flow->start->tv_nsec - flow->end.tv_nsec;
+	  t1--;
+	}
+      c->t_sec = t1;
+      c->t_nsec = t2;
+      INFO ("Flow time: %lus %luns", t1, t2);
+      double tmp = t1;
+      tmp += t2 / 1e+9;
+      tmp = flow->tx / tmp;
+      INFO ("Average pps: %f", tmp);
+      INFO ("Stopping flow...");
+      goto stop_flow;
+    }
+
+  return;
+
+stop_flow:
+  if (flow)
+    {
+      if (flow->start)
+	free (flow->start);
+      free (flow);
+    }
+  flow = NULL;
+  return;
+}
+
+int
+icmpr_send (long index, long packet_num, char *input)
+{
+  if (flow)
+    {
+      printf ("only one flow allowed\n");
+      return 0;
+    }
+
+  memif_connection_t *c = &memif_connection[index];
+  char *end;
+  char *ui;
+  uint8_t tmp[6];
+  if (c->conn == NULL)
+    return -1;
+
+  flow = malloc (sizeof (icmpr_flow_t));
+  flow->index = index;
+  flow->packet_count = packet_num;
+  flow->sequence = 0;
+  flow->tx = 0;
+  flow->start = NULL;
+  memset (&flow->end, 0, sizeof (struct timespec));
+
+  INFO ("packet count: %lu", flow->packet_count);
+  printf ("%s\n", input);
+
+  ui = strtok (input, ".");
+  if (ui == NULL)
+    goto error;
+  tmp[0] = strtol (ui, &end, 10);
+
+  ui = strtok (NULL, ".");
+  if (ui == NULL)
+    goto error;
+  tmp[1] = strtol (ui, &end, 10);
+
+  ui = strtok (NULL, ".");
+  if (ui == NULL)
+    goto error;
+  tmp[2] = strtol (ui, &end, 10);
+
+  ui = strtok (NULL, ".");
+  if (ui == NULL)
+    goto error;
+  tmp[3] = strtol (ui, &end, 10);
+
+  flow->ip_daddr[0] = tmp[0];
+  flow->ip_daddr[1] = tmp[1];
+  flow->ip_daddr[2] = tmp[2];
+  flow->ip_daddr[3] = tmp[3];
+
+  ui = strtok (NULL, " ");
+  if (ui == NULL)
+    {
+      flow->mode = ICMPR_FLOW_MODE_IP;
+      return 0;
+    }
+
+  ui = strtok (NULL, ":");
+  if (ui == NULL)
+    goto error;
+  tmp[0] = strtol (ui, &end, 16);
+  ui = strtok (NULL, ":");
+  if (ui == NULL)
+    goto error;
+  tmp[1] = strtol (ui, &end, 16);
+  ui = strtok (NULL, ":");
+  if (ui == NULL)
+    goto error;
+  tmp[2] = strtol (ui, &end, 16);
+  ui = strtok (NULL, ":");
+  if (ui == NULL)
+    goto error;
+  tmp[3] = strtol (ui, &end, 16);
+  ui = strtok (NULL, ":");
+  if (ui == NULL)
+    goto error;
+  tmp[4] = strtol (ui, &end, 16);
+  ui = strtok (NULL, ":");
+  if (ui == NULL)
+    goto error;
+  tmp[5] = strtol (ui, &end, 16);
+
+  flow->hw_daddr[0] = tmp[0];
+  flow->hw_daddr[1] = tmp[1];
+  flow->hw_daddr[2] = tmp[2];
+  flow->hw_daddr[3] = tmp[3];
+  flow->hw_daddr[4] = tmp[4];
+  flow->hw_daddr[5] = tmp[5];
+
+  flow->mode = ICMPR_FLOW_MODE_ETH;
+
+  return 0;
+
+error:
+  INFO ("Invalid input\n");
+  if (flow)
+    free (flow);
+  flow = NULL;
+  return 0;
+}
+
+int
+user_input_handler ()
+{
+  int i;
+  char *in = (char *) malloc (256);
+  char *ui = fgets (in, 256, stdin);
+  char *end;
+  long a;
+  if (in[0] == '\n')
+    goto done;
+  ui = strtok (in, " ");
+  if (strncmp (ui, "exit", 4) == 0)
+    {
+      free (in);
+      icmpr_free ();
+      exit (EXIT_SUCCESS);
+    }
+  else if (strncmp (ui, "help", 4) == 0)
+    {
+      print_help ();
+      goto done;
+    }
+  else if (strncmp (ui, "conn", 4) == 0)
+    {
+      ui = strtok (NULL, " ");
+      if (ui != NULL)
+	a = strtol (ui, &end, 10);
+      else
+	{
+	  INFO ("expected id");
+	  goto done;
+	}
+      ui = strtok (NULL, " ");
+      if (ui != NULL)
+	icmpr_memif_create (a, strtol (ui, &end, 10), strtok (NULL, " "));
+      else
+	INFO ("expected mode <0|1>");
+      goto done;
+    }
+  else if (strncmp (ui, "del", 3) == 0)
+    {
+      ui = strtok (NULL, " ");
+      if (ui != NULL)
+	icmpr_memif_delete (strtol (ui, &end, 10));
+      else
+	INFO ("expected id");
+      goto done;
+    }
+  else if (strncmp (ui, "show", 4) == 0)
+    {
+      print_memif_details ();
+      goto done;
+    }
+  else if (strncmp (ui, "ip-set", 6) == 0)
+    {
+      ui = strtok (NULL, " ");
+      if (ui != NULL)
+	icmpr_set_ip (strtol (ui, &end, 10), strtok (NULL, " "));
+      else
+	INFO ("expected id");
+      goto done;
+    }
+  else if (strncmp (ui, "rx-mode", 7) == 0)
+    {
+      ui = strtok (NULL, " ");
+      if (ui != NULL)
+	a = strtol (ui, &end, 10);
+      else
+	{
+	  INFO ("expected id");
+	  goto done;
+	}
+      ui = strtok (NULL, " ");
+      if (ui != NULL)
+	icmpr_set_rx_mode (a, strtol (ui, &end, 10), strtok (NULL, " "));
+      else
+	INFO ("expected qid");
+      goto done;
+    }
+  else if (strncmp (ui, "sh-count", 8) == 0)
+    {
+      icmpr_print_counters ();
+    }
+  else if (strncmp (ui, "cl-count", 8) == 0)
+    {
+      icmpr_reset_counters ();
+    }
+  else if (strncmp (ui, "send", 4) == 0)
+    {
+      ui = strtok (NULL, " ");
+      if (ui != NULL)
+	a = strtol (ui, &end, 10);
+      else
+	{
+	  INFO ("expected id");
+	  goto done;
+	}
+      ui = strtok (NULL, " ");
+      if (ui != NULL)
+	icmpr_send (a, strtol (ui, &end, 10), strtok (NULL, " "));
+      else
+	INFO ("expected count");
+      goto done;
+    }
+  else
+    {
+      INFO ("unknown command: %s", ui);
+      goto done;
+    }
+
+  return 0;
+done:
+  free (in);
+  return 0;
+}
+
+int
+poll_event (int timeout)
+{
+  struct epoll_event evt, *e;
+  int app_err = 0, memif_err = 0, en = 0;
+  int tmp, nfd;
+  uint32_t events = 0;
+  struct timespec start, end;
+  memset (&evt, 0, sizeof (evt));
+  evt.events = EPOLLIN | EPOLLOUT;
+  sigset_t sigset;
+  sigemptyset (&sigset);
+  en = epoll_pwait (epfd, &evt, 1, timeout, &sigset);
+  /* id event polled */
+  timespec_get (&start, TIME_UTC);
+  if (en < 0)
+    {
+      DBG ("epoll_pwait: %s", strerror (errno));
+      return -1;
+    }
+  if (en > 0)
+    {
+      /* this app does not use any other file descriptors than stds and memif control fds */
+      if (evt.data.fd > 2)
+	{
+	  /* event of memif control fd */
+	  /* convert epolle events to memif events */
+	  if (evt.events & EPOLLIN)
+	    events |= MEMIF_FD_EVENT_READ;
+	  if (evt.events & EPOLLOUT)
+	    events |= MEMIF_FD_EVENT_WRITE;
+	  if (evt.events & EPOLLERR)
+	    events |= MEMIF_FD_EVENT_ERROR;
+	  memif_err = memif_control_fd_handler (evt.data.fd, events);
+	  if (memif_err != MEMIF_ERR_SUCCESS)
+	    INFO ("memif_control_fd_handler: %s", memif_strerror (memif_err));
+	}
+      else if (evt.data.fd == 0)
+	{
+	  app_err = user_input_handler ();
+	}
+      else
+	{
+	  DBG ("unexpected event at memif_epfd. fd %d", evt.data.fd);
+	}
+    }
+
+  timespec_get (&end, TIME_UTC);
+  LOG ("interrupt: %ld", end.tv_nsec - start.tv_nsec);
+
+  if ((app_err < 0) || (memif_err < 0))
+    {
+      if (app_err < 0)
+	DBG ("user input handler error");
+      if (memif_err < 0)
+	DBG ("memif control fd handler error");
+      return -1;
+    }
+
+  return 0;
+}
+
+int
+main ()
+{
+  epfd = epoll_create (1);
+  add_epoll_fd (0, EPOLLIN);
+
+  flow = NULL;
+
+#ifdef LOG_FILE
+  remove (LOG_FILE);
+  enable_log = 0;
+
+  out_fd = open (LOG_FILE, O_WRONLY | O_CREAT, S_IRWXO);
+  if (out_fd < 0)
+    INFO ("Error opening log file: %s", strerror (errno));
+#endif /* LOG_FILE */
+
+  /* initialize memory interface */
+  int err, i;
+  /* if valid callback is passed as argument, fd event polling will be done by user
+     all file descriptors and events will be passed to user in this callback */
+  /* if callback is set to NULL libmemif will handle fd event polling */
+  err = memif_init (control_fd_update, APP_NAME, NULL, NULL);
+  if (err != MEMIF_ERR_SUCCESS)
+    {
+      INFO ("memif_init: %s", memif_strerror (err));
+      icmpr_free ();
+      exit (-1);
+    }
+
+  for (i = 0; i < MAX_CONNS; i++)
+    {
+      memif_connection[i].conn = NULL;
+      ctx[i] = i;
+    }
+
+  print_help ();
+
+  /* main loop */
+  while (1)
+    {
+      if (poll_event (0) < 0)
+	{
+	  DBG ("poll_event error!");
+	}
+      if (flow)
+	{
+	  icmpr_send_proc ();
+	}
+    }
+}
diff --git a/extras/libmemif/examples/icmp_responder/icmp_proto.c b/extras/libmemif/examples/icmp_responder/icmp_proto.c
index b27abb6a231..e1d255aca27 100644
--- a/extras/libmemif/examples/icmp_responder/icmp_proto.c
+++ b/extras/libmemif/examples/icmp_responder/icmp_proto.c
@@ -37,6 +37,7 @@
 #include <net/if_arp.h>
 #include <asm/byteorder.h>
 #include <byteswap.h>
+#include <assert.h>
 
 #include <icmp_proto.h>
 
@@ -337,3 +338,188 @@ generate_packet (void *pck, uint32_t * size, uint8_t saddr[4],
 
   return 0;
 }
+
+int
+generate_packet2 (void *pck, uint32_t * size, uint8_t saddr[4],
+		  uint8_t daddr[4], uint8_t hw_daddr[6], uint32_t seq,
+		  icmpr_flow_mode_t mode)
+{
+  struct ether_header *eh;
+  struct iphdr *ip;
+  struct icmphdr *icmp;
+
+  *size = 0;
+
+  if (mode == ICMPR_FLOW_MODE_ETH)
+    {
+      eh = (struct ether_header *) pck;
+      *size += generate_eth (eh, hw_daddr);
+    }
+
+  ip = (struct iphdr *) (pck + *size);
+  *size += generate_ip (ip, saddr, daddr);
+
+  icmp = (struct icmphdr *) (pck + *size);
+  *size += generate_icmp (icmp, seq);
+
+  ((struct icmphdr *) (pck + *size - sizeof (struct icmphdr)))->checksum =
+    cksum (pck + *size - sizeof (struct icmphdr), sizeof (struct icmphdr));
+
+  ip->tot_len = __bswap_16 (*size - sizeof (struct ether_header));
+  ip->check = 0;
+  ip->check = cksum (ip, sizeof (struct iphdr));
+
+  return 0;
+}
+
+#define GET_HEADER(out,hdr,src,off) do {	\
+					out = (hdr*)(src + off); \
+					off += sizeof (hdr); \
+				} while (0)
+
+int
+resolve_packet2 (void *pck, uint32_t * size, uint8_t ip_addr[4])
+{
+  struct ether_header *eh;
+  struct ether_arp *eah;
+  struct iphdr *ip;
+  struct icmphdr *icmp;
+  uint32_t offset = 0;
+
+  if (pck == NULL)
+    return 0;
+
+  GET_HEADER (eh, struct ether_header, pck, offset);
+
+  memcpy (eh->ether_dhost, eh->ether_shost, 6);
+  memcpy (eh->ether_shost, "aaaaaa", 6);
+
+  if (eh->ether_type == 0x0608)
+    {
+      GET_HEADER (eah, struct ether_arp, pck, offset);
+      struct arphdr *arp = &eah->ea_hdr;
+
+      arp->ar_hrd = __bswap_16 (ARPHRD_ETHER);
+      arp->ar_pro = __bswap_16 (0x0800);
+
+      arp->ar_hln = 6;
+      arp->ar_pln = 4;
+
+      arp->ar_op = __bswap_16 (ARPOP_REPLY);
+
+      memcpy (eah->arp_tha, eah->arp_sha, 6);
+      memcpy (eah->arp_tpa, eah->arp_spa, 4);
+
+      memcpy (eah->arp_sha, eh->ether_shost, 6);
+      memcpy (eah->arp_spa, ip_addr, 4);
+    }
+
+  else if (eh->ether_type == 0x0008)
+    {
+      GET_HEADER (ip, struct iphdr, pck, offset);
+
+      if (ip->protocol == 1)
+	{
+	  ip->ihl = 5;
+	  ip->version = 4;
+	  ip->tos = 0;
+	  ip->tot_len = 0x0000;
+	  ip->id = 0;
+	  ip->frag_off = 0;
+	  ip->ttl = 0x40;
+	  ip->protocol = 1;
+	  ip->check = 0x0000;
+	  ip->daddr = ip->saddr;
+	  ((uint8_t *) & ip->saddr)[0] = ip_addr[0];
+	  ((uint8_t *) & ip->saddr)[1] = ip_addr[1];
+	  ((uint8_t *) & ip->saddr)[2] = ip_addr[2];
+	  ((uint8_t *) & ip->saddr)[3] = ip_addr[3];
+
+	  GET_HEADER (icmp, struct icmphdr, pck, offset);
+
+	  icmp->type = 0x00;
+	  icmp->code = 0;
+	  icmp->checksum = cksum (icmp, sizeof (struct icmphdr));
+
+	  /* rest is payload */
+	  offset = *size;
+
+	  ip->tot_len = __bswap_16 (offset - sizeof (struct ether_header));
+	  ip->check = cksum (ip, sizeof (struct iphdr));
+	}
+    }
+
+  assert (offset == *size && "unsupported protocol");
+  return 0;
+}
+
+
+int
+resolve_packet3 (void **pck_, uint32_t * size, uint8_t ip_addr[4])
+{
+  struct ether_header *eh;
+  struct ether_arp *eah;
+  struct iphdr *ip;
+  struct icmphdr *icmp;
+  int32_t offset = 0;
+  uint16_t encap_size = sizeof (struct ether_header);
+  void *pck = *pck_;
+
+  if (pck == NULL)
+    return 0;
+
+  *pck_ -= encap_size;
+  offset -= encap_size;
+
+  GET_HEADER (eh, struct ether_header, pck, offset);
+
+  uint8_t hw_daddr[6];
+  memset (hw_daddr, 0, sizeof (uint8_t) * 6);
+
+  generate_eth (eh, hw_daddr);
+
+  if (eh->ether_type == 0x0008)
+    {
+      GET_HEADER (ip, struct iphdr, pck, offset);
+
+      if (ip->protocol == 1)
+	{
+	  ip->ihl = 5;
+	  ip->version = 4;
+	  ip->tos = 0;
+	  ip->tot_len = 0x0000;
+	  ip->id = 0;
+	  ip->frag_off = 0;
+	  ip->ttl = 0x40;
+	  ip->protocol = 1;
+	  ip->check = 0x0000;
+	  ip->daddr = ip->saddr;
+	  ((uint8_t *) & ip->saddr)[0] = ip_addr[0];
+	  ((uint8_t *) & ip->saddr)[1] = ip_addr[1];
+	  ((uint8_t *) & ip->saddr)[2] = ip_addr[2];
+	  ((uint8_t *) & ip->saddr)[3] = ip_addr[3];
+
+	  GET_HEADER (icmp, struct icmphdr, pck, offset);
+
+	  icmp->type = 0x00;
+	  icmp->code = 0;
+	  icmp->checksum = cksum (icmp, sizeof (struct icmphdr));
+
+	  /* rest is payload */
+	  offset = *size;
+
+	  ip->tot_len = __bswap_16 (offset - sizeof (struct ether_header));
+	  ip->check = cksum (ip, sizeof (struct iphdr));
+	}
+    }
+
+  offset += encap_size;
+
+  assert (offset != *size &&
+	  "new packet length must be increased by encap size");
+
+  /* overwrite packet size */
+  *size = offset;
+
+  return 0;
+}
diff --git a/extras/libmemif/examples/icmp_responder/icmp_proto.h b/extras/libmemif/examples/icmp_responder/icmp_proto.h
index ca72b2fe026..1575719bd6a 100644
--- a/extras/libmemif/examples/icmp_responder/icmp_proto.h
+++ b/extras/libmemif/examples/icmp_responder/icmp_proto.h
@@ -18,12 +18,28 @@
 #ifndef _ICMP_PROTO_H_
 #define _ICMP_PROTO_H_
 
+typedef enum
+{
+  ICMPR_FLOW_MODE_ETH = 0,
+  ICMPR_FLOW_MODE_IP,
+} icmpr_flow_mode_t;
+
 int resolve_packet (void *in_pck, ssize_t in_size, void *out_pck,
 		    uint32_t * out_size, uint8_t ip_addr[4]);
 
+/* resolve packet in place */
+int resolve_packet2 (void *pck, uint32_t * size, uint8_t ip_addr[4]);
+
+/* resolve packet in place and add eth encap */
+int resolve_packet3 (void **pck, uint32_t * size, uint8_t ip_addr[4]);
+
 int generate_packet (void *pck, uint32_t * size, uint8_t saddr[4],
 		     uint8_t daddr[4], uint8_t hw_daddr[6], uint32_t seq);
 
+int generate_packet2 (void *pck, uint32_t * size, uint8_t saddr[4],
+		      uint8_t daddr[4], uint8_t hw_daddr[6], uint32_t seq,
+		      icmpr_flow_mode_t);
+
 int print_packet (void *pck);
 
 #endif /* _ICMP_PROTO_H_ */
diff --git a/extras/libmemif/examples/icmp_responder/main.c b/extras/libmemif/examples/icmp_responder/main.c
index b820ad3b12d..5e78acd3537 100644
--- a/extras/libmemif/examples/icmp_responder/main.c
+++ b/extras/libmemif/examples/icmp_responder/main.c
@@ -225,7 +225,7 @@ icmpr_buffer_alloc (long n, uint16_t qid)
   int err;
   uint16_t r;
   /* set data pointer to shared memory and set buffer_len to shared mmeory buffer len */
-  err = memif_buffer_alloc (c->conn, qid, c->tx_bufs, n, &r, 128);
+  err = memif_buffer_alloc (c->conn, qid, c->tx_bufs, n, &r, 0);
   if (err != MEMIF_ERR_SUCCESS)
     {
       INFO ("memif_buffer_alloc: %s", memif_strerror (err));
@@ -311,24 +311,25 @@ on_interrupt (memif_conn_handle_t conn, void *private_ctx, uint16_t qid)
 		      &(c->tx_bufs + i)->len, c->ip_addr);
     }
 
+  uint16_t fb;
   /* mark memif buffers and shared memory buffers as free */
-  err = memif_refill_queue (c->conn, qid, rx);
-  c->rx_buf_num -= rx;
+  err = memif_refill_queue (c->conn, qid, rx, 0);
+  c->rx_buf_num -= fb;
 
   DBG ("freed %d buffers. %u/%u alloc/free buffers",
-       rx, c->rx_buf_num, MAX_MEMIF_BUFS - c->rx_buf_num);
+       fb, c->rx_buf_num, MAX_MEMIF_BUFS - c->rx_buf_num);
 
   icmpr_tx_burst (c->tx_qid);
 
   return 0;
 
 error:
-  err = memif_refill_queue (c->conn, qid, rx);
+  err = memif_refill_queue (c->conn, qid, rx, 0);
   if (err != MEMIF_ERR_SUCCESS)
     INFO ("memif_buffer_free: %s", memif_strerror (err));
-  c->rx_buf_num -= rx;
+  c->rx_buf_num -= fb;
   DBG ("freed %d buffers. %u/%u alloc/free buffers",
-       rx, c->rx_buf_num, MAX_MEMIF_BUFS - c->rx_buf_num);
+       fb, c->rx_buf_num, MAX_MEMIF_BUFS - c->rx_buf_num);
   return 0;
 }
 
diff --git a/extras/libmemif/src/libmemif.h b/extras/libmemif/src/libmemif.h
index e1f96f90c80..88b6a10f7b4 100644
--- a/extras/libmemif/src/libmemif.h
+++ b/extras/libmemif/src/libmemif.h
@@ -203,15 +203,20 @@ typedef enum
 
 /** \brief Memif buffer
     @param desc_index - ring descriptor index
-    @param len - buffer length
+    @param ring - pointer to ring containing descriptor for this buffer
+    @param len - available length
     @param flags - memif buffer flags
     @param data - pointer to shared memory data
 */
 typedef struct
 {
   uint16_t desc_index;
+  void *ring;
   uint32_t len;
+/** next buffer present (chained buffers) */
 #define MEMIF_BUFFER_FLAG_NEXT (1 << 0)
+/** states that buffer is from rx ring */
+#define MEMIF_BUFFER_FLAG_RX (1 << 1)
   uint8_t flags;
   void *data;
 } memif_buffer_t;
@@ -423,6 +428,22 @@ int memif_control_fd_handler (int fd, uint8_t events);
 */
 int memif_delete (memif_conn_handle_t * conn);
 
+/** \brief Memif buffer enq tx
+    @param conn - memif conenction handle
+    @param qid - number indentifying queue
+    @param bufs - memif buffers
+    @param count - number of memif buffers to enque
+    @param count_out - returns number of allocated buffers
+
+    Slave is producer of buffers.
+    If connection handle points to master returns MEMIF_ERR_INVAL_ARG.
+
+    \return memif_err_t
+*/
+int memif_buffer_enq_tx (memif_conn_handle_t conn, uint16_t qid,
+			 memif_buffer_t * bufs, uint16_t count,
+			 uint16_t * count_out);
+
 /** \brief Memif buffer alloc
     @param conn - memif conenction handle
     @param qid - number indentifying queue
@@ -441,11 +462,12 @@ int memif_buffer_alloc (memif_conn_handle_t conn, uint16_t qid,
     @param conn - memif conenction handle
     @param qid - number indentifying queue
     @param count - number of buffers to be placed on ring
+    @param headroom - offset the buffer by headroom
 
     \return memif_err_t
 */
 int memif_refill_queue (memif_conn_handle_t conn, uint16_t qid,
-			uint16_t count);
+			uint16_t count, uint16_t headroom);
 
 /** \brief Memif transmit buffer burst
     @param conn - memif conenction handle
diff --git a/extras/libmemif/src/main.c b/extras/libmemif/src/main.c
index dc4a6231118..56458515485 100644
--- a/extras/libmemif/src/main.c
+++ b/extras/libmemif/src/main.c
@@ -144,7 +144,7 @@ const char *memif_errlist[ERRLIST_LEN] = {	/* MEMIF_ERR_SUCCESS */
   /* MEMIF_ERR_POLL_CANCEL */
   "Memif event polling was canceled.",
   /* MEMIF_ERR_MAX_RING */
-  "Maximum log2 ring size is 15.",
+  "Maximum log2 ring size is 15",
   /* MEMIF_ERR_PRIVHDR */
   "Private headers not supported."
 };
@@ -1324,7 +1324,6 @@ int
 memif_init_regions_and_queues (memif_connection_t * conn)
 {
   memif_ring_t *ring = NULL;
-  uint64_t buffer_offset;
   memif_region_t *r;
   int i, j;
   libmemif_main_t *lm = &libmemif_main;
@@ -1335,18 +1334,19 @@ memif_init_regions_and_queues (memif_connection_t * conn)
     return MEMIF_ERR_NOMEM;
   r = conn->regions;
 
-  buffer_offset =
+  r->buffer_offset =
     (conn->run_args.num_s2m_rings +
      conn->run_args.num_m2s_rings) * (sizeof (memif_ring_t) +
 				      sizeof (memif_desc_t) *
 				      (1 << conn->run_args.log2_ring_size));
 
-  r->region_size = buffer_offset +
+  r->region_size = r->buffer_offset +
     conn->run_args.buffer_size * (1 << conn->run_args.log2_ring_size) *
     (conn->run_args.num_s2m_rings + conn->run_args.num_m2s_rings);
 
   if ((r->fd =
        memfd_create ("memif region 0", MFD_ALLOW_SEALING)) == -1)
+  if ((r->fd = memfd_create ("memif region 0", MFD_ALLOW_SEALING)) == -1)
     return memif_syscall_error_handler (errno);
 
   if ((fcntl (r->fd, F_ADD_SEALS, F_SEAL_SHRINK)) == -1)
@@ -1370,7 +1370,7 @@ memif_init_regions_and_queues (memif_connection_t * conn)
 	{
 	  uint16_t slot = i * (1 << conn->run_args.log2_ring_size) + j;
 	  ring->desc[j].region = 0;
-	  ring->desc[j].offset = buffer_offset +
+	  ring->desc[j].offset = r->buffer_offset +
 	    (uint32_t) (slot * conn->run_args.buffer_size);
 	  ring->desc[j].length = conn->run_args.buffer_size;
 	}
@@ -1389,7 +1389,7 @@ memif_init_regions_and_queues (memif_connection_t * conn)
 	     conn->run_args.num_s2m_rings) *
 	    (1 << conn->run_args.log2_ring_size) + j;
 	  ring->desc[j].region = 0;
-	  ring->desc[j].offset = buffer_offset +
+	  ring->desc[j].offset = r->buffer_offset +
 	    (uint32_t) (slot * conn->run_args.buffer_size);
 	  ring->desc[j].length = conn->run_args.buffer_size;
 	}
@@ -1417,7 +1417,6 @@ memif_init_regions_and_queues (memif_connection_t * conn)
       mq[x].offset =
 	(void *) mq[x].ring - (void *) conn->regions[mq->region].shm;
       mq[x].last_head = 0;
-      mq[x].last_tail = 0;
       mq[x].alloc_bufs = 0;
     }
   conn->tx_queues = mq;
@@ -1443,7 +1442,6 @@ memif_init_regions_and_queues (memif_connection_t * conn)
       mq[x].offset =
 	(void *) mq[x].ring - (void *) conn->regions[mq->region].shm;
       mq[x].last_head = 0;
-      mq[x].last_tail = 0;
       mq[x].alloc_bufs = 0;
     }
   conn->rx_queues = mq;
@@ -1452,6 +1450,87 @@ memif_init_regions_and_queues (memif_connection_t * conn)
 }
 
 int
+memif_buffer_enq_tx (memif_conn_handle_t conn, uint16_t qid,
+		     memif_buffer_t * bufs, uint16_t count,
+		     uint16_t * count_out)
+{
+  memif_connection_t *c = (memif_connection_t *) conn;
+  if (EXPECT_FALSE (c == NULL))
+    return MEMIF_ERR_NOCONN;
+  if (EXPECT_FALSE (c->fd < 0))
+    return MEMIF_ERR_DISCONNECTED;
+  uint8_t num =
+    (c->args.is_master) ? c->run_args.num_m2s_rings : c->run_args.
+    num_s2m_rings;
+  if (EXPECT_FALSE (qid >= num))
+    return MEMIF_ERR_QID;
+  if (EXPECT_FALSE (!count_out))
+    return MEMIF_ERR_INVAL_ARG;
+  if (EXPECT_FALSE (c->args.is_master))
+    return MEMIF_ERR_INVAL_ARG;
+
+  memif_queue_t *mq = &c->tx_queues[qid];
+  memif_ring_t *ring = mq->ring;
+  memif_buffer_t *b0;
+  uint16_t mask = (1 << mq->log2_ring_size) - 1;
+  uint16_t ring_size;
+  uint16_t slot, ns;
+  int i, err = MEMIF_ERR_SUCCESS;	/* 0 */
+  *count_out = 0;
+
+  ring_size = (1 << mq->log2_ring_size);
+  ns = ring->tail - mq->last_tail;
+  mq->last_tail += ns;
+  slot = (c->args.is_master) ? ring->tail : ring->head;
+  slot += mq->alloc_bufs;
+
+  /* can only be called by slave */
+  ns = ring_size - ring->head + mq->alloc_bufs + mq->last_tail;
+
+  b0 = bufs;
+
+  while (count && ns)
+    {
+      if (EXPECT_FALSE ((b0->flags & MEMIF_BUFFER_FLAG_RX) == 0))
+	{
+	  /* not a valid buffer */
+	  count--;
+	  continue;
+	}
+      b0->flags &= ~MEMIF_BUFFER_FLAG_RX;
+
+      ((memif_ring_t *) b0->ring)->desc[b0->desc_index & mask].offset = ring->desc[slot & mask].offset;	/* put free buffer on rx ring */
+
+      ring->desc[slot & mask].offset =
+	(uint32_t) (b0->data - c->regions->shm);
+      ring->desc[slot & mask].flags |=
+	(b0->flags & MEMIF_BUFFER_FLAG_NEXT) ? MEMIF_DESC_FLAG_NEXT : 0;
+
+      b0->desc_index = slot;
+
+      mq->alloc_bufs++;
+      slot++;
+
+      count--;
+      ns--;
+      b0++;
+      *count_out += 1;
+    }
+
+  DBG ("allocated: %u/%u bufs. Total %u allocated bufs", *count_out, count,
+       mq->alloc_bufs);
+
+  if (count)
+    {
+      DBG ("ring buffer full! qid: %u", qid);
+      err = MEMIF_ERR_NOBUF_RING;
+    }
+
+error:
+  return err;
+}
+
+int
 memif_buffer_alloc (memif_conn_handle_t conn, uint16_t qid,
 		    memif_buffer_t * bufs, uint16_t count,
 		    uint16_t * count_out, uint16_t size)
@@ -1503,12 +1582,13 @@ memif_buffer_alloc (memif_conn_handle_t conn, uint16_t qid,
       ring->desc[slot & mask].flags = 0;
 
       /* slave can produce buffer with original length */
-      dst_left = (c->args.is_master) ? ring->desc[slot & mask].length : c->run_args.buffer_size;	/* - headroom */
+      dst_left = (c->args.is_master) ? ring->desc[slot & mask].length :
+	c->run_args.buffer_size;
       src_left = size;
 
       while (src_left)
 	{
-	  if (dst_left == 0)
+	  if (EXPECT_FALSE (dst_left == 0))
 	    {
 	      if (count && ns)
 		{
@@ -1524,7 +1604,9 @@ memif_buffer_alloc (memif_conn_handle_t conn, uint16_t qid,
 
 		  b0 = (bufs + *count_out);
 		  b0->desc_index = slot;
-		  dst_left = (c->args.is_master) ? ring->desc[slot & mask].length : c->run_args.buffer_size;	/* - headroom */
+		  dst_left =
+		    (c->args.is_master) ? ring->desc[slot & mask].length : c->
+		    run_args.buffer_size;
 		  ring->desc[slot & mask].flags = 0;
 		}
 	      else
@@ -1538,7 +1620,18 @@ memif_buffer_alloc (memif_conn_handle_t conn, uint16_t qid,
 		}
 	    }
 	  b0->len = memif_min (dst_left, src_left);
-	  b0->data = memif_get_buffer (c, ring, slot & mask);
+
+	  /* slave resets buffer offset */
+	  if (c->args.is_master == 0)
+	    {
+	      uint16_t x =
+		(ring->desc[slot & mask].offset -
+		 c->regions->buffer_offset) / c->run_args.buffer_size;
+	      ring->desc[slot & mask].offset =
+		c->regions->buffer_offset + (x * c->run_args.buffer_size);
+	    }
+
+	  b0->data = c->regions->shm + ring->desc[slot & mask].offset;
 
 	  src_left -= b0->len;
 	  dst_left -= b0->len;
@@ -1562,11 +1655,13 @@ no_ns:
       err = MEMIF_ERR_NOBUF_RING;
     }
 
+error:
   return err;
 }
 
 int
-memif_refill_queue (memif_conn_handle_t conn, uint16_t qid, uint16_t count)
+memif_refill_queue (memif_conn_handle_t conn, uint16_t qid, uint16_t count,
+		    uint16_t headroom)
 {
   memif_connection_t *c = (memif_connection_t *) conn;
   if (EXPECT_FALSE (c == NULL))
@@ -1581,6 +1676,8 @@ memif_refill_queue (memif_conn_handle_t conn, uint16_t qid, uint16_t count)
   libmemif_main_t *lm = &libmemif_main;
   memif_queue_t *mq = &c->rx_queues[qid];
   memif_ring_t *ring = mq->ring;
+  uint16_t mask = (1 << mq->log2_ring_size) - 1;
+  uint16_t slot;
 
   if (c->args.is_master)
     {
@@ -1588,16 +1685,32 @@ memif_refill_queue (memif_conn_handle_t conn, uint16_t qid, uint16_t count)
       ring->tail =
 	(ring->tail + count <=
 	 mq->last_head) ? ring->tail + count : mq->last_head;
+      return MEMIF_ERR_SUCCESS;
     }
-  else
+
+  uint16_t head = ring->head;
+  uint16_t ns = (1 << mq->log2_ring_size) - head + mq->last_tail;
+  head += (count < ns) ? count : ns;
+
+  if (headroom)
     {
-      uint16_t head = ring->head;
-      uint16_t ns = (1 << mq->log2_ring_size) - head + mq->last_tail;
-      head += ns;
-      MEMIF_MEMORY_BARRIER ();
-      ring->head = (ring->head + count <= head) ? ring->head + count : head;
+      slot = (c->args.is_master) ? ring->head : ring->tail;
+      while (slot < head)
+	{
+	  uint16_t x =
+	    (ring->desc[slot & mask].offset -
+	     c->regions->buffer_offset) / c->run_args.buffer_size;
+	  ring->desc[slot & mask].offset =
+	    c->regions->buffer_offset + (x * c->run_args.buffer_size) +
+	    headroom;
+
+	  slot++;
+	}
     }
 
+  MEMIF_MEMORY_BARRIER ();
+  ring->head = head;
+
   return MEMIF_ERR_SUCCESS;	/* 0 */
 }
 
@@ -1620,32 +1733,40 @@ memif_tx_burst (memif_conn_handle_t conn, uint16_t qid,
 
   memif_queue_t *mq = &c->tx_queues[qid];
   memif_ring_t *ring = mq->ring;
-  uint16_t slot;
-
-  slot = (c->args.is_master) ? ring->tail : ring->head;
-  *tx = (count <= mq->alloc_bufs) ? count : mq->alloc_bufs;
-
-#ifdef MEMIF_DBG_SHM
-  uint16_t curr_buf = 0;
   uint16_t mask = (1 << mq->log2_ring_size) - 1;
   memif_buffer_t *b0;
-  for (curr_buf = 0; curr_buf < count; curr_buf++)
+  *tx = 0;
+
+  if (count > mq->alloc_bufs)
+    count = mq->alloc_bufs;
+
+  if (EXPECT_FALSE (count == 0))
+    return MEMIF_ERR_SUCCESS;
+
+  while (count)
     {
-      b0 = (bufs + curr_buf);
+      b0 = (bufs + *tx);
+      ring->desc[b0->desc_index & mask].length = b0->len;
+
+#ifdef MEMIF_DBG_SHM
+      printf ("offset: %-6d\n", ring->desc[b0->desc_index & mask].offset);
+      printf ("data: %p\n",
+	      memif_get_buffer (c, ring, b0->desc_index & mask));
+      printf ("index: %u\n", b0->desc_index);
       print_bytes (memif_get_buffer (c, ring, b0->desc_index & mask),
 		   ring->desc[b0->desc_index & mask].length, DBG_TX_BUF);
+#endif /* MEMIF_DBG_SHM */
 
+      *tx += 1;
+      count--;
     }
-#endif /* MEMIF_DBG_SHM */
+
 
   MEMIF_MEMORY_BARRIER ();
   if (c->args.is_master)
-    ring->tail = slot + *tx;
+    ring->tail = b0->desc_index + 1;
   else
-    ring->head = slot + *tx;
-
-  /* zero out buffer fields so the client cant modify transmitted data */
-  memset (bufs, 0, sizeof (memif_buffer_t) * *tx);
+    ring->head = b0->desc_index + 1;
 
   mq->alloc_bufs -= *tx;
 
@@ -1687,7 +1808,7 @@ memif_rx_burst (memif_conn_handle_t conn, uint16_t qid,
 
   uint64_t b;
   ssize_t r = read (mq->int_fd, &b, sizeof (b));
-  if ((r == -1) && (errno != EAGAIN))
+  if (EXPECT_FALSE ((r == -1) && (errno != EAGAIN)))
     return memif_syscall_error_handler (errno);
 
   cur_slot = (c->args.is_master) ? mq->last_head : mq->last_tail;
@@ -1709,13 +1830,19 @@ memif_rx_burst (memif_conn_handle_t conn, uint16_t qid,
 	{
 	  ring->desc[cur_slot & mask].length = c->run_args.buffer_size;
 	}
+
+      b0->flags = MEMIF_BUFFER_FLAG_RX;
       if (ring->desc[cur_slot & mask].flags & MEMIF_DESC_FLAG_NEXT)
 	{
-	  b0->flags = MEMIF_BUFFER_FLAG_NEXT;
+	  b0->flags |= MEMIF_BUFFER_FLAG_NEXT;
 	  ring->desc[cur_slot & mask].flags &= ~MEMIF_DESC_FLAG_NEXT;
 	}
-
+/*      b0->offset = ring->desc[cur_slot & mask].offset;*/
+      b0->ring = ring;
 #ifdef MEMIF_DBG_SHM
+      printf ("data: %p\n", b0->data);
+      printf ("index: %u\n", b0->desc_index);
+      printf ("ring: %p\n", b0->ring);
       print_bytes (b0->data, b0->len, DBG_RX_BUF);
 #endif /* MEMIF_DBG_SHM */
       ns--;
diff --git a/extras/libmemif/src/memif_private.h b/extras/libmemif/src/memif_private.h
index accdac361b9..5ea5494decb 100644
--- a/extras/libmemif/src/memif_private.h
+++ b/extras/libmemif/src/memif_private.h
@@ -69,6 +69,7 @@ typedef struct
 {
   void *shm;
   uint32_t region_size;
+  uint32_t buffer_offset;
   int fd;
 } memif_region_t;
 
diff --git a/extras/libmemif/src/socket.c b/extras/libmemif/src/socket.c
index a471e0ba943..e568079ab25 100644
--- a/extras/libmemif/src/socket.c
+++ b/extras/libmemif/src/socket.c
@@ -590,9 +590,6 @@ memif_msg_receive_connect (memif_connection_t * c, memif_msg_t * msg)
 	  add_list_elt (&elt, &lm->interrupt_list, &lm->interrupt_list_len);
 
 	  lm->control_fd_update (c->rx_queues[i].int_fd, MEMIF_FD_EVENT_READ);
-
-	  /* refill ring buffers */
-	  memif_refill_queue ((void *) c, i, -1);
 	}
 
     }
@@ -623,9 +620,6 @@ memif_msg_receive_connected (memif_connection_t * c, memif_msg_t * msg)
       for (i = 0; i < c->run_args.num_s2m_rings; i++)
 	{
 	  lm->control_fd_update (c->rx_queues[i].int_fd, MEMIF_FD_EVENT_READ);
-
-	  /* refill ring buffers */
-	  memif_refill_queue ((void *) c, i, -1);
 	}
     }