vcl: refactor test client to support workers

Change-Id: I7fcfddc2bc7d9a64f8aa0d57ba5d11d325a15ce1
Signed-off-by: Florin Coras <fcoras@cisco.com>

7 files changed, 588 insertions, 639 deletions

diff --git a/src/vcl/sock_test_client.c b/src/vcl/sock_test_client.c
index 15616768cdd..de77fc9afe2 100644
--- a/src/vcl/sock_test_client.c
+++ b/src/vcl/sock_test_client.c
@@ -44,13 +44,13 @@ typedef struct
   uint8_t dump_cfg;
 } sock_client_main_t;
 
-sock_client_main_t sock_client_main;
+sock_client_main_t vcl_client_main;
 
 
 static int
 sock_test_cfg_sync (sock_test_socket_t * socket)
 {
-  sock_client_main_t *scm = &sock_client_main;
+  sock_client_main_t *scm = &vcl_client_main;
   sock_test_socket_t *ctrl = &scm->ctrl_socket;
   sock_test_cfg_t *rl_cfg = (sock_test_cfg_t *) socket->rxbuf;
   int rx_bytes, tx_bytes;
@@ -117,7 +117,7 @@ sock_test_cfg_sync (sock_test_socket_t * socket)
 static void
 echo_test_client ()
 {
-  sock_client_main_t *scm = &sock_client_main;
+  sock_client_main_t *scm = &vcl_client_main;
   sock_test_socket_t *ctrl = &scm->ctrl_socket;
   sock_test_socket_t *tsock;
   int rx_bytes, tx_bytes, nbytes;
@@ -361,7 +361,7 @@ echo_test_client ()
 static void
 stream_test_client (sock_test_t test)
 {
-  sock_client_main_t *scm = &sock_client_main;
+  sock_client_main_t *scm = &vcl_client_main;
   sock_test_socket_t *ctrl = &scm->ctrl_socket;
   sock_test_socket_t *tsock;
   int tx_bytes;
@@ -536,7 +536,7 @@ stream_test_client (sock_test_t test)
 static void
 exit_client (void)
 {
-  sock_client_main_t *scm = &sock_client_main;
+  sock_client_main_t *scm = &vcl_client_main;
   sock_test_socket_t *ctrl = &scm->ctrl_socket;
   sock_test_socket_t *tsock;
   int i;
@@ -578,7 +578,7 @@ exit_client (void)
 static int
 sock_test_connect_test_sockets (uint32_t num_test_sockets)
 {
-  sock_client_main_t *scm = &sock_client_main;
+  sock_client_main_t *scm = &vcl_client_main;
   sock_test_socket_t *ctrl = &scm->ctrl_socket;
   sock_test_socket_t *tsock;
   int i, rv, errno_val;
@@ -711,7 +711,7 @@ dump_help (void)
 static void
 cfg_txbuf_size_set (void)
 {
-  sock_client_main_t *scm = &sock_client_main;
+  sock_client_main_t *scm = &vcl_client_main;
   sock_test_socket_t *ctrl = &scm->ctrl_socket;
   char *p = ctrl->txbuf + strlen (SOCK_TEST_TOKEN_TXBUF_SIZE);
   uint64_t txbuf_size = strtoull ((const char *) p, NULL, 10);
@@ -733,7 +733,7 @@ cfg_txbuf_size_set (void)
 static void
 cfg_num_writes_set (void)
 {
-  sock_client_main_t *scm = &sock_client_main;
+  sock_client_main_t *scm = &vcl_client_main;
   sock_test_socket_t *ctrl = &scm->ctrl_socket;
   char *p = ctrl->txbuf + strlen (SOCK_TEST_TOKEN_NUM_WRITES);
   uint32_t num_writes = strtoul ((const char *) p, NULL, 10);
@@ -753,7 +753,7 @@ cfg_num_writes_set (void)
 static void
 cfg_num_test_sockets_set (void)
 {
-  sock_client_main_t *scm = &sock_client_main;
+  sock_client_main_t *scm = &vcl_client_main;
   sock_test_socket_t *ctrl = &scm->ctrl_socket;
   char *p = ctrl->txbuf + strlen (SOCK_TEST_TOKEN_NUM_TEST_SCKTS);
   uint32_t num_test_sockets = strtoul ((const char *) p, NULL, 10);
@@ -777,7 +777,7 @@ cfg_num_test_sockets_set (void)
 static void
 cfg_rxbuf_size_set (void)
 {
-  sock_client_main_t *scm = &sock_client_main;
+  sock_client_main_t *scm = &vcl_client_main;
   sock_test_socket_t *ctrl = &scm->ctrl_socket;
   char *p = ctrl->txbuf + strlen (SOCK_TEST_TOKEN_RXBUF_SIZE);
   uint64_t rxbuf_size = strtoull ((const char *) p, NULL, 10);
@@ -798,7 +798,7 @@ cfg_rxbuf_size_set (void)
 static void
 cfg_verbose_toggle (void)
 {
-  sock_client_main_t *scm = &sock_client_main;
+  sock_client_main_t *scm = &vcl_client_main;
   sock_test_socket_t *ctrl = &scm->ctrl_socket;
 
   ctrl->cfg.verbose = ctrl->cfg.verbose ? 0 : 1;
@@ -809,7 +809,7 @@ cfg_verbose_toggle (void)
 static sock_test_t
 parse_input ()
 {
-  sock_client_main_t *scm = &sock_client_main;
+  sock_client_main_t *scm = &vcl_client_main;
   sock_test_socket_t *ctrl = &scm->ctrl_socket;
   sock_test_t rv = SOCK_TEST_TYPE_NONE;
 
@@ -884,7 +884,7 @@ print_usage_and_exit (void)
 int
 main (int argc, char **argv)
 {
-  sock_client_main_t *scm = &sock_client_main;
+  sock_client_main_t *scm = &vcl_client_main;
   sock_test_socket_t *ctrl = &scm->ctrl_socket;
   int c, rv, errno_val;
   sock_test_t post_test = SOCK_TEST_TYPE_NONE;
diff --git a/src/vcl/vcl_bapi.c b/src/vcl/vcl_bapi.c
index 0c46d8220fd..d702d7c0736 100644
--- a/src/vcl/vcl_bapi.c
+++ b/src/vcl/vcl_bapi.c
@@ -312,7 +312,7 @@ vl_api_disconnect_session_reply_t_handler (vl_api_disconnect_session_reply_t *
 }
 
 static void
-vl_api_connect_sock_reply_t_handler (vl_api_connect_sock_reply_t * mp)
+vl_api_connect_session_reply_t_handler (vl_api_connect_sock_reply_t * mp)
 {
   if (mp->retval)
     clib_warning ("VCL<%d>: ERROR: sid %u: connect failed: %U",
@@ -324,7 +324,7 @@ vl_api_connect_sock_reply_t_handler (vl_api_connect_sock_reply_t * mp)
 _(SESSION_ENABLE_DISABLE_REPLY, session_enable_disable_reply)   	\
 _(BIND_SOCK_REPLY, bind_sock_reply)                             	\
 _(UNBIND_SOCK_REPLY, unbind_sock_reply)                         	\
-_(CONNECT_SOCK_REPLY, connect_sock_reply)                         	\
+_(CONNECT_SESSION_REPLY, connect_session_reply)                        	\
 _(DISCONNECT_SESSION_REPLY, disconnect_session_reply)			\
 _(APPLICATION_ATTACH_REPLY, application_attach_reply)           	\
 _(APPLICATION_DETACH_REPLY, application_detach_reply)           	\
@@ -429,6 +429,7 @@ vcl_send_app_worker_add_del (u8 is_add)
 
   mp->_vl_msg_id = ntohs (VL_API_APP_WORKER_ADD_DEL);
   mp->client_index = vcm->my_client_index;
+  mp->app_api_index = clib_host_to_net_u32 (vcm->my_client_index);
   mp->context = wrk_index;
   mp->is_add = is_add;
   if (!is_add)
diff --git a/src/vcl/vcl_debug.h b/src/vcl/vcl_debug.h
index 3faa45822f8..52a3d94b63f 100644
--- a/src/vcl/vcl_debug.h
+++ b/src/vcl/vcl_debug.h
@@ -22,7 +22,7 @@
 
 #define VDBG(_lvl, _fmt, _args...) 					\
   if (vcm->debug > _lvl)						\
-    clib_warning ("vcl<w%u>: " _fmt, __vcl_worker_index, ##_args)
+    clib_warning ("vcl<w%d>: " _fmt, __vcl_worker_index, ##_args)
 
 #define foreach_vcl_dbg_evt						\
   _(INIT, "vcl init track")						\
diff --git a/src/vcl/vcl_test.h b/src/vcl/vcl_test.h
index bdfb89c08a2..8808c0f5c5e 100644
--- a/src/vcl/vcl_test.h
+++ b/src/vcl/vcl_test.h
@@ -23,6 +23,29 @@
 #include <vcl/vppcom.h>
 #include <vcl/sock_test_common.h>
 
+#define vtfail(_fn, _rv)						\
+{									\
+  errno = -_rv;								\
+  perror ("ERROR when calling " _fn);					\
+  fprintf (stderr, "\nERROR: " _fn " failed (errno = %d)!\n", -_rv);	\
+  exit (1);								\
+}
+
+#define vterr(_fn, _rv)							\
+{									\
+  errno = -_rv;								\
+  fprintf (stderr, "\nERROR: " _fn " failed (errno = %d)!\n", -_rv);	\
+}
+
+#define vtwrn(_fmt, _args...)						\
+  fprintf (stderr, "\nERROR: " _fmt "\n", ##_args)			\
+
+#define vtinf(_fmt, _args...)						\
+  fprintf (stdout, "vt<w%u>: " _fmt "\n", __wrk_index, ##_args)
+
+#define vt_atomic_add(_ptr, _val) 					\
+  __atomic_fetch_add (_ptr, _val, __ATOMIC_RELEASE)
+
 static inline int
 vcl_test_read (int fd, uint8_t *buf, uint32_t nbytes,
                 sock_test_stats_t *stats)
diff --git a/src/vcl/vcl_test_client.c b/src/vcl/vcl_test_client.c
index 4e61045b055..b2431df80fa 100644
--- a/src/vcl/vcl_test_client.c
+++ b/src/vcl/vcl_test_client.c
@@ -23,523 +23,515 @@
 #include <time.h>
 #include <arpa/inet.h>
 #include <vcl/vcl_test.h>
+#include <pthread.h>
 
 typedef struct
 {
+  sock_test_socket_t *sessions;
+  uint32_t n_sessions;
+  uint32_t wrk_index;
+  fd_set wr_fdset;
+  fd_set rd_fdset;
+  int max_fd_index;
+  pthread_t thread_handle;
+  sock_test_cfg_t cfg;
+} vcl_test_client_worker_t;
+
+typedef struct
+{
+  vcl_test_client_worker_t *workers;
   vppcom_endpt_t server_endpt;
-  struct sockaddr_storage server_addr;
-  uint32_t server_addr_size;
   uint32_t cfg_seq_num;
   sock_test_socket_t ctrl_socket;
-  sock_test_socket_t *test_socket;
-  uint32_t num_test_sockets;
+  sock_test_socket_t *sessions;
   uint8_t dump_cfg;
-} sock_client_main_t;
+  sock_test_t post_test;
+  uint32_t proto;
+  uint32_t n_workers;
+  volatile int active_workers;
+  struct sockaddr_storage server_addr;
+} vcl_test_client_main_t;
+
+static __thread int __wrk_index = 0;
 
-sock_client_main_t sock_client_main;
+vcl_test_client_main_t vcl_client_main;
 
+#define vtc_min(a, b) (a < b ? a : b)
+#define vtc_max(a, b) (a > b ? a : b)
 
 static int
-sock_test_cfg_sync (sock_test_socket_t * socket)
+vtc_cfg_sync (sock_test_socket_t * sock)
 {
-  sock_client_main_t *scm = &sock_client_main;
-  sock_test_socket_t *ctrl = &scm->ctrl_socket;
-  sock_test_cfg_t *rl_cfg = (sock_test_cfg_t *) socket->rxbuf;
+  vcl_test_client_main_t *vcm = &vcl_client_main;
+  sock_test_cfg_t *rx_cfg = (sock_test_cfg_t *) sock->rxbuf;
   int rx_bytes, tx_bytes;
 
-  if (socket->cfg.verbose)
-    sock_test_cfg_dump (&socket->cfg, 1 /* is_client */ );
-
-  ctrl->cfg.seq_num = ++scm->cfg_seq_num;
-  if (socket->cfg.verbose)
+  vt_atomic_add (&sock->cfg.seq_num, 1);
+  if (sock->cfg.verbose)
     {
-      printf ("CLIENT (fd %d): Sending config sent to server.\n", socket->fd);
-      sock_test_cfg_dump (&ctrl->cfg, 1 /* is_client */ );
+      vtinf ("(fd %d): Sending config to server.", sock->fd);
+      sock_test_cfg_dump (&sock->cfg, 1 /* is_client */ );
     }
-  tx_bytes = vcl_test_write (socket->fd, (uint8_t *) & ctrl->cfg,
-			     sizeof (ctrl->cfg), NULL, ctrl->cfg.verbose);
+  tx_bytes = vcl_test_write (sock->fd, (uint8_t *) & sock->cfg,
+			     sizeof (sock->cfg), NULL, sock->cfg.verbose);
   if (tx_bytes < 0)
     {
-      fprintf (stderr, "CLIENT (fd %d): ERROR: write test cfg failed (%d)!\n",
-	       socket->fd, tx_bytes);
+      vtwrn ("(fd %d): write test cfg failed (%d)!", sock->fd, tx_bytes);
       return tx_bytes;
     }
 
-  rx_bytes = vcl_test_read (socket->fd, (uint8_t *) socket->rxbuf,
+  rx_bytes = vcl_test_read (sock->fd, (uint8_t *) sock->rxbuf,
 			    sizeof (sock_test_cfg_t), NULL);
   if (rx_bytes < 0)
     return rx_bytes;
 
-  if (rl_cfg->magic != SOCK_TEST_CFG_CTRL_MAGIC)
+  if (rx_cfg->magic != SOCK_TEST_CFG_CTRL_MAGIC)
     {
-      fprintf (stderr, "CLIENT (fd %d): ERROR: Bad server reply cfg "
-	       "-- aborting!\n", socket->fd);
+      vtwrn ("(fd %d): Bad server reply cfg -- aborting!", sock->fd);
       return -1;
     }
   if ((rx_bytes != sizeof (sock_test_cfg_t))
-      || !sock_test_cfg_verify (rl_cfg, &ctrl->cfg))
+      || !sock_test_cfg_verify (rx_cfg, &sock->cfg))
     {
-      fprintf (stderr, "CLIENT (fd %d): ERROR: Invalid config received "
-	       "from server!\n", socket->fd);
+      vtwrn ("(fd %d): Invalid config received from server!", sock->fd);
       if (rx_bytes != sizeof (sock_test_cfg_t))
 	{
-	  fprintf (stderr, "\tRx bytes %d != cfg size %lu\n",
-		   rx_bytes, sizeof (sock_test_cfg_t));
+	  vtinf ("\tRx bytes %d != cfg size %lu", rx_bytes,
+		 sizeof (sock_test_cfg_t));
 	}
       else
 	{
-	  sock_test_cfg_dump (rl_cfg, 1 /* is_client */ );
-	  fprintf (stderr, "CLIENT (fd %d): Valid config sent to server.\n",
-		   socket->fd);
-	  sock_test_cfg_dump (&ctrl->cfg, 1 /* is_client */ );
+	  sock_test_cfg_dump (rx_cfg, 1 /* is_client */ );
+	  vtinf ("(fd %d): Valid config sent to server.", sock->fd);
+	  sock_test_cfg_dump (&sock->cfg, 1 /* is_client */ );
 	}
       return -1;
     }
-  else if (socket->cfg.verbose)
+  if (sock->cfg.verbose)
     {
-      printf ("CLIENT (fd %d): Got config back from server.\n", socket->fd);
-      sock_test_cfg_dump (rl_cfg, 1 /* is_client */ );
+      vtinf ("(fd %d): Got config back from server.", sock->fd);
+      sock_test_cfg_dump (rx_cfg, 1 /* is_client */ );
     }
-  ctrl->cfg.ctrl_handle = ((ctrl->cfg.ctrl_handle == ~0) ?
-			   rl_cfg->ctrl_handle : ctrl->cfg.ctrl_handle);
 
   return 0;
 }
 
-static void
-echo_test_client ()
+static int
+vtc_connect_test_sessions (vcl_test_client_worker_t * wrk)
 {
-  sock_client_main_t *scm = &sock_client_main;
-  sock_test_socket_t *ctrl = &scm->ctrl_socket;
+  vcl_test_client_main_t *vcm = &vcl_client_main;
   sock_test_socket_t *tsock;
-  int rv, nfds = 0, rx_bytes, tx_bytes, nbytes;
-  uint32_t i, n, sidx;
-  fd_set wr_fdset, rd_fdset;
-  fd_set _wfdset, *wfdset = &_wfdset;
-  fd_set _rfdset, *rfdset = &_rfdset;
+  uint32_t n_test_sessions;
+  int i, rv;
 
-  FD_ZERO (&wr_fdset);
-  FD_ZERO (&rd_fdset);
-  memset (&ctrl->stats, 0, sizeof (ctrl->stats));
-  ctrl->cfg.total_bytes = nbytes = strlen (ctrl->txbuf) + 1;
-  for (n = 0; n != ctrl->cfg.num_test_sockets; n++)
+  n_test_sessions = wrk->cfg.num_test_sockets;
+  if (n_test_sessions < 1)
     {
-      tsock = &scm->test_socket[n];
-      tsock->cfg = ctrl->cfg;
-      sock_test_socket_buf_alloc (tsock);
-      if (sock_test_cfg_sync (tsock))
-	return;
+      errno = EINVAL;
+      return -1;
+    }
 
-      memcpy (tsock->txbuf, ctrl->txbuf, nbytes);
-      memset (&tsock->stats, 0, sizeof (tsock->stats));
+  if (wrk->n_sessions >= n_test_sessions)
+    goto done;
 
-      FD_SET (tsock->fd, &wr_fdset);
-      FD_SET (tsock->fd, &rd_fdset);
-      sidx = vppcom_session_index (tsock->fd);
-      nfds = sidx > nfds ? sidx : nfds;
-    }
+  if (wrk->n_sessions)
+    wrk->sessions = realloc (wrk->sessions,
+			     n_test_sessions * sizeof (sock_test_socket_t));
+  else
+    wrk->sessions = calloc (n_test_sessions, sizeof (sock_test_socket_t));
 
-  nfds++;
-  clock_gettime (CLOCK_REALTIME, &ctrl->stats.start);
-  while (n)
+  if (!wrk->sessions)
     {
-      _wfdset = wr_fdset;
-      _rfdset = rd_fdset;
+      vterr ("failed to alloc sessions", -errno);
+      return errno;
+    }
 
-      rv = vppcom_select (nfds, (uint64_t *) rfdset, (uint64_t *) wfdset,
-			  NULL, 0);
-      if (rv < 0)
+  for (i = 0; i < n_test_sessions; i++)
+    {
+      tsock = &wrk->sessions[i];
+      tsock->fd = vppcom_session_create (vcm->proto, 1 /* is_nonblocking */ );
+      if (tsock->fd < 0)
 	{
-	  perror ("select()");
-	  fprintf (stderr, "\nCLIENT: ERROR: select() failed -- "
-		   "aborting test!\n");
-	  return;
+	  vterr ("vppcom_session_create()", tsock->fd);
+	  return tsock->fd;
 	}
-      else if (rv == 0)
-	continue;
 
-      for (i = 0; i < ctrl->cfg.num_test_sockets; i++)
+      rv = vppcom_session_connect (tsock->fd, &vcm->server_endpt);
+      if (rv < 0)
 	{
-	  tsock = &scm->test_socket[i];
-	  if (!((tsock->stats.stop.tv_sec == 0) &&
-		(tsock->stats.stop.tv_nsec == 0)))
-	    continue;
-
-	  if (FD_ISSET (vppcom_session_index (tsock->fd), wfdset)
-	      && (tsock->stats.tx_bytes < ctrl->cfg.total_bytes))
-
-	    {
-	      tx_bytes = vcl_test_write (tsock->fd, (uint8_t *) tsock->txbuf,
-					 nbytes, &tsock->stats,
-					 ctrl->cfg.verbose);
-	      if (tx_bytes < 0)
-		{
-		  fprintf (stderr, "\nCLIENT: ERROR: vcl_test_write(%d) "
-			   "failed -- aborting test!\n", tsock->fd);
-		  return;
-		}
-
-	    }
-
-	  if ((FD_ISSET (vppcom_session_index (tsock->fd), rfdset)) &&
-	      (tsock->stats.rx_bytes < ctrl->cfg.total_bytes))
-	    {
-	      rx_bytes = vcl_test_read (tsock->fd, (uint8_t *) tsock->rxbuf,
-					nbytes, &tsock->stats);
-	      if (rx_bytes > 0)
-		{
-		  printf ("CLIENT (fd %d): RX (%d bytes) - '%s'\n",
-			  tsock->fd, rx_bytes, tsock->rxbuf);
-
-		  if (tsock->stats.rx_bytes != tsock->stats.tx_bytes)
-		    printf ("CLIENT: WARNING: bytes read (%lu) "
-			    "!= bytes written (%lu)!\n",
-			    tsock->stats.rx_bytes, tsock->stats.tx_bytes);
-		}
-	    }
-
-	  if (tsock->stats.rx_bytes >= ctrl->cfg.total_bytes)
-	    {
-	      clock_gettime (CLOCK_REALTIME, &tsock->stats.stop);
-	      n--;
-	    }
+	  vterr ("vppcom_session_connect()", rv);
+	  return rv;
 	}
+      vtinf ("(fd %d): Test socket %d connected.", tsock->fd, i);
     }
-  clock_gettime (CLOCK_REALTIME, &ctrl->stats.stop);
-
-  for (i = 0; i < ctrl->cfg.num_test_sockets; i++)
-    {
-      tsock = &scm->test_socket[i];
-      tsock->stats.start = ctrl->stats.start;
+  wrk->n_sessions = n_test_sessions;
 
-      if (ctrl->cfg.verbose)
-	{
-	  static char buf[64];
+done:
+  vtinf ("All test sessions (%d) connected!", n_test_sessions);
+  return 0;
+}
 
-	  sprintf (buf, "CLIENT (fd %d) RESULTS", tsock->fd);
-	  sock_test_stats_dump (buf, &tsock->stats,
-				1 /* show_rx */ , 1 /* show tx */ ,
-				ctrl->cfg.verbose);
-	}
+static int
+vtc_worker_test_setup (vcl_test_client_worker_t * wrk)
+{
+  vcl_test_client_main_t *vcm = &vcl_client_main;
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
+  sock_test_cfg_t *cfg = &wrk->cfg;
+  sock_test_socket_t *tsock;
+  uint32_t sidx;
+  int i, j;
 
-      sock_test_stats_accumulate (&ctrl->stats, &tsock->stats);
-    }
+  FD_ZERO (&wrk->wr_fdset);
+  FD_ZERO (&wrk->rd_fdset);
 
-  if (ctrl->cfg.verbose)
+  for (i = 0; i < cfg->num_test_sockets; i++)
     {
-      sock_test_stats_dump ("CLIENT RESULTS", &ctrl->stats,
-			    1 /* show_rx */ , 1 /* show tx */ ,
-			    ctrl->cfg.verbose);
-      sock_test_cfg_dump (&ctrl->cfg, 1 /* is_client */ );
+      tsock = &wrk->sessions[i];
+      tsock->cfg = wrk->cfg;
+      sock_test_socket_buf_alloc (tsock);
 
-      if (ctrl->cfg.verbose > 1)
+      switch (cfg->test)
 	{
-	  printf ("  ctrl socket info\n"
-		  SOCK_TEST_SEPARATOR_STRING
-		  "          fd:  %d (0x%08x)\n"
-		  "       rxbuf:  %p\n"
-		  "  rxbuf size:  %u (0x%08x)\n"
-		  "       txbuf:  %p\n"
-		  "  txbuf size:  %u (0x%08x)\n"
-		  SOCK_TEST_SEPARATOR_STRING,
-		  ctrl->fd, (uint32_t) ctrl->fd,
-		  ctrl->rxbuf, ctrl->rxbuf_size, ctrl->rxbuf_size,
-		  ctrl->txbuf, ctrl->txbuf_size, ctrl->txbuf_size);
+	case SOCK_TEST_TYPE_ECHO:
+	  memcpy (tsock->txbuf, ctrl->txbuf, cfg->total_bytes);
+	  break;
+	case SOCK_TEST_TYPE_UNI:
+	case SOCK_TEST_TYPE_BI:
+	  for (j = 0; j < tsock->txbuf_size; j++)
+	    tsock->txbuf[j] = j & 0xff;
+	  break;
 	}
+
+      FD_SET (vppcom_session_index (tsock->fd), &wrk->wr_fdset);
+      FD_SET (vppcom_session_index (tsock->fd), &wrk->rd_fdset);
+      sidx = vppcom_session_index (tsock->fd);
+      wrk->max_fd_index = vtc_max (sidx, wrk->max_fd_index);
     }
+  wrk->max_fd_index += 1;
+
+  return 0;
 }
 
-static void
-stream_test_client (sock_test_t test)
+static int
+vtc_worker_init (vcl_test_client_worker_t * wrk)
 {
-  sock_client_main_t *scm = &sock_client_main;
-  sock_test_socket_t *ctrl = &scm->ctrl_socket;
+  vcl_test_client_main_t *vcm = &vcl_client_main;
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
+  sock_test_cfg_t *cfg = &wrk->cfg;
   sock_test_socket_t *tsock;
-  int tx_bytes, rv, nfds = 0;
-  uint32_t i, n, sidx;
-  fd_set wr_fdset, rd_fdset;
-  fd_set _wfdset, *wfdset = &_wfdset;
-  fd_set _rfdset, *rfdset = (test == SOCK_TEST_TYPE_BI) ? &_rfdset : 0;
+  uint32_t i, n;
+  int rv, nbytes;
 
-  ctrl->cfg.total_bytes = ctrl->cfg.num_writes * ctrl->cfg.txbuf_size;
-  ctrl->cfg.ctrl_handle = ~0;
+  __wrk_index = wrk->wrk_index;
 
-  printf ("\n" SOCK_TEST_BANNER_STRING
-	  "CLIENT (fd %d): %s-directional Stream Test!\n\n"
-	  "CLIENT (fd %d): Sending config to server on ctrl socket...\n",
-	  ctrl->fd, test == SOCK_TEST_TYPE_BI ? "Bi" : "Uni", ctrl->fd);
+  vtinf ("Initializing worker ...");
 
-  if (sock_test_cfg_sync (ctrl))
+  if (wrk->wrk_index)
     {
-      fprintf (stderr, "CLIENT: ERROR: test cfg sync failed -- aborting!");
-      return;
-    }
+      if (vppcom_worker_register ())
+	{
+	  vtwrn ("failed to register worker");
+	  return -1;
+	}
 
-  FD_ZERO (&wr_fdset);
-  FD_ZERO (&rd_fdset);
-  memset (&ctrl->stats, 0, sizeof (ctrl->stats));
-  for (n = 0; n != ctrl->cfg.num_test_sockets; n++)
+      vt_atomic_add (&vcm->active_workers, 1);
+    }
+  rv = vtc_connect_test_sessions (wrk);
+  if (rv)
     {
-      tsock = &scm->test_socket[n];
-      tsock->cfg = ctrl->cfg;
-      sock_test_socket_buf_alloc (tsock);
-      printf ("CLIENT (fd %d): Sending config to server on "
-	      "test socket %d...\n", tsock->fd, n);
-      sock_test_cfg_sync (tsock);
+      vterr ("vtc_connect_test_sockets ()", rv);
+      return rv;
+    }
+
+  if (vtc_worker_test_setup (wrk))
+    return -1;
 
-      /* Fill payload with incrementing uint32's */
-      for (i = 0; i < tsock->txbuf_size; i++)
-	tsock->txbuf[i] = i & 0xff;
+  vtinf ("Sending config to server on all sessions ...");
 
+  for (n = 0; n < cfg->num_test_sockets; n++)
+    {
+      tsock = &wrk->sessions[n];
+      if (vtc_cfg_sync (tsock))
+	return -1;
       memset (&tsock->stats, 0, sizeof (tsock->stats));
-      FD_SET (tsock->fd, &wr_fdset);
-      FD_SET (tsock->fd, &rd_fdset);
-      sidx = vppcom_session_index (tsock->fd);
-      nfds = sidx > nfds ? sidx : nfds;
     }
 
-  nfds++;
-  clock_gettime (CLOCK_REALTIME, &ctrl->stats.start);
-  while (n)
+  return 0;
+}
+
+static void *
+vtc_worker_loop (void *arg)
+{
+  vcl_test_client_main_t *vcm = &vcl_client_main;
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
+  vcl_test_client_worker_t *wrk = arg;
+  fd_set _wfdset, *wfdset = &_wfdset;
+  fd_set _rfdset, *rfdset = &_rfdset;
+  sock_test_socket_t *tsock;
+  uint32_t n_active_sessions;
+  int i, rv, check_rx = 0;
+
+  rv = vtc_worker_init (wrk);
+  if (rv)
+    {
+      vterr ("vtc_worker_init()", rv);
+      return 0;
+    }
+
+  vtinf ("Starting test ...");
+
+  if (wrk->wrk_index == 0)
+    clock_gettime (CLOCK_REALTIME, &ctrl->stats.start);
+
+  check_rx = wrk->cfg.test != SOCK_TEST_TYPE_UNI;
+  n_active_sessions = wrk->cfg.num_test_sockets;
+  while (n_active_sessions)
     {
-      _wfdset = wr_fdset;
-      _rfdset = rd_fdset;
+      _wfdset = wrk->wr_fdset;
+      _rfdset = wrk->rd_fdset;
 
-      rv = vppcom_select (nfds, (uint64_t *) rfdset, (uint64_t *) wfdset,
-			  NULL, 0);
+      rv = vppcom_select (wrk->max_fd_index, (uint64_t *) rfdset,
+			  (uint64_t *) wfdset, NULL, 0);
       if (rv < 0)
 	{
-	  perror ("select()");
-	  fprintf (stderr, "\nCLIENT: ERROR: select() failed -- "
-		   "aborting test!\n");
-	  return;
+	  vterr ("vppcom_select()", rv);
+	  goto exit;
 	}
       else if (rv == 0)
 	continue;
 
-      for (i = 0; i < ctrl->cfg.num_test_sockets; i++)
+      for (i = 0; i < wrk->cfg.num_test_sockets; i++)
 	{
-	  tsock = &scm->test_socket[i];
+	  tsock = &wrk->sessions[i];
 	  if (!((tsock->stats.stop.tv_sec == 0) &&
 		(tsock->stats.stop.tv_nsec == 0)))
 	    continue;
 
-	  if ((test == SOCK_TEST_TYPE_BI) &&
-	      FD_ISSET (vppcom_session_index (tsock->fd), rfdset) &&
-	      (tsock->stats.rx_bytes < ctrl->cfg.total_bytes))
+	  if (FD_ISSET (vppcom_session_index (tsock->fd), rfdset)
+	      && tsock->stats.rx_bytes < tsock->cfg.total_bytes)
 	    {
-	      (void) vcl_test_read (tsock->fd,
-				    (uint8_t *) tsock->rxbuf,
+	      (void) vcl_test_read (tsock->fd, (uint8_t *) tsock->rxbuf,
 				    tsock->rxbuf_size, &tsock->stats);
 	    }
 
 	  if (FD_ISSET (vppcom_session_index (tsock->fd), wfdset)
-	      && (tsock->stats.tx_bytes < ctrl->cfg.total_bytes))
+	      && tsock->stats.tx_bytes < tsock->cfg.total_bytes)
 	    {
-	      tx_bytes = vcl_test_write (tsock->fd, (uint8_t *) tsock->txbuf,
-					 ctrl->cfg.txbuf_size, &tsock->stats,
-					 ctrl->cfg.verbose);
-	      if (tx_bytes < 0)
+	      rv = vcl_test_write (tsock->fd, (uint8_t *) tsock->txbuf,
+				   tsock->cfg.txbuf_size, &tsock->stats,
+				   tsock->cfg.verbose);
+	      if (rv < 0)
 		{
-		  fprintf (stderr, "\nCLIENT: ERROR: vcl_test_write(%d) "
-			   "failed -- aborting test!\n", tsock->fd);
-		  return;
+		  vtwrn ("vppcom_test_write (%d) failed -- aborting test",
+			 tsock->fd);
+		  goto exit;
 		}
 	    }
 
-	  if (((test == SOCK_TEST_TYPE_UNI) &&
-	       (tsock->stats.tx_bytes >= ctrl->cfg.total_bytes)) ||
-	      ((test == SOCK_TEST_TYPE_BI) &&
-	       (tsock->stats.rx_bytes >= ctrl->cfg.total_bytes)))
+	  if ((!check_rx && tsock->stats.tx_bytes >= tsock->cfg.total_bytes)
+	      || (check_rx
+		  && tsock->stats.rx_bytes >= tsock->cfg.total_bytes))
 	    {
 	      clock_gettime (CLOCK_REALTIME, &tsock->stats.stop);
-	      n--;
+	      n_active_sessions--;
 	    }
 	}
     }
-  clock_gettime (CLOCK_REALTIME, &ctrl->stats.stop);
+exit:
+  if (wrk->wrk_index)
+    vt_atomic_add (&vcm->active_workers, -1);
+  return 0;
+}
 
-  printf ("CLIENT (fd %d): Sending config to server on ctrl socket...\n",
-	  ctrl->fd);
+static void
+vtc_accumulate_stats (vcl_test_client_worker_t * wrk,
+		      sock_test_socket_t * ctrl)
+{
+  sock_test_socket_t *tsock;
+  static char buf[64];
+  int i, show_rx = 0;
 
-  if (sock_test_cfg_sync (ctrl))
-    {
-      fprintf (stderr, "CLIENT: ERROR: test cfg sync failed -- aborting!");
-      return;
-    }
+  if (ctrl->cfg.test == SOCK_TEST_TYPE_BI
+      || ctrl->cfg.test == SOCK_TEST_TYPE_ECHO)
+    show_rx = 1;
 
-  for (i = 0; i < ctrl->cfg.num_test_sockets; i++)
+  for (i = 0; i < wrk->cfg.num_test_sockets; i++)
     {
-      tsock = &scm->test_socket[i];
+      tsock = &wrk->sessions[i];
+      tsock->stats.start = ctrl->stats.start;
 
-      if (ctrl->cfg.verbose)
+      if (ctrl->cfg.verbose > 1)
 	{
-	  static char buf[64];
-
 	  sprintf (buf, "CLIENT (fd %d) RESULTS", tsock->fd);
-	  sock_test_stats_dump (buf, &tsock->stats,
-				test == SOCK_TEST_TYPE_BI /* show_rx */ ,
-				1 /* show tx */ , ctrl->cfg.verbose);
+	  sock_test_stats_dump (buf, &tsock->stats, show_rx, 1 /* show tx */ ,
+				ctrl->cfg.verbose);
 	}
 
       sock_test_stats_accumulate (&ctrl->stats, &tsock->stats);
     }
+}
+
+static void
+vtc_print_stats (sock_test_socket_t * ctrl)
+{
+  int is_echo = ctrl->cfg.test == SOCK_TEST_TYPE_ECHO;
+  int show_rx = 0;
+  char buf[64];
+
+  if (ctrl->cfg.test == SOCK_TEST_TYPE_BI
+      || ctrl->cfg.test == SOCK_TEST_TYPE_ECHO)
+    show_rx = 1;
 
   sock_test_stats_dump ("CLIENT RESULTS", &ctrl->stats,
-			test == SOCK_TEST_TYPE_BI /* show_rx */ ,
-			1 /* show tx */ , ctrl->cfg.verbose);
+			show_rx, 1 /* show tx */ ,
+			ctrl->cfg.verbose);
   sock_test_cfg_dump (&ctrl->cfg, 1 /* is_client */ );
 
   if (ctrl->cfg.verbose)
     {
-      printf ("  ctrl socket info\n"
-	      SOCK_TEST_SEPARATOR_STRING
-	      "          fd:  %d (0x%08x)\n"
-	      "       rxbuf:  %p\n"
-	      "  rxbuf size:  %u (0x%08x)\n"
-	      "       txbuf:  %p\n"
-	      "  txbuf size:  %u (0x%08x)\n"
-	      SOCK_TEST_SEPARATOR_STRING,
-	      ctrl->fd, (uint32_t) ctrl->fd,
-	      ctrl->rxbuf, ctrl->rxbuf_size, ctrl->rxbuf_size,
-	      ctrl->txbuf, ctrl->txbuf_size, ctrl->txbuf_size);
+      vtinf ("  ctrl socket info\n"
+	     SOCK_TEST_SEPARATOR_STRING
+	     "          fd:  %d (0x%08x)\n"
+	     "       rxbuf:  %p\n"
+	     "  rxbuf size:  %u (0x%08x)\n"
+	     "       txbuf:  %p\n"
+	     "  txbuf size:  %u (0x%08x)\n"
+	     SOCK_TEST_SEPARATOR_STRING,
+	     ctrl->fd, (uint32_t) ctrl->fd,
+	     ctrl->rxbuf, ctrl->rxbuf_size, ctrl->rxbuf_size,
+	     ctrl->txbuf, ctrl->txbuf_size, ctrl->txbuf_size);
     }
 
-  ctrl->cfg.test = SOCK_TEST_TYPE_ECHO;
-  if (sock_test_cfg_sync (ctrl))
-    fprintf (stderr, "CLIENT: ERROR: post-test cfg sync failed!");
+  if (is_echo)
+    sprintf (buf, "Echo");
+  else
+    sprintf (buf, "%s-directional Stream",
+	     ctrl->cfg.test == SOCK_TEST_TYPE_BI ? "Bi" : "Uni");
 
-  printf ("CLIENT (fd %d): %s-directional Stream Test Complete!\n"
-	  SOCK_TEST_BANNER_STRING "\n", ctrl->fd,
-	  test == SOCK_TEST_TYPE_BI ? "Bi" : "Uni");
+  vtinf ("(fd %d): %s Test Complete!\n"
+	 SOCK_TEST_BANNER_STRING, ctrl->fd, buf);
 }
 
 static void
-exit_client (void)
+vtc_echo_client (vcl_test_client_main_t * vcm)
 {
-  sock_client_main_t *scm = &sock_client_main;
-  sock_test_socket_t *ctrl = &scm->ctrl_socket;
-  sock_test_socket_t *tsock;
-  int i;
+  vcl_test_client_worker_t *wrk;
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
+  sock_test_cfg_t *cfg = &ctrl->cfg;
 
-  for (i = 0; i < ctrl->cfg.num_test_sockets; i++)
-    {
-      tsock = &scm->test_socket[i];
-      tsock->cfg.test = SOCK_TEST_TYPE_EXIT;
+  cfg->total_bytes = strlen (ctrl->txbuf) + 1;
+  memset (&ctrl->stats, 0, sizeof (ctrl->stats));
 
-      /* coverity[COPY_PASTE_ERROR] */
-      if (ctrl->cfg.verbose)
-	{
-	  printf ("\nCLIENT (fd %d): Sending exit cfg to server...\n",
-		  tsock->fd);
-	  sock_test_cfg_dump (&tsock->cfg, 1 /* is_client */ );
-	}
-      (void) vcl_test_write (tsock->fd, (uint8_t *) & tsock->cfg,
-			     sizeof (tsock->cfg), &tsock->stats,
-			     ctrl->cfg.verbose);
-    }
+  /* Echo works with only one worker */
+  wrk = vcm->workers;
+  wrk->wrk_index = 0;
+  wrk->cfg = *cfg;
 
-  ctrl->cfg.test = SOCK_TEST_TYPE_EXIT;
-  if (ctrl->cfg.verbose)
-    {
-      printf ("\nCLIENT (fd %d): Sending exit cfg to server...\n", ctrl->fd);
-      sock_test_cfg_dump (&ctrl->cfg, 1 /* is_client */ );
-    }
-  (void) vcl_test_write (ctrl->fd, (uint8_t *) & ctrl->cfg,
-			 sizeof (ctrl->cfg), &ctrl->stats, ctrl->cfg.verbose);
-  printf ("\nCLIENT: So long and thanks for all the fish!\n\n");
-  sleep (1);
+  vtc_worker_loop (wrk);
+
+  clock_gettime (CLOCK_REALTIME, &ctrl->stats.stop);
+  vtc_accumulate_stats (wrk, ctrl);
+  vtc_print_stats (ctrl);
 }
 
-static int
-sock_test_connect_test_sockets (uint32_t num_test_sockets)
+static void
+vtc_stream_client (vcl_test_client_main_t * vcm)
 {
-  sock_client_main_t *scm = &sock_client_main;
-  sock_test_socket_t *ctrl = &scm->ctrl_socket;
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
+  sock_test_cfg_t *cfg = &ctrl->cfg;
+  vcl_test_client_worker_t *wrk;
   sock_test_socket_t *tsock;
-  int i, rv, errno_val;
+  int tx_bytes, rv;
+  uint32_t i, n, sidx, n_conn, n_conn_per_wrk;
+
+  vtinf (SOCK_TEST_BANNER_STRING "%s-directional Stream Test!\n",
+	 ctrl->cfg.test == SOCK_TEST_TYPE_BI ? "Bi" : "Uni");
 
-  if (num_test_sockets < 1)
+  cfg->total_bytes = cfg->num_writes * cfg->txbuf_size;
+  cfg->ctrl_handle = ~0;
+  if (vtc_cfg_sync (ctrl))
     {
-      errno = EINVAL;
-      return -1;
+      vtwrn ("test cfg sync failed -- aborting!");
+      return;
     }
+  cfg->ctrl_handle = ((sock_test_cfg_t *) ctrl->rxbuf)->ctrl_handle;
+  memset (&ctrl->stats, 0, sizeof (ctrl->stats));
 
-  if (num_test_sockets < scm->num_test_sockets)
+  n_conn = cfg->num_test_sockets;
+  n_conn_per_wrk = n_conn / vcm->n_workers;
+  for (i = 0; i < vcm->n_workers; i++)
     {
-      for (i = scm->num_test_sockets - 1; i >= num_test_sockets; i--)
-	{
-	  tsock = &scm->test_socket[i];
-	  vppcom_session_close (tsock->fd);
-	  free (tsock->txbuf);
-	  free (tsock->rxbuf);
-	}
+      wrk = &vcm->workers[i];
+      wrk->wrk_index = i;
+      wrk->cfg = ctrl->cfg;
+      wrk->cfg.num_test_sockets = vtc_min (n_conn_per_wrk, n_conn);
+      n_conn -= wrk->cfg.num_test_sockets;
     }
 
-  else if (num_test_sockets > scm->num_test_sockets)
+  for (i = 1; i < vcm->n_workers; i++)
     {
-      tsock = realloc (scm->test_socket,
-		       sizeof (sock_test_socket_t) * num_test_sockets);
-      if (!tsock)
-	{
-	  errno_val = errno;
-	  perror ("ERROR in sock_test_connect_test_sockets()");
-	  fprintf (stderr, "CLIENT: ERROR: socket failed (errno = %d)!\n",
-		   errno_val);
-	  return -1;
-	}
+      wrk = &vcm->workers[i];
+      pthread_create (&wrk->thread_handle, NULL, vtc_worker_loop,
+		      (void *) wrk);
+    }
+  vtc_worker_loop (&vcm->workers[0]);
 
-      if (!scm->test_socket)
-	memset (tsock, 0, sizeof (*tsock));
+  while (vcm->active_workers > 0)
+    ;
 
-      scm->test_socket = tsock;
-      for (i = scm->num_test_sockets; i < num_test_sockets; i++)
-	{
-	  tsock = &scm->test_socket[i];
-	  tsock->fd = vppcom_session_create (ctrl->cfg.transport_udp ?
-					     VPPCOM_PROTO_UDP :
-					     VPPCOM_PROTO_TCP,
-					     1 /* is_nonblocking */ );
-	  if (tsock->fd < 0)
-	    {
-	      errno = -tsock->fd;
-	      tsock->fd = -1;
-	    }
-	  if (tsock->fd < 0)
-	    {
-	      errno_val = errno;
-	      perror ("ERROR in sock_test_connect_test_sockets()");
-	      fprintf (stderr, "CLIENT: ERROR: socket failed (errno = %d)!\n",
-		       errno_val);
-	      return tsock->fd;
-	    }
+  clock_gettime (CLOCK_REALTIME, &ctrl->stats.stop);
 
-	  rv = vppcom_session_connect (tsock->fd, &scm->server_endpt);
-	  if (rv)
-	    {
-	      errno = -rv;
-	      rv = -1;
-	    }
-	  if (rv < 0)
-	    {
-	      errno_val = errno;
-	      perror ("ERROR in sock_test_connect_test_sockets()");
-	      fprintf (stderr, "CLIENT: ERROR: connect failed "
-		       "(errno = %d)!\n", errno_val);
-	      return -1;
-	    }
-	  tsock->cfg = ctrl->cfg;
-	  sock_test_socket_buf_alloc (tsock);
-	  sock_test_cfg_sync (tsock);
+  vtinf ("(fd %d): Sending config on ctrl socket for stats...\n", ctrl->fd);
+  if (vtc_cfg_sync (ctrl))
+    {
+      vtwrn ("test cfg sync failed -- aborting!");
+      return;
+    }
+
+  for (i = 0; i < vcm->n_workers; i++)
+    vtc_accumulate_stats (&vcm->workers[i], ctrl);
+  vtc_print_stats (ctrl);
 
-	  printf ("CLIENT (fd %d): Test socket %d connected.\n",
-		  tsock->fd, i);
+  ctrl->cfg.test = SOCK_TEST_TYPE_ECHO;
+  ctrl->cfg.total_bytes = 0;
+  if (vtc_cfg_sync (ctrl))
+    vtwrn ("post-test cfg sync failed!");
+}
+
+static void
+vtc_client_exit (void)
+{
+  vcl_test_client_main_t *vcm = &vcl_client_main;
+  vcl_test_client_worker_t *wrk = &vcm->workers[0];
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
+  sock_test_socket_t *tsock;
+  int i, verbose;
+
+  verbose = ctrl->cfg.verbose;
+  for (i = 0; i < wrk->cfg.num_test_sockets; i++)
+    {
+      tsock = &wrk->sessions[i];
+      tsock->cfg.test = SOCK_TEST_TYPE_EXIT;
+
+      if (verbose)
+	{
+	  vtinf ("(fd %d): Sending exit cfg to server...", tsock->fd);
+	  sock_test_cfg_dump (&tsock->cfg, 1 /* is_client */ );
 	}
+      (void) vcl_test_write (tsock->fd, (uint8_t *) & tsock->cfg,
+			     sizeof (tsock->cfg), &tsock->stats, verbose);
     }
 
-  scm->num_test_sockets = num_test_sockets;
-  printf ("CLIENT: All sockets (%d) connected!\n", scm->num_test_sockets + 1);
-  return 0;
+  ctrl->cfg.test = SOCK_TEST_TYPE_EXIT;
+  if (verbose)
+    {
+      vtinf ("(fd %d): Sending exit cfg to server...", ctrl->fd);
+      sock_test_cfg_dump (&ctrl->cfg, 1 /* is_client */ );
+    }
+  (void) vcl_test_write (ctrl->fd, (uint8_t *) & ctrl->cfg,
+			 sizeof (ctrl->cfg), &ctrl->stats, verbose);
+  vtinf ("So long and thanks for all the fish!\n\n");
+  sleep (1);
 }
 
 static void
@@ -571,8 +563,8 @@ dump_help (void)
 static void
 cfg_txbuf_size_set (void)
 {
-  sock_client_main_t *scm = &sock_client_main;
-  sock_test_socket_t *ctrl = &scm->ctrl_socket;
+  vcl_test_client_main_t *vcm = &vcl_client_main;
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
   char *p = ctrl->txbuf + strlen (SOCK_TEST_TOKEN_TXBUF_SIZE);
   uint64_t txbuf_size = strtoull ((const char *) p, NULL, 10);
 
@@ -593,8 +585,8 @@ cfg_txbuf_size_set (void)
 static void
 cfg_num_writes_set (void)
 {
-  sock_client_main_t *scm = &sock_client_main;
-  sock_test_socket_t *ctrl = &scm->ctrl_socket;
+  vcl_test_client_main_t *vcm = &vcl_client_main;
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
   char *p = ctrl->txbuf + strlen (SOCK_TEST_TOKEN_NUM_WRITES);
   uint32_t num_writes = strtoul ((const char *) p, NULL, 10);
 
@@ -613,8 +605,8 @@ cfg_num_writes_set (void)
 static void
 cfg_num_test_sockets_set (void)
 {
-  sock_client_main_t *scm = &sock_client_main;
-  sock_test_socket_t *ctrl = &scm->ctrl_socket;
+  vcl_test_client_main_t *vcm = &vcl_client_main;
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
   char *p = ctrl->txbuf + strlen (SOCK_TEST_TOKEN_NUM_TEST_SCKTS);
   uint32_t num_test_sockets = strtoul ((const char *) p, NULL, 10);
 
@@ -622,23 +614,20 @@ cfg_num_test_sockets_set (void)
       (num_test_sockets <= SOCK_TEST_CFG_MAX_TEST_SCKTS))
     {
       ctrl->cfg.num_test_sockets = num_test_sockets;
-      sock_test_connect_test_sockets (num_test_sockets);
-
       sock_test_cfg_dump (&ctrl->cfg, 1 /* is_client */ );
     }
   else
     {
-      fprintf (stderr, "CLIENT: ERROR: invalid num test sockets: "
-	       "%u, (%d max)\n",
-	       num_test_sockets, SOCK_TEST_CFG_MAX_TEST_SCKTS);
+      vtwrn ("invalid num test sockets: %u, (%d max)",
+	     num_test_sockets, SOCK_TEST_CFG_MAX_TEST_SCKTS);
     }
 }
 
 static void
 cfg_rxbuf_size_set (void)
 {
-  sock_client_main_t *scm = &sock_client_main;
-  sock_test_socket_t *ctrl = &scm->ctrl_socket;
+  vcl_test_client_main_t *vcm = &vcl_client_main;
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
   char *p = ctrl->txbuf + strlen (SOCK_TEST_TOKEN_RXBUF_SIZE);
   uint64_t rxbuf_size = strtoull ((const char *) p, NULL, 10);
 
@@ -658,8 +647,8 @@ cfg_rxbuf_size_set (void)
 static void
 cfg_verbose_toggle (void)
 {
-  sock_client_main_t *scm = &sock_client_main;
-  sock_test_socket_t *ctrl = &scm->ctrl_socket;
+  vcl_test_client_main_t *vcm = &vcl_client_main;
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
 
   ctrl->cfg.verbose = ctrl->cfg.verbose ? 0 : 1;
   sock_test_cfg_dump (&ctrl->cfg, 1 /* is_client */ );
@@ -669,8 +658,8 @@ cfg_verbose_toggle (void)
 static sock_test_t
 parse_input ()
 {
-  sock_client_main_t *scm = &sock_client_main;
-  sock_test_socket_t *ctrl = &scm->ctrl_socket;
+  vcl_test_client_main_t *vcm = &vcl_client_main;
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
   sock_test_t rv = SOCK_TEST_TYPE_NONE;
 
   if (!strncmp (SOCK_TEST_TOKEN_EXIT, ctrl->txbuf,
@@ -683,7 +672,7 @@ parse_input ()
 
   else if (!strncmp (SOCK_TEST_TOKEN_SHOW_CFG, ctrl->txbuf,
 		     strlen (SOCK_TEST_TOKEN_SHOW_CFG)))
-    scm->dump_cfg = 1;
+    vcm->dump_cfg = 1;
 
   else if (!strncmp (SOCK_TEST_TOKEN_VERBOSE, ctrl->txbuf,
 		     strlen (SOCK_TEST_TOKEN_VERBOSE)))
@@ -741,58 +730,56 @@ print_usage_and_exit (void)
   exit (1);
 }
 
-int
-main (int argc, char **argv)
+static void
+vtc_process_opts (vcl_test_client_main_t * vcm, int argc, char **argv)
 {
-  sock_client_main_t *scm = &sock_client_main;
-  sock_test_socket_t *ctrl = &scm->ctrl_socket;
-  int c, rv, errno_val;
-  sock_test_t post_test = SOCK_TEST_TYPE_NONE;
-
-  sock_test_cfg_init (&ctrl->cfg);
-  sock_test_socket_buf_alloc (ctrl);
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
+  int c, v;
 
   opterr = 0;
   while ((c = getopt (argc, argv, "chn:w:XE:I:N:R:T:UBV6D")) != -1)
     switch (c)
       {
       case 'c':
-	scm->dump_cfg = 1;
+	vcm->dump_cfg = 1;
 	break;
 
       case 's':
 	if (sscanf (optarg, "0x%x", &ctrl->cfg.num_test_sockets) != 1)
 	  if (sscanf (optarg, "%u", &ctrl->cfg.num_test_sockets) != 1)
 	    {
-	      fprintf (stderr, "CLIENT: ERROR: Invalid value for "
-		       "option -%c!\n", c);
+	      vtwrn ("Invalid value for option -%c!", c);
 	      print_usage_and_exit ();
 	    }
 	if (!ctrl->cfg.num_test_sockets ||
 	    (ctrl->cfg.num_test_sockets > FD_SETSIZE))
 	  {
-	    fprintf (stderr, "CLIENT: ERROR: Invalid number of "
-		     "sockets (%d) specified for option -%c!\n"
-		     "       Valid range is 1 - %d\n",
-		     ctrl->cfg.num_test_sockets, c, FD_SETSIZE);
+	    vtwrn ("Invalid number of sockets (%d) specified for option -%c!"
+		   "\n       Valid range is 1 - %d",
+		   ctrl->cfg.num_test_sockets, c, FD_SETSIZE);
 	    print_usage_and_exit ();
 	  }
 	break;
 
       case 'w':
-	fprintf (stderr, "CLIENT: Writing test results to files is TBD.\n");
+	if (sscanf (optarg, "%d", &v) != 1)
+	  {
+	    vtwrn ("Invalid value for option -%c!", c);
+	    print_usage_and_exit ();
+	  }
+	if (v > 1)
+	  vcm->n_workers = v;
 	break;
 
       case 'X':
-	post_test = SOCK_TEST_TYPE_EXIT;
+	vcm->post_test = SOCK_TEST_TYPE_EXIT;
 	break;
 
       case 'E':
 	if (strlen (optarg) > ctrl->txbuf_size)
 	  {
-	    fprintf (stderr, "CLIENT: ERROR: Option -%c value "
-		     "larger than txbuf size (%d)!\n",
-		     optopt, ctrl->txbuf_size);
+	    vtwrn ("Option -%c value larger than txbuf size (%d)!",
+		   optopt, ctrl->txbuf_size);
 	    print_usage_and_exit ();
 	  }
 	strcpy (ctrl->txbuf, optarg);
@@ -803,15 +790,13 @@ main (int argc, char **argv)
 	if (sscanf (optarg, "0x%x", &ctrl->cfg.num_test_sockets) != 1)
 	  if (sscanf (optarg, "%d", &ctrl->cfg.num_test_sockets) != 1)
 	    {
-	      fprintf (stderr, "CLIENT: ERROR: Invalid value for "
-		       "option -%c!\n", c);
+	      vtwrn ("Invalid value for option -%c!", c);
 	      print_usage_and_exit ();
 	    }
 	if (ctrl->cfg.num_test_sockets > SOCK_TEST_CFG_MAX_TEST_SCKTS)
 	  {
-	    fprintf (stderr, "CLIENT: ERROR: value greater than max "
-		     "number test sockets (%d)!",
-		     SOCK_TEST_CFG_MAX_TEST_SCKTS);
+	    vtwrn ("value greater than max number test sockets (%d)!",
+		   SOCK_TEST_CFG_MAX_TEST_SCKTS);
 	    print_usage_and_exit ();
 	  }
 	break;
@@ -820,8 +805,7 @@ main (int argc, char **argv)
 	if (sscanf (optarg, "0x%lx", &ctrl->cfg.num_writes) != 1)
 	  if (sscanf (optarg, "%ld", &ctrl->cfg.num_writes) != 1)
 	    {
-	      fprintf (stderr, "CLIENT: ERROR: Invalid value for "
-		       "option -%c!\n", c);
+	      vtwrn ("Invalid value for option -%c!", c);
 	      print_usage_and_exit ();
 	    }
 	ctrl->cfg.total_bytes = ctrl->cfg.num_writes * ctrl->cfg.txbuf_size;
@@ -831,8 +815,7 @@ main (int argc, char **argv)
 	if (sscanf (optarg, "0x%lx", &ctrl->cfg.rxbuf_size) != 1)
 	  if (sscanf (optarg, "%ld", &ctrl->cfg.rxbuf_size) != 1)
 	    {
-	      fprintf (stderr, "CLIENT: ERROR: Invalid value for "
-		       "option -%c!\n", c);
+	      vtwrn ("Invalid value for option -%c!", c);
 	      print_usage_and_exit ();
 	    }
 	if (ctrl->cfg.rxbuf_size >= SOCK_TEST_CFG_BUF_SIZE_MIN)
@@ -844,9 +827,8 @@ main (int argc, char **argv)
 	  }
 	else
 	  {
-	    fprintf (stderr, "CLIENT: ERROR: rxbuf size (%lu) "
-		     "less than minumum (%u)\n",
-		     ctrl->cfg.rxbuf_size, SOCK_TEST_CFG_BUF_SIZE_MIN);
+	    vtwrn ("rxbuf size (%lu) less than minumum (%u)",
+		   ctrl->cfg.rxbuf_size, SOCK_TEST_CFG_BUF_SIZE_MIN);
 	    print_usage_and_exit ();
 	  }
 
@@ -856,8 +838,7 @@ main (int argc, char **argv)
 	if (sscanf (optarg, "0x%lx", &ctrl->cfg.txbuf_size) != 1)
 	  if (sscanf (optarg, "%ld", &ctrl->cfg.txbuf_size) != 1)
 	    {
-	      fprintf (stderr, "CLIENT: ERROR: Invalid value "
-		       "for option -%c!\n", c);
+	      vtwrn ("Invalid value for option -%c!", c);
 	      print_usage_and_exit ();
 	    }
 	if (ctrl->cfg.txbuf_size >= SOCK_TEST_CFG_BUF_SIZE_MIN)
@@ -871,9 +852,8 @@ main (int argc, char **argv)
 	  }
 	else
 	  {
-	    fprintf (stderr, "CLIENT: ERROR: txbuf size (%lu) "
-		     "less than minumum (%u)!\n",
-		     ctrl->cfg.txbuf_size, SOCK_TEST_CFG_BUF_SIZE_MIN);
+	    vtwrn ("txbuf size (%lu) less than minumum (%u)!",
+		   ctrl->cfg.txbuf_size, SOCK_TEST_CFG_BUF_SIZE_MIN);
 	    print_usage_and_exit ();
 	  }
 	break;
@@ -907,17 +887,14 @@ main (int argc, char **argv)
 	  case 'R':
 	  case 'T':
 	  case 'w':
-	    fprintf (stderr, "CLIENT: ERROR: Option -%c "
-		     "requires an argument.\n", optopt);
+	    vtwrn ("Option -%c requires an argument.", optopt);
 	    break;
 
 	  default:
 	    if (isprint (optopt))
-	      fprintf (stderr, "CLIENT: ERROR: Unknown "
-		       "option `-%c'.\n", optopt);
+	      vtwrn ("Unknown option `-%c'.", optopt);
 	    else
-	      fprintf (stderr, "CLIENT: ERROR: Unknown "
-		       "option character `\\x%x'.\n", optopt);
+	      vtwrn ("Unknown option character `\\x%x'.", optopt);
 	  }
 	/* fall thru */
       case 'h':
@@ -927,118 +904,107 @@ main (int argc, char **argv)
 
   if (argc < (optind + 2))
     {
-      fprintf (stderr, "CLIENT: ERROR: Insufficient number of arguments!\n");
+      vtwrn ("Insufficient number of arguments!");
       print_usage_and_exit ();
     }
+  vcm->proto = ctrl->cfg.transport_udp ? VPPCOM_PROTO_UDP : VPPCOM_PROTO_TCP;
 
-  ctrl->fd = vppcom_app_create ("vcl_test_client");
-  if (ctrl->fd < 0)
-    {
-      errno = -ctrl->fd;
-      ctrl->fd = -1;
-    }
-  else
-    {
-      ctrl->fd = vppcom_session_create (ctrl->cfg.transport_udp ?
-					VPPCOM_PROTO_UDP :
-					VPPCOM_PROTO_TCP,
-					0 /* is_nonblocking */ );
-      if (ctrl->fd < 0)
-	{
-	  errno = -ctrl->fd;
-	  ctrl->fd = -1;
-	}
-    }
-
-  if (ctrl->fd < 0)
-    {
-      errno_val = errno;
-      perror ("ERROR in main()");
-      fprintf (stderr, "CLIENT: ERROR: socket "
-	       "failed (errno = %d)!\n", errno_val);
-      return ctrl->fd;
-    }
-
-  memset (&scm->server_addr, 0, sizeof (scm->server_addr));
+  memset (&vcm->server_addr, 0, sizeof (vcm->server_addr));
   if (ctrl->cfg.address_ip6)
     {
-      struct sockaddr_in6 *server_addr =
-	(struct sockaddr_in6 *) &scm->server_addr;
-      scm->server_addr_size = sizeof (*server_addr);
-      server_addr->sin6_family = AF_INET6;
-      inet_pton (AF_INET6, argv[optind++], &(server_addr->sin6_addr));
-      server_addr->sin6_port = htons (atoi (argv[optind]));
+      struct sockaddr_in6 *sddr6 = (struct sockaddr_in6 *) &vcm->server_addr;
+      sddr6->sin6_family = AF_INET6;
+      inet_pton (AF_INET6, argv[optind++], &(sddr6->sin6_addr));
+      sddr6->sin6_port = htons (atoi (argv[optind]));
+
+      vcm->server_endpt.is_ip4 = 0;
+      vcm->server_endpt.ip = (uint8_t *) & sddr6->sin6_addr;
+      vcm->server_endpt.port = (uint16_t) sddr6->sin6_port;
     }
   else
     {
-      struct sockaddr_in *server_addr =
-	(struct sockaddr_in *) &scm->server_addr;
-      scm->server_addr_size = sizeof (*server_addr);
-      server_addr->sin_family = AF_INET;
-      inet_pton (AF_INET, argv[optind++], &(server_addr->sin_addr));
-      server_addr->sin_port = htons (atoi (argv[optind]));
+      struct sockaddr_in *saddr4 = (struct sockaddr_in *) &vcm->server_addr;
+      saddr4->sin_family = AF_INET;
+      inet_pton (AF_INET, argv[optind++], &(saddr4->sin_addr));
+      saddr4->sin_port = htons (atoi (argv[optind]));
+
+      vcm->server_endpt.is_ip4 = 1;
+      vcm->server_endpt.ip = (uint8_t *) & saddr4->sin_addr;
+      vcm->server_endpt.port = (uint16_t) saddr4->sin_port;
     }
+}
 
-  if (ctrl->cfg.address_ip6)
-    {
-      struct sockaddr_in6 *server_addr =
-	(struct sockaddr_in6 *) &scm->server_addr;
-      scm->server_endpt.is_ip4 = 0;
-      scm->server_endpt.ip = (uint8_t *) & server_addr->sin6_addr;
-      scm->server_endpt.port = (uint16_t) server_addr->sin6_port;
-    }
-  else
-    {
-      struct sockaddr_in *server_addr =
-	(struct sockaddr_in *) &scm->server_addr;
-      scm->server_endpt.is_ip4 = 1;
-      scm->server_endpt.ip = (uint8_t *) & server_addr->sin_addr;
-      scm->server_endpt.port = (uint16_t) server_addr->sin_port;
-    }
+static void
+vtc_read_user_input (sock_test_socket_t * ctrl)
+{
+  printf ("\nType some characters and hit <return>\n"
+	  "('" SOCK_TEST_TOKEN_HELP "' for help): ");
 
-  do
+  if (fgets (ctrl->txbuf, ctrl->txbuf_size, stdin) != NULL)
     {
-      printf ("\nCLIENT: Connecting to server...\n");
-
-      rv = vppcom_session_connect (ctrl->fd, &scm->server_endpt);
-      if (rv)
-	{
-	  errno = -rv;
-	  rv = -1;
-	}
-      if (rv < 0)
+      if (strlen (ctrl->txbuf) == 1)
 	{
-	  errno_val = errno;
-	  perror ("ERROR in main()");
-	  fprintf (stderr, "CLIENT: ERROR: connect failed (errno = %d)!\n",
-		   errno_val);
-	  return -1;
+	  printf ("\nNothing to send!  Please try again...\n");
+	  return;
 	}
+      ctrl->txbuf[strlen (ctrl->txbuf) - 1] = 0;	// chomp the newline.
 
-      sock_test_cfg_sync (ctrl);
-      printf ("CLIENT (fd %d): Control socket connected.\n", ctrl->fd);
+      /* Parse input for keywords */
+      ctrl->cfg.test = parse_input ();
     }
-  while (rv < 0);
+}
 
-  sock_test_connect_test_sockets (ctrl->cfg.num_test_sockets);
+int
+main (int argc, char **argv)
+{
+  vcl_test_client_main_t *vcm = &vcl_client_main;
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
+  int rv, errno_val;
+
+  vcm->n_workers = 1;
+  sock_test_cfg_init (&ctrl->cfg);
+  sock_test_socket_buf_alloc (ctrl);
+  vtc_process_opts (vcm, argc, argv);
+
+  vcm->workers = calloc (vcm->n_workers, sizeof (vcl_test_client_worker_t));
+  rv = vppcom_app_create ("vcl_test_client");
+  if (rv < 0)
+    vtfail ("vppcom_app_create()", rv);
+
+  ctrl->fd = vppcom_session_create (vcm->proto, 0 /* is_nonblocking */ );
+  if (ctrl->fd < 0)
+    vtfail ("vppcom_session_create()", ctrl->fd);
+
+  vtinf ("Connecting to server...");
+  rv = vppcom_session_connect (ctrl->fd, &vcm->server_endpt);
+  if (rv)
+    vtfail ("vppcom_session_connect()", rv);
+  vtinf ("(fd %d): Control socket connected.", ctrl->fd);
+
+  rv = vtc_cfg_sync (ctrl);
+  if (rv)
+    vtfail ("vtc_cfg_sync()", rv);
+
+  ctrl->cfg.ctrl_handle = ((sock_test_cfg_t *) ctrl->rxbuf)->ctrl_handle;
+  memset (&ctrl->stats, 0, sizeof (ctrl->stats));
 
   while (ctrl->cfg.test != SOCK_TEST_TYPE_EXIT)
     {
-      if (scm->dump_cfg)
+      if (vcm->dump_cfg)
 	{
 	  sock_test_cfg_dump (&ctrl->cfg, 1 /* is_client */ );
-	  scm->dump_cfg = 0;
+	  vcm->dump_cfg = 0;
 	}
 
       switch (ctrl->cfg.test)
 	{
 	case SOCK_TEST_TYPE_ECHO:
-	  echo_test_client ();
+	  vtc_echo_client (vcm);
 	  break;
 
 	case SOCK_TEST_TYPE_UNI:
 	case SOCK_TEST_TYPE_BI:
-	  stream_test_client (ctrl->cfg.test);
+	  vtc_stream_client (vcm);
 	  break;
 
 	case SOCK_TEST_TYPE_EXIT:
@@ -1048,7 +1014,7 @@ main (int argc, char **argv)
 	default:
 	  break;
 	}
-      switch (post_test)
+      switch (vcm->post_test)
 	{
 	case SOCK_TEST_TYPE_EXIT:
 	  switch (ctrl->cfg.test)
@@ -1077,26 +1043,13 @@ main (int argc, char **argv)
       memset (ctrl->txbuf, 0, ctrl->txbuf_size);
       memset (ctrl->rxbuf, 0, ctrl->rxbuf_size);
 
-      printf ("\nCLIENT: Type some characters and hit <return>\n"
-	      "('" SOCK_TEST_TOKEN_HELP "' for help): ");
-
-      if (fgets (ctrl->txbuf, ctrl->txbuf_size, stdin) != NULL)
-	{
-	  if (strlen (ctrl->txbuf) == 1)
-	    {
-	      printf ("\nCLIENT: Nothing to send!  Please try again...\n");
-	      continue;
-	    }
-	  ctrl->txbuf[strlen (ctrl->txbuf) - 1] = 0;	// chomp the newline.
-
-	  /* Parse input for keywords */
-	  ctrl->cfg.test = parse_input ();
-	}
+      vtc_read_user_input (ctrl);
     }
 
-  exit_client ();
+  vtc_client_exit ();
   vppcom_session_close (ctrl->fd);
   vppcom_app_destroy ();
+  free (vcm->workers);
   return 0;
 }
 
diff --git a/src/vcl/vcl_test_server.c b/src/vcl/vcl_test_server.c
index d1f2db72872..8d97d0cfaa6 100644
--- a/src/vcl/vcl_test_server.c
+++ b/src/vcl/vcl_test_server.c
@@ -78,26 +78,6 @@ static __thread int __wrk_index = 0;
 
 static vcl_test_server_main_t sock_server_main;
 
-#define vtfail(_fn, _rv)						\
-{									\
-  errno = -_rv;								\
-  perror ("ERROR when calling " _fn);					\
-  fprintf (stderr, "\nSERVER ERROR: " _fn " failed (errno = %d)!\n", -_rv);\
-  exit (1);								\
-}
-
-#define vterr(_fn, _rv)							\
-{									\
-  errno = -_rv;								\
-  fprintf (stderr, "\nSERVER ERROR: " _fn " failed (errno = %d)!\n", -_rv);\
-}
-
-#define vtwrn(_fmt, _args...)						\
-  fprintf (stderr, "\nSERVER ERROR: " _fmt "\n", ##_args)		\
-
-#define vtinf(_fmt, _args...)						\
-  fprintf (stdout, "\nvts<w%u>: " _fmt "\n", __wrk_index, ##_args)	\
-
 static inline void
 conn_pool_expand (vcl_test_server_worker_t * wrk, size_t expand_size)
 {
@@ -171,67 +151,60 @@ sync_config_and_reply (vcl_test_server_conn_t * conn,
 }
 
 static void
-stream_test_server_start_stop (vcl_test_server_worker_t * wrk,
-			       vcl_test_server_conn_t * conn,
-			       sock_test_cfg_t * rx_cfg)
+vts_server_start_stop (vcl_test_server_worker_t * wrk,
+		       vcl_test_server_conn_t * conn,
+		       sock_test_cfg_t * rx_cfg)
 {
-  int client_fd = conn->fd;
-  sock_test_t test = rx_cfg->test;
+  u8 is_bi = rx_cfg->test == SOCK_TEST_TYPE_BI;
+  int client_fd = conn->fd, i;
+  vcl_test_server_conn_t *tc;
+  char buf[64];
 
   if (rx_cfg->ctrl_handle == conn->fd)
     {
-      int i;
       clock_gettime (CLOCK_REALTIME, &conn->stats.stop);
 
       for (i = 0; i < wrk->conn_pool_size; i++)
 	{
-	  vcl_test_server_conn_t *tc = &wrk->conn_pool[i];
-
-	  if (tc->cfg.ctrl_handle == conn->fd)
-	    {
-	      sock_test_stats_accumulate (&conn->stats, &tc->stats);
+	  tc = &wrk->conn_pool[i];
+	  if (tc->cfg.ctrl_handle != conn->fd)
+	    continue;
 
-	      if (conn->cfg.verbose)
-		{
-		  static char buf[64];
+	  sock_test_stats_accumulate (&conn->stats, &tc->stats);
 
-		  sprintf (buf, "SERVER (fd %d) RESULTS", tc->fd);
-		  sock_test_stats_dump (buf, &tc->stats, 1 /* show_rx */ ,
-					test == SOCK_TEST_TYPE_BI
-					/* show tx */ ,
-					conn->cfg.verbose);
-		}
+	  if (conn->cfg.verbose)
+	    {
+	      sprintf (buf, "SERVER (fd %d) RESULTS", tc->fd);
+	      sock_test_stats_dump (buf, &tc->stats, 1 /* show_rx */ ,
+				    is_bi /* show tx */ , conn->cfg.verbose);
 	    }
 	}
 
       sock_test_stats_dump ("SERVER RESULTS", &conn->stats, 1 /* show_rx */ ,
-			    (test == SOCK_TEST_TYPE_BI) /* show_tx */ ,
-			    conn->cfg.verbose);
+			    is_bi /* show_tx */ , conn->cfg.verbose);
       sock_test_cfg_dump (&conn->cfg, 0 /* is_client */ );
       if (conn->cfg.verbose)
 	{
-	  printf ("  sock server main\n"
-		  SOCK_TEST_SEPARATOR_STRING
-		  "       buf:  %p\n"
-		  "  buf size:  %u (0x%08x)\n"
-		  SOCK_TEST_SEPARATOR_STRING,
-		  conn->buf, conn->buf_size, conn->buf_size);
+	  vtinf ("  sock server main\n"
+		 SOCK_TEST_SEPARATOR_STRING
+		 "       buf:  %p\n"
+		 "  buf size:  %u (0x%08x)\n"
+		 SOCK_TEST_SEPARATOR_STRING,
+		 conn->buf, conn->buf_size, conn->buf_size);
 	}
 
       sync_config_and_reply (conn, rx_cfg);
-      printf ("\nSERVER (fd %d): %s-directional Stream Test Complete!\n"
-	      SOCK_TEST_BANNER_STRING "\n", conn->fd,
-	      test == SOCK_TEST_TYPE_BI ? "Bi" : "Uni");
+      vtinf ("(fd %d): %s-directional Stream Test Complete!\n"
+	     SOCK_TEST_BANNER_STRING "\n", conn->fd, is_bi ? "Bi" : "Uni");
     }
   else
     {
-      printf ("\n" SOCK_TEST_BANNER_STRING
-	      "SERVER (fd %d): %s-directional Stream Test!\n"
-	      "  Sending client the test cfg to start streaming data...\n",
-	      client_fd, test == SOCK_TEST_TYPE_BI ? "Bi" : "Uni");
+      vtinf (SOCK_TEST_BANNER_STRING "(fd %d): %s-directional Stream Test!\n"
+	     "  Sending client the test cfg to start streaming data...\n",
+	     client_fd, is_bi ? "Bi" : "Uni");
 
-      rx_cfg->ctrl_handle = (rx_cfg->ctrl_handle == ~0) ? conn->fd :
-	rx_cfg->ctrl_handle;
+      if (rx_cfg->ctrl_handle == ~0)
+	rx_cfg->ctrl_handle = conn->fd;
 
       sync_config_and_reply (conn, rx_cfg);
 
@@ -241,9 +214,8 @@ stream_test_server_start_stop (vcl_test_server_worker_t * wrk,
     }
 }
 
-
 static inline void
-stream_test_server (vcl_test_server_conn_t * conn, int rx_bytes)
+vts_server_rx (vcl_test_server_conn_t * conn, int rx_bytes)
 {
   int client_fd = conn->fd;
   sock_test_t test = conn->cfg.test;
@@ -253,13 +225,11 @@ stream_test_server (vcl_test_server_conn_t * conn, int rx_bytes)
 			   conn->cfg.verbose);
 
   if (conn->stats.rx_bytes >= conn->cfg.total_bytes)
-    {
-      clock_gettime (CLOCK_REALTIME, &conn->stats.stop);
-    }
+    clock_gettime (CLOCK_REALTIME, &conn->stats.stop);
 }
 
 static void
-vcl_test_server_echo (vcl_test_server_conn_t * conn, int rx_bytes)
+vts_server_echo (vcl_test_server_conn_t * conn, int rx_bytes)
 {
   int tx_bytes, nbytes, pos;
 
@@ -281,7 +251,7 @@ vcl_test_server_echo (vcl_test_server_conn_t * conn, int rx_bytes)
 }
 
 static inline void
-vcl_test_server_new_client (vcl_test_server_worker_t * wrk)
+vts_new_client (vcl_test_server_worker_t * wrk)
 {
   int client_fd;
   vcl_test_server_conn_t *conn;
@@ -370,7 +340,7 @@ vcl_test_init_endpoint_addr (vcl_test_server_main_t * ssm)
     }
 }
 
-void
+static void
 vcl_test_server_process_opts (vcl_test_server_main_t * ssm, int argc,
 			      char **argv)
 {
@@ -403,11 +373,9 @@ vcl_test_server_process_opts (vcl_test_server_main_t * ssm, int argc,
 	  {
 	  default:
 	    if (isprint (optopt))
-	      fprintf (stderr, "SERVER: ERROR: Unknown "
-		       "option `-%c'.\n", optopt);
+	      vtwrn ("Unknown option `-%c'.", optopt);
 	    else
-	      fprintf (stderr, "SERVER: ERROR: Unknown "
-		       "option character `\\x%x'.\n", optopt);
+	      vtwrn ("Unknown option character `\\x%x'.", optopt);
 	  }
 	/* fall thru */
       case 'h':
@@ -433,9 +401,8 @@ vcl_test_server_process_opts (vcl_test_server_main_t * ssm, int argc,
 }
 
 int
-vcl_test_server_handle_cfg (vcl_test_server_worker_t * wrk,
-			    sock_test_cfg_t * rx_cfg,
-			    vcl_test_server_conn_t * conn, int rx_bytes)
+vts_handle_cfg (vcl_test_server_worker_t * wrk, sock_test_cfg_t * rx_cfg,
+		vcl_test_server_conn_t * conn, int rx_bytes)
 {
   if (rx_cfg->verbose)
     {
@@ -468,7 +435,7 @@ vcl_test_server_handle_cfg (vcl_test_server_worker_t * wrk,
 
     case SOCK_TEST_TYPE_BI:
     case SOCK_TEST_TYPE_UNI:
-      stream_test_server_start_stop (wrk, conn, rx_cfg);
+      vts_server_start_stop (wrk, conn, rx_cfg);
       break;
 
     case SOCK_TEST_TYPE_EXIT:
@@ -488,7 +455,7 @@ vcl_test_server_handle_cfg (vcl_test_server_worker_t * wrk,
 }
 
 static void
-vcl_test_server_worker_init (vcl_test_server_worker_t * wrk)
+vts_worker_init (vcl_test_server_worker_t * wrk)
 {
   vcl_test_server_main_t *ssm = &sock_server_main;
   struct epoll_event listen_ev;
@@ -533,7 +500,7 @@ vcl_test_server_worker_init (vcl_test_server_worker_t * wrk)
 }
 
 static void *
-vcl_test_server_worker_loop (void *arg)
+vts_worker_loop (void *arg)
 {
   vcl_test_server_main_t *ssm = &sock_server_main;
   vcl_test_server_worker_t *wrk = arg;
@@ -542,7 +509,7 @@ vcl_test_server_worker_loop (void *arg)
   sock_test_cfg_t *rx_cfg;
 
   if (wrk->wrk_index)
-    vcl_test_server_worker_init (wrk);
+    vts_worker_init (wrk);
 
   while (1)
     {
@@ -568,7 +535,7 @@ vcl_test_server_worker_loop (void *arg)
 	    }
 	  if (wrk->wait_events[i].data.u32 == ~0)
 	    {
-	      vcl_test_server_new_client (wrk);
+	      vts_new_client (wrk);
 	      continue;
 	    }
 
@@ -591,7 +558,7 @@ vcl_test_server_worker_loop (void *arg)
 	      rx_cfg = (sock_test_cfg_t *) conn->buf;
 	      if (rx_cfg->magic == SOCK_TEST_CFG_CTRL_MAGIC)
 		{
-		  vcl_test_server_handle_cfg (wrk, rx_cfg, conn, rx_bytes);
+		  vts_handle_cfg (wrk, rx_cfg, conn, rx_bytes);
 		  if (!wrk->nfds)
 		    {
 		      vtinf ("All client connections closed\n");
@@ -603,12 +570,12 @@ vcl_test_server_worker_loop (void *arg)
 	      else if ((conn->cfg.test == SOCK_TEST_TYPE_UNI)
 		       || (conn->cfg.test == SOCK_TEST_TYPE_BI))
 		{
-		  stream_test_server (conn, rx_bytes);
+		  vts_server_rx (conn, rx_bytes);
 		  continue;
 		}
 	      else if (isascii (conn->buf[0]))
 		{
-		  vcl_test_server_echo (conn, rx_bytes);
+		  vts_server_echo (conn, rx_bytes);
 		}
 	      else
 		{
@@ -637,37 +604,36 @@ done:
 int
 main (int argc, char **argv)
 {
-  vcl_test_server_main_t *ssm = &sock_server_main;
+  vcl_test_server_main_t *vsm = &sock_server_main;
   int rv, i;
 
   clib_mem_init_thread_safe (0, 64 << 20);
-  ssm->cfg.port = SOCK_TEST_SERVER_PORT;
-  ssm->cfg.workers = 1;
-  ssm->active_workers = 1;
-  vcl_test_server_process_opts (ssm, argc, argv);
+  vsm->cfg.port = SOCK_TEST_SERVER_PORT;
+  vsm->cfg.workers = 1;
+  vsm->active_workers = 1;
+  vcl_test_server_process_opts (vsm, argc, argv);
 
   rv = vppcom_app_create ("vcl_test_server");
   if (rv)
     vtfail ("vppcom_app_create()", rv);
 
-  ssm->workers = calloc (ssm->cfg.workers, sizeof (*ssm->workers));
-  vcl_test_server_worker_init (&ssm->workers[0]);
-  for (i = 1; i < ssm->cfg.workers; i++)
+  vsm->workers = calloc (vsm->cfg.workers, sizeof (*vsm->workers));
+  vts_worker_init (&vsm->workers[0]);
+  for (i = 1; i < vsm->cfg.workers; i++)
     {
-      ssm->workers[i].wrk_index = i;
-      rv = pthread_create (&ssm->workers[i].thread_handle, NULL,
-			   vcl_test_server_worker_loop,
-			   (void *) &ssm->workers[i]);
+      vsm->workers[i].wrk_index = i;
+      rv = pthread_create (&vsm->workers[i].thread_handle, NULL,
+			   vts_worker_loop, (void *) &vsm->workers[i]);
     }
-  vcl_test_server_worker_loop (&ssm->workers[0]);
+  vts_worker_loop (&vsm->workers[0]);
 
-  while (ssm->active_workers > 0)
+  while (vsm->active_workers > 0)
     ;
 
   vppcom_app_destroy ();
-  free (ssm->workers);
+  free (vsm->workers);
 
-  return ssm->worker_fails;
+  return vsm->worker_fails;
 }
 
 /*
diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c
index 1991e3e7277..15622014dcd 100644
--- a/src/vcl/vppcom.c
+++ b/src/vcl/vppcom.c
@@ -1580,6 +1580,14 @@ vcl_mq_dequeue_batch (vcl_worker_t * wrk, svm_msg_q_t * mq)
   return n_msgs;
 }
 
+#define vcl_fifo_rx_evt_valid_or_break(_fifo)			\
+if (PREDICT_FALSE (svm_fifo_is_empty (_fifo)))			\
+  {								\
+    svm_fifo_unset_event (_fifo);				\
+    if (svm_fifo_is_empty (_fifo))				\
+	break;							\
+  }								\
+
 static int
 vcl_select_handle_mq (vcl_worker_t * wrk, svm_msg_q_t * mq,
 		      unsigned long n_bits, unsigned long *read_map,
@@ -1633,6 +1641,7 @@ vcl_select_handle_mq (vcl_worker_t * wrk, svm_msg_q_t * mq,
       switch (e->event_type)
 	{
 	case FIFO_EVENT_APP_RX:
+	  vcl_fifo_rx_evt_valid_or_break (e->fifo);
 	  sid = e->fifo->client_session_index;
 	  session = vcl_session_get (wrk, sid);
 	  if (!session)
@@ -1655,12 +1664,7 @@ vcl_select_handle_mq (vcl_worker_t * wrk, svm_msg_q_t * mq,
 	    }
 	  break;
 	case SESSION_IO_EVT_CT_TX:
-	  if (svm_fifo_is_empty (e->fifo))
-	    {
-	      svm_fifo_unset_event (e->fifo);
-	      if (svm_fifo_is_empty (e->fifo))
-		break;
-	    }
+	  vcl_fifo_rx_evt_valid_or_break (e->fifo);
 	  session = vcl_ct_session_get_from_fifo (wrk, e->fifo, 0);
 	  if (!session)
 	    break;
@@ -2228,6 +2232,7 @@ vcl_epoll_wait_handle_mq (vcl_worker_t * wrk, svm_msg_q_t * mq,
       switch (e->event_type)
 	{
 	case FIFO_EVENT_APP_RX:
+	  vcl_fifo_rx_evt_valid_or_break (e->fifo);
 	  sid = e->fifo->client_session_index;
 	  session = vcl_session_get (wrk, sid);
 	  session_events = session->vep.ev.events;
@@ -2248,6 +2253,7 @@ vcl_epoll_wait_handle_mq (vcl_worker_t * wrk, svm_msg_q_t * mq,
 	  session_evt_data = session->vep.ev.data.u64;
 	  break;
 	case SESSION_IO_EVT_CT_TX:
+	  vcl_fifo_rx_evt_valid_or_break (e->fifo);
 	  session = vcl_ct_session_get_from_fifo (wrk, e->fifo, 0);
 	  sid = session->session_index;
 	  session_events = session->vep.ev.events;
@@ -3156,7 +3162,7 @@ vppcom_session_index (uint32_t session_handle)
 int
 vppcom_worker_register (void)
 {
-  if (!vcl_worker_alloc_and_init ())
+  if (vcl_worker_alloc_and_init ())
     return VPPCOM_OK;
   return VPPCOM_EEXIST;
 }