vcl: refactor test client to support workers

Change-Id: I7fcfddc2bc7d9a64f8aa0d57ba5d11d325a15ce1
Signed-off-by: Florin Coras <fcoras@cisco.com>

1 files changed, 473 insertions, 520 deletions

diff --git a/src/vcl/vcl_test_client.c b/src/vcl/vcl_test_client.c
index 4e61045b055..b2431df80fa 100644
--- a/src/vcl/vcl_test_client.c
+++ b/src/vcl/vcl_test_client.c
@@ -23,523 +23,515 @@
 #include <time.h>
 #include <arpa/inet.h>
 #include <vcl/vcl_test.h>
+#include <pthread.h>
 
 typedef struct
 {
+  sock_test_socket_t *sessions;
+  uint32_t n_sessions;
+  uint32_t wrk_index;
+  fd_set wr_fdset;
+  fd_set rd_fdset;
+  int max_fd_index;
+  pthread_t thread_handle;
+  sock_test_cfg_t cfg;
+} vcl_test_client_worker_t;
+
+typedef struct
+{
+  vcl_test_client_worker_t *workers;
   vppcom_endpt_t server_endpt;
-  struct sockaddr_storage server_addr;
-  uint32_t server_addr_size;
   uint32_t cfg_seq_num;
   sock_test_socket_t ctrl_socket;
-  sock_test_socket_t *test_socket;
-  uint32_t num_test_sockets;
+  sock_test_socket_t *sessions;
   uint8_t dump_cfg;
-} sock_client_main_t;
+  sock_test_t post_test;
+  uint32_t proto;
+  uint32_t n_workers;
+  volatile int active_workers;
+  struct sockaddr_storage server_addr;
+} vcl_test_client_main_t;
+
+static __thread int __wrk_index = 0;
 
-sock_client_main_t sock_client_main;
+vcl_test_client_main_t vcl_client_main;
 
+#define vtc_min(a, b) (a < b ? a : b)
+#define vtc_max(a, b) (a > b ? a : b)
 
 static int
-sock_test_cfg_sync (sock_test_socket_t * socket)
+vtc_cfg_sync (sock_test_socket_t * sock)
 {
-  sock_client_main_t *scm = &sock_client_main;
-  sock_test_socket_t *ctrl = &scm->ctrl_socket;
-  sock_test_cfg_t *rl_cfg = (sock_test_cfg_t *) socket->rxbuf;
+  vcl_test_client_main_t *vcm = &vcl_client_main;
+  sock_test_cfg_t *rx_cfg = (sock_test_cfg_t *) sock->rxbuf;
   int rx_bytes, tx_bytes;
 
-  if (socket->cfg.verbose)
-    sock_test_cfg_dump (&socket->cfg, 1 /* is_client */ );
-
-  ctrl->cfg.seq_num = ++scm->cfg_seq_num;
-  if (socket->cfg.verbose)
+  vt_atomic_add (&sock->cfg.seq_num, 1);
+  if (sock->cfg.verbose)
     {
-      printf ("CLIENT (fd %d): Sending config sent to server.\n", socket->fd);
-      sock_test_cfg_dump (&ctrl->cfg, 1 /* is_client */ );
+      vtinf ("(fd %d): Sending config to server.", sock->fd);
+      sock_test_cfg_dump (&sock->cfg, 1 /* is_client */ );
     }
-  tx_bytes = vcl_test_write (socket->fd, (uint8_t *) & ctrl->cfg,
-			     sizeof (ctrl->cfg), NULL, ctrl->cfg.verbose);
+  tx_bytes = vcl_test_write (sock->fd, (uint8_t *) & sock->cfg,
+			     sizeof (sock->cfg), NULL, sock->cfg.verbose);
   if (tx_bytes < 0)
     {
-      fprintf (stderr, "CLIENT (fd %d): ERROR: write test cfg failed (%d)!\n",
-	       socket->fd, tx_bytes);
+      vtwrn ("(fd %d): write test cfg failed (%d)!", sock->fd, tx_bytes);
       return tx_bytes;
     }
 
-  rx_bytes = vcl_test_read (socket->fd, (uint8_t *) socket->rxbuf,
+  rx_bytes = vcl_test_read (sock->fd, (uint8_t *) sock->rxbuf,
 			    sizeof (sock_test_cfg_t), NULL);
   if (rx_bytes < 0)
     return rx_bytes;
 
-  if (rl_cfg->magic != SOCK_TEST_CFG_CTRL_MAGIC)
+  if (rx_cfg->magic != SOCK_TEST_CFG_CTRL_MAGIC)
     {
-      fprintf (stderr, "CLIENT (fd %d): ERROR: Bad server reply cfg "
-	       "-- aborting!\n", socket->fd);
+      vtwrn ("(fd %d): Bad server reply cfg -- aborting!", sock->fd);
       return -1;
     }
   if ((rx_bytes != sizeof (sock_test_cfg_t))
-      || !sock_test_cfg_verify (rl_cfg, &ctrl->cfg))
+      || !sock_test_cfg_verify (rx_cfg, &sock->cfg))
     {
-      fprintf (stderr, "CLIENT (fd %d): ERROR: Invalid config received "
-	       "from server!\n", socket->fd);
+      vtwrn ("(fd %d): Invalid config received from server!", sock->fd);
       if (rx_bytes != sizeof (sock_test_cfg_t))
 	{
-	  fprintf (stderr, "\tRx bytes %d != cfg size %lu\n",
-		   rx_bytes, sizeof (sock_test_cfg_t));
+	  vtinf ("\tRx bytes %d != cfg size %lu", rx_bytes,
+		 sizeof (sock_test_cfg_t));
 	}
       else
 	{
-	  sock_test_cfg_dump (rl_cfg, 1 /* is_client */ );
-	  fprintf (stderr, "CLIENT (fd %d): Valid config sent to server.\n",
-		   socket->fd);
-	  sock_test_cfg_dump (&ctrl->cfg, 1 /* is_client */ );
+	  sock_test_cfg_dump (rx_cfg, 1 /* is_client */ );
+	  vtinf ("(fd %d): Valid config sent to server.", sock->fd);
+	  sock_test_cfg_dump (&sock->cfg, 1 /* is_client */ );
 	}
       return -1;
     }
-  else if (socket->cfg.verbose)
+  if (sock->cfg.verbose)
     {
-      printf ("CLIENT (fd %d): Got config back from server.\n", socket->fd);
-      sock_test_cfg_dump (rl_cfg, 1 /* is_client */ );
+      vtinf ("(fd %d): Got config back from server.", sock->fd);
+      sock_test_cfg_dump (rx_cfg, 1 /* is_client */ );
     }
-  ctrl->cfg.ctrl_handle = ((ctrl->cfg.ctrl_handle == ~0) ?
-			   rl_cfg->ctrl_handle : ctrl->cfg.ctrl_handle);
 
   return 0;
 }
 
-static void
-echo_test_client ()
+static int
+vtc_connect_test_sessions (vcl_test_client_worker_t * wrk)
 {
-  sock_client_main_t *scm = &sock_client_main;
-  sock_test_socket_t *ctrl = &scm->ctrl_socket;
+  vcl_test_client_main_t *vcm = &vcl_client_main;
   sock_test_socket_t *tsock;
-  int rv, nfds = 0, rx_bytes, tx_bytes, nbytes;
-  uint32_t i, n, sidx;
-  fd_set wr_fdset, rd_fdset;
-  fd_set _wfdset, *wfdset = &_wfdset;
-  fd_set _rfdset, *rfdset = &_rfdset;
+  uint32_t n_test_sessions;
+  int i, rv;
 
-  FD_ZERO (&wr_fdset);
-  FD_ZERO (&rd_fdset);
-  memset (&ctrl->stats, 0, sizeof (ctrl->stats));
-  ctrl->cfg.total_bytes = nbytes = strlen (ctrl->txbuf) + 1;
-  for (n = 0; n != ctrl->cfg.num_test_sockets; n++)
+  n_test_sessions = wrk->cfg.num_test_sockets;
+  if (n_test_sessions < 1)
     {
-      tsock = &scm->test_socket[n];
-      tsock->cfg = ctrl->cfg;
-      sock_test_socket_buf_alloc (tsock);
-      if (sock_test_cfg_sync (tsock))
-	return;
+      errno = EINVAL;
+      return -1;
+    }
 
-      memcpy (tsock->txbuf, ctrl->txbuf, nbytes);
-      memset (&tsock->stats, 0, sizeof (tsock->stats));
+  if (wrk->n_sessions >= n_test_sessions)
+    goto done;
 
-      FD_SET (tsock->fd, &wr_fdset);
-      FD_SET (tsock->fd, &rd_fdset);
-      sidx = vppcom_session_index (tsock->fd);
-      nfds = sidx > nfds ? sidx : nfds;
-    }
+  if (wrk->n_sessions)
+    wrk->sessions = realloc (wrk->sessions,
+			     n_test_sessions * sizeof (sock_test_socket_t));
+  else
+    wrk->sessions = calloc (n_test_sessions, sizeof (sock_test_socket_t));
 
-  nfds++;
-  clock_gettime (CLOCK_REALTIME, &ctrl->stats.start);
-  while (n)
+  if (!wrk->sessions)
     {
-      _wfdset = wr_fdset;
-      _rfdset = rd_fdset;
+      vterr ("failed to alloc sessions", -errno);
+      return errno;
+    }
 
-      rv = vppcom_select (nfds, (uint64_t *) rfdset, (uint64_t *) wfdset,
-			  NULL, 0);
-      if (rv < 0)
+  for (i = 0; i < n_test_sessions; i++)
+    {
+      tsock = &wrk->sessions[i];
+      tsock->fd = vppcom_session_create (vcm->proto, 1 /* is_nonblocking */ );
+      if (tsock->fd < 0)
 	{
-	  perror ("select()");
-	  fprintf (stderr, "\nCLIENT: ERROR: select() failed -- "
-		   "aborting test!\n");
-	  return;
+	  vterr ("vppcom_session_create()", tsock->fd);
+	  return tsock->fd;
 	}
-      else if (rv == 0)
-	continue;
 
-      for (i = 0; i < ctrl->cfg.num_test_sockets; i++)
+      rv = vppcom_session_connect (tsock->fd, &vcm->server_endpt);
+      if (rv < 0)
 	{
-	  tsock = &scm->test_socket[i];
-	  if (!((tsock->stats.stop.tv_sec == 0) &&
-		(tsock->stats.stop.tv_nsec == 0)))
-	    continue;
-
-	  if (FD_ISSET (vppcom_session_index (tsock->fd), wfdset)
-	      && (tsock->stats.tx_bytes < ctrl->cfg.total_bytes))
-
-	    {
-	      tx_bytes = vcl_test_write (tsock->fd, (uint8_t *) tsock->txbuf,
-					 nbytes, &tsock->stats,
-					 ctrl->cfg.verbose);
-	      if (tx_bytes < 0)
-		{
-		  fprintf (stderr, "\nCLIENT: ERROR: vcl_test_write(%d) "
-			   "failed -- aborting test!\n", tsock->fd);
-		  return;
-		}
-
-	    }
-
-	  if ((FD_ISSET (vppcom_session_index (tsock->fd), rfdset)) &&
-	      (tsock->stats.rx_bytes < ctrl->cfg.total_bytes))
-	    {
-	      rx_bytes = vcl_test_read (tsock->fd, (uint8_t *) tsock->rxbuf,
-					nbytes, &tsock->stats);
-	      if (rx_bytes > 0)
-		{
-		  printf ("CLIENT (fd %d): RX (%d bytes) - '%s'\n",
-			  tsock->fd, rx_bytes, tsock->rxbuf);
-
-		  if (tsock->stats.rx_bytes != tsock->stats.tx_bytes)
-		    printf ("CLIENT: WARNING: bytes read (%lu) "
-			    "!= bytes written (%lu)!\n",
-			    tsock->stats.rx_bytes, tsock->stats.tx_bytes);
-		}
-	    }
-
-	  if (tsock->stats.rx_bytes >= ctrl->cfg.total_bytes)
-	    {
-	      clock_gettime (CLOCK_REALTIME, &tsock->stats.stop);
-	      n--;
-	    }
+	  vterr ("vppcom_session_connect()", rv);
+	  return rv;
 	}
+      vtinf ("(fd %d): Test socket %d connected.", tsock->fd, i);
     }
-  clock_gettime (CLOCK_REALTIME, &ctrl->stats.stop);
-
-  for (i = 0; i < ctrl->cfg.num_test_sockets; i++)
-    {
-      tsock = &scm->test_socket[i];
-      tsock->stats.start = ctrl->stats.start;
+  wrk->n_sessions = n_test_sessions;
 
-      if (ctrl->cfg.verbose)
-	{
-	  static char buf[64];
+done:
+  vtinf ("All test sessions (%d) connected!", n_test_sessions);
+  return 0;
+}
 
-	  sprintf (buf, "CLIENT (fd %d) RESULTS", tsock->fd);
-	  sock_test_stats_dump (buf, &tsock->stats,
-				1 /* show_rx */ , 1 /* show tx */ ,
-				ctrl->cfg.verbose);
-	}
+static int
+vtc_worker_test_setup (vcl_test_client_worker_t * wrk)
+{
+  vcl_test_client_main_t *vcm = &vcl_client_main;
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
+  sock_test_cfg_t *cfg = &wrk->cfg;
+  sock_test_socket_t *tsock;
+  uint32_t sidx;
+  int i, j;
 
-      sock_test_stats_accumulate (&ctrl->stats, &tsock->stats);
-    }
+  FD_ZERO (&wrk->wr_fdset);
+  FD_ZERO (&wrk->rd_fdset);
 
-  if (ctrl->cfg.verbose)
+  for (i = 0; i < cfg->num_test_sockets; i++)
     {
-      sock_test_stats_dump ("CLIENT RESULTS", &ctrl->stats,
-			    1 /* show_rx */ , 1 /* show tx */ ,
-			    ctrl->cfg.verbose);
-      sock_test_cfg_dump (&ctrl->cfg, 1 /* is_client */ );
+      tsock = &wrk->sessions[i];
+      tsock->cfg = wrk->cfg;
+      sock_test_socket_buf_alloc (tsock);
 
-      if (ctrl->cfg.verbose > 1)
+      switch (cfg->test)
 	{
-	  printf ("  ctrl socket info\n"
-		  SOCK_TEST_SEPARATOR_STRING
-		  "          fd:  %d (0x%08x)\n"
-		  "       rxbuf:  %p\n"
-		  "  rxbuf size:  %u (0x%08x)\n"
-		  "       txbuf:  %p\n"
-		  "  txbuf size:  %u (0x%08x)\n"
-		  SOCK_TEST_SEPARATOR_STRING,
-		  ctrl->fd, (uint32_t) ctrl->fd,
-		  ctrl->rxbuf, ctrl->rxbuf_size, ctrl->rxbuf_size,
-		  ctrl->txbuf, ctrl->txbuf_size, ctrl->txbuf_size);
+	case SOCK_TEST_TYPE_ECHO:
+	  memcpy (tsock->txbuf, ctrl->txbuf, cfg->total_bytes);
+	  break;
+	case SOCK_TEST_TYPE_UNI:
+	case SOCK_TEST_TYPE_BI:
+	  for (j = 0; j < tsock->txbuf_size; j++)
+	    tsock->txbuf[j] = j & 0xff;
+	  break;
 	}
+
+      FD_SET (vppcom_session_index (tsock->fd), &wrk->wr_fdset);
+      FD_SET (vppcom_session_index (tsock->fd), &wrk->rd_fdset);
+      sidx = vppcom_session_index (tsock->fd);
+      wrk->max_fd_index = vtc_max (sidx, wrk->max_fd_index);
     }
+  wrk->max_fd_index += 1;
+
+  return 0;
 }
 
-static void
-stream_test_client (sock_test_t test)
+static int
+vtc_worker_init (vcl_test_client_worker_t * wrk)
 {
-  sock_client_main_t *scm = &sock_client_main;
-  sock_test_socket_t *ctrl = &scm->ctrl_socket;
+  vcl_test_client_main_t *vcm = &vcl_client_main;
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
+  sock_test_cfg_t *cfg = &wrk->cfg;
   sock_test_socket_t *tsock;
-  int tx_bytes, rv, nfds = 0;
-  uint32_t i, n, sidx;
-  fd_set wr_fdset, rd_fdset;
-  fd_set _wfdset, *wfdset = &_wfdset;
-  fd_set _rfdset, *rfdset = (test == SOCK_TEST_TYPE_BI) ? &_rfdset : 0;
+  uint32_t i, n;
+  int rv, nbytes;
 
-  ctrl->cfg.total_bytes = ctrl->cfg.num_writes * ctrl->cfg.txbuf_size;
-  ctrl->cfg.ctrl_handle = ~0;
+  __wrk_index = wrk->wrk_index;
 
-  printf ("\n" SOCK_TEST_BANNER_STRING
-	  "CLIENT (fd %d): %s-directional Stream Test!\n\n"
-	  "CLIENT (fd %d): Sending config to server on ctrl socket...\n",
-	  ctrl->fd, test == SOCK_TEST_TYPE_BI ? "Bi" : "Uni", ctrl->fd);
+  vtinf ("Initializing worker ...");
 
-  if (sock_test_cfg_sync (ctrl))
+  if (wrk->wrk_index)
     {
-      fprintf (stderr, "CLIENT: ERROR: test cfg sync failed -- aborting!");
-      return;
-    }
+      if (vppcom_worker_register ())
+	{
+	  vtwrn ("failed to register worker");
+	  return -1;
+	}
 
-  FD_ZERO (&wr_fdset);
-  FD_ZERO (&rd_fdset);
-  memset (&ctrl->stats, 0, sizeof (ctrl->stats));
-  for (n = 0; n != ctrl->cfg.num_test_sockets; n++)
+      vt_atomic_add (&vcm->active_workers, 1);
+    }
+  rv = vtc_connect_test_sessions (wrk);
+  if (rv)
     {
-      tsock = &scm->test_socket[n];
-      tsock->cfg = ctrl->cfg;
-      sock_test_socket_buf_alloc (tsock);
-      printf ("CLIENT (fd %d): Sending config to server on "
-	      "test socket %d...\n", tsock->fd, n);
-      sock_test_cfg_sync (tsock);
+      vterr ("vtc_connect_test_sockets ()", rv);
+      return rv;
+    }
+
+  if (vtc_worker_test_setup (wrk))
+    return -1;
 
-      /* Fill payload with incrementing uint32's */
-      for (i = 0; i < tsock->txbuf_size; i++)
-	tsock->txbuf[i] = i & 0xff;
+  vtinf ("Sending config to server on all sessions ...");
 
+  for (n = 0; n < cfg->num_test_sockets; n++)
+    {
+      tsock = &wrk->sessions[n];
+      if (vtc_cfg_sync (tsock))
+	return -1;
       memset (&tsock->stats, 0, sizeof (tsock->stats));
-      FD_SET (tsock->fd, &wr_fdset);
-      FD_SET (tsock->fd, &rd_fdset);
-      sidx = vppcom_session_index (tsock->fd);
-      nfds = sidx > nfds ? sidx : nfds;
     }
 
-  nfds++;
-  clock_gettime (CLOCK_REALTIME, &ctrl->stats.start);
-  while (n)
+  return 0;
+}
+
+static void *
+vtc_worker_loop (void *arg)
+{
+  vcl_test_client_main_t *vcm = &vcl_client_main;
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
+  vcl_test_client_worker_t *wrk = arg;
+  fd_set _wfdset, *wfdset = &_wfdset;
+  fd_set _rfdset, *rfdset = &_rfdset;
+  sock_test_socket_t *tsock;
+  uint32_t n_active_sessions;
+  int i, rv, check_rx = 0;
+
+  rv = vtc_worker_init (wrk);
+  if (rv)
+    {
+      vterr ("vtc_worker_init()", rv);
+      return 0;
+    }
+
+  vtinf ("Starting test ...");
+
+  if (wrk->wrk_index == 0)
+    clock_gettime (CLOCK_REALTIME, &ctrl->stats.start);
+
+  check_rx = wrk->cfg.test != SOCK_TEST_TYPE_UNI;
+  n_active_sessions = wrk->cfg.num_test_sockets;
+  while (n_active_sessions)
     {
-      _wfdset = wr_fdset;
-      _rfdset = rd_fdset;
+      _wfdset = wrk->wr_fdset;
+      _rfdset = wrk->rd_fdset;
 
-      rv = vppcom_select (nfds, (uint64_t *) rfdset, (uint64_t *) wfdset,
-			  NULL, 0);
+      rv = vppcom_select (wrk->max_fd_index, (uint64_t *) rfdset,
+			  (uint64_t *) wfdset, NULL, 0);
       if (rv < 0)
 	{
-	  perror ("select()");
-	  fprintf (stderr, "\nCLIENT: ERROR: select() failed -- "
-		   "aborting test!\n");
-	  return;
+	  vterr ("vppcom_select()", rv);
+	  goto exit;
 	}
       else if (rv == 0)
 	continue;
 
-      for (i = 0; i < ctrl->cfg.num_test_sockets; i++)
+      for (i = 0; i < wrk->cfg.num_test_sockets; i++)
 	{
-	  tsock = &scm->test_socket[i];
+	  tsock = &wrk->sessions[i];
 	  if (!((tsock->stats.stop.tv_sec == 0) &&
 		(tsock->stats.stop.tv_nsec == 0)))
 	    continue;
 
-	  if ((test == SOCK_TEST_TYPE_BI) &&
-	      FD_ISSET (vppcom_session_index (tsock->fd), rfdset) &&
-	      (tsock->stats.rx_bytes < ctrl->cfg.total_bytes))
+	  if (FD_ISSET (vppcom_session_index (tsock->fd), rfdset)
+	      && tsock->stats.rx_bytes < tsock->cfg.total_bytes)
 	    {
-	      (void) vcl_test_read (tsock->fd,
-				    (uint8_t *) tsock->rxbuf,
+	      (void) vcl_test_read (tsock->fd, (uint8_t *) tsock->rxbuf,
 				    tsock->rxbuf_size, &tsock->stats);
 	    }
 
 	  if (FD_ISSET (vppcom_session_index (tsock->fd), wfdset)
-	      && (tsock->stats.tx_bytes < ctrl->cfg.total_bytes))
+	      && tsock->stats.tx_bytes < tsock->cfg.total_bytes)
 	    {
-	      tx_bytes = vcl_test_write (tsock->fd, (uint8_t *) tsock->txbuf,
-					 ctrl->cfg.txbuf_size, &tsock->stats,
-					 ctrl->cfg.verbose);
-	      if (tx_bytes < 0)
+	      rv = vcl_test_write (tsock->fd, (uint8_t *) tsock->txbuf,
+				   tsock->cfg.txbuf_size, &tsock->stats,
+				   tsock->cfg.verbose);
+	      if (rv < 0)
 		{
-		  fprintf (stderr, "\nCLIENT: ERROR: vcl_test_write(%d) "
-			   "failed -- aborting test!\n", tsock->fd);
-		  return;
+		  vtwrn ("vppcom_test_write (%d) failed -- aborting test",
+			 tsock->fd);
+		  goto exit;
 		}
 	    }
 
-	  if (((test == SOCK_TEST_TYPE_UNI) &&
-	       (tsock->stats.tx_bytes >= ctrl->cfg.total_bytes)) ||
-	      ((test == SOCK_TEST_TYPE_BI) &&
-	       (tsock->stats.rx_bytes >= ctrl->cfg.total_bytes)))
+	  if ((!check_rx && tsock->stats.tx_bytes >= tsock->cfg.total_bytes)
+	      || (check_rx
+		  && tsock->stats.rx_bytes >= tsock->cfg.total_bytes))
 	    {
 	      clock_gettime (CLOCK_REALTIME, &tsock->stats.stop);
-	      n--;
+	      n_active_sessions--;
 	    }
 	}
     }
-  clock_gettime (CLOCK_REALTIME, &ctrl->stats.stop);
+exit:
+  if (wrk->wrk_index)
+    vt_atomic_add (&vcm->active_workers, -1);
+  return 0;
+}
 
-  printf ("CLIENT (fd %d): Sending config to server on ctrl socket...\n",
-	  ctrl->fd);
+static void
+vtc_accumulate_stats (vcl_test_client_worker_t * wrk,
+		      sock_test_socket_t * ctrl)
+{
+  sock_test_socket_t *tsock;
+  static char buf[64];
+  int i, show_rx = 0;
 
-  if (sock_test_cfg_sync (ctrl))
-    {
-      fprintf (stderr, "CLIENT: ERROR: test cfg sync failed -- aborting!");
-      return;
-    }
+  if (ctrl->cfg.test == SOCK_TEST_TYPE_BI
+      || ctrl->cfg.test == SOCK_TEST_TYPE_ECHO)
+    show_rx = 1;
 
-  for (i = 0; i < ctrl->cfg.num_test_sockets; i++)
+  for (i = 0; i < wrk->cfg.num_test_sockets; i++)
     {
-      tsock = &scm->test_socket[i];
+      tsock = &wrk->sessions[i];
+      tsock->stats.start = ctrl->stats.start;
 
-      if (ctrl->cfg.verbose)
+      if (ctrl->cfg.verbose > 1)
 	{
-	  static char buf[64];
-
 	  sprintf (buf, "CLIENT (fd %d) RESULTS", tsock->fd);
-	  sock_test_stats_dump (buf, &tsock->stats,
-				test == SOCK_TEST_TYPE_BI /* show_rx */ ,
-				1 /* show tx */ , ctrl->cfg.verbose);
+	  sock_test_stats_dump (buf, &tsock->stats, show_rx, 1 /* show tx */ ,
+				ctrl->cfg.verbose);
 	}
 
       sock_test_stats_accumulate (&ctrl->stats, &tsock->stats);
     }
+}
+
+static void
+vtc_print_stats (sock_test_socket_t * ctrl)
+{
+  int is_echo = ctrl->cfg.test == SOCK_TEST_TYPE_ECHO;
+  int show_rx = 0;
+  char buf[64];
+
+  if (ctrl->cfg.test == SOCK_TEST_TYPE_BI
+      || ctrl->cfg.test == SOCK_TEST_TYPE_ECHO)
+    show_rx = 1;
 
   sock_test_stats_dump ("CLIENT RESULTS", &ctrl->stats,
-			test == SOCK_TEST_TYPE_BI /* show_rx */ ,
-			1 /* show tx */ , ctrl->cfg.verbose);
+			show_rx, 1 /* show tx */ ,
+			ctrl->cfg.verbose);
   sock_test_cfg_dump (&ctrl->cfg, 1 /* is_client */ );
 
   if (ctrl->cfg.verbose)
     {
-      printf ("  ctrl socket info\n"
-	      SOCK_TEST_SEPARATOR_STRING
-	      "          fd:  %d (0x%08x)\n"
-	      "       rxbuf:  %p\n"
-	      "  rxbuf size:  %u (0x%08x)\n"
-	      "       txbuf:  %p\n"
-	      "  txbuf size:  %u (0x%08x)\n"
-	      SOCK_TEST_SEPARATOR_STRING,
-	      ctrl->fd, (uint32_t) ctrl->fd,
-	      ctrl->rxbuf, ctrl->rxbuf_size, ctrl->rxbuf_size,
-	      ctrl->txbuf, ctrl->txbuf_size, ctrl->txbuf_size);
+      vtinf ("  ctrl socket info\n"
+	     SOCK_TEST_SEPARATOR_STRING
+	     "          fd:  %d (0x%08x)\n"
+	     "       rxbuf:  %p\n"
+	     "  rxbuf size:  %u (0x%08x)\n"
+	     "       txbuf:  %p\n"
+	     "  txbuf size:  %u (0x%08x)\n"
+	     SOCK_TEST_SEPARATOR_STRING,
+	     ctrl->fd, (uint32_t) ctrl->fd,
+	     ctrl->rxbuf, ctrl->rxbuf_size, ctrl->rxbuf_size,
+	     ctrl->txbuf, ctrl->txbuf_size, ctrl->txbuf_size);
     }
 
-  ctrl->cfg.test = SOCK_TEST_TYPE_ECHO;
-  if (sock_test_cfg_sync (ctrl))
-    fprintf (stderr, "CLIENT: ERROR: post-test cfg sync failed!");
+  if (is_echo)
+    sprintf (buf, "Echo");
+  else
+    sprintf (buf, "%s-directional Stream",
+	     ctrl->cfg.test == SOCK_TEST_TYPE_BI ? "Bi" : "Uni");
 
-  printf ("CLIENT (fd %d): %s-directional Stream Test Complete!\n"
-	  SOCK_TEST_BANNER_STRING "\n", ctrl->fd,
-	  test == SOCK_TEST_TYPE_BI ? "Bi" : "Uni");
+  vtinf ("(fd %d): %s Test Complete!\n"
+	 SOCK_TEST_BANNER_STRING, ctrl->fd, buf);
 }
 
 static void
-exit_client (void)
+vtc_echo_client (vcl_test_client_main_t * vcm)
 {
-  sock_client_main_t *scm = &sock_client_main;
-  sock_test_socket_t *ctrl = &scm->ctrl_socket;
-  sock_test_socket_t *tsock;
-  int i;
+  vcl_test_client_worker_t *wrk;
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
+  sock_test_cfg_t *cfg = &ctrl->cfg;
 
-  for (i = 0; i < ctrl->cfg.num_test_sockets; i++)
-    {
-      tsock = &scm->test_socket[i];
-      tsock->cfg.test = SOCK_TEST_TYPE_EXIT;
+  cfg->total_bytes = strlen (ctrl->txbuf) + 1;
+  memset (&ctrl->stats, 0, sizeof (ctrl->stats));
 
-      /* coverity[COPY_PASTE_ERROR] */
-      if (ctrl->cfg.verbose)
-	{
-	  printf ("\nCLIENT (fd %d): Sending exit cfg to server...\n",
-		  tsock->fd);
-	  sock_test_cfg_dump (&tsock->cfg, 1 /* is_client */ );
-	}
-      (void) vcl_test_write (tsock->fd, (uint8_t *) & tsock->cfg,
-			     sizeof (tsock->cfg), &tsock->stats,
-			     ctrl->cfg.verbose);
-    }
+  /* Echo works with only one worker */
+  wrk = vcm->workers;
+  wrk->wrk_index = 0;
+  wrk->cfg = *cfg;
 
-  ctrl->cfg.test = SOCK_TEST_TYPE_EXIT;
-  if (ctrl->cfg.verbose)
-    {
-      printf ("\nCLIENT (fd %d): Sending exit cfg to server...\n", ctrl->fd);
-      sock_test_cfg_dump (&ctrl->cfg, 1 /* is_client */ );
-    }
-  (void) vcl_test_write (ctrl->fd, (uint8_t *) & ctrl->cfg,
-			 sizeof (ctrl->cfg), &ctrl->stats, ctrl->cfg.verbose);
-  printf ("\nCLIENT: So long and thanks for all the fish!\n\n");
-  sleep (1);
+  vtc_worker_loop (wrk);
+
+  clock_gettime (CLOCK_REALTIME, &ctrl->stats.stop);
+  vtc_accumulate_stats (wrk, ctrl);
+  vtc_print_stats (ctrl);
 }
 
-static int
-sock_test_connect_test_sockets (uint32_t num_test_sockets)
+static void
+vtc_stream_client (vcl_test_client_main_t * vcm)
 {
-  sock_client_main_t *scm = &sock_client_main;
-  sock_test_socket_t *ctrl = &scm->ctrl_socket;
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
+  sock_test_cfg_t *cfg = &ctrl->cfg;
+  vcl_test_client_worker_t *wrk;
   sock_test_socket_t *tsock;
-  int i, rv, errno_val;
+  int tx_bytes, rv;
+  uint32_t i, n, sidx, n_conn, n_conn_per_wrk;
+
+  vtinf (SOCK_TEST_BANNER_STRING "%s-directional Stream Test!\n",
+	 ctrl->cfg.test == SOCK_TEST_TYPE_BI ? "Bi" : "Uni");
 
-  if (num_test_sockets < 1)
+  cfg->total_bytes = cfg->num_writes * cfg->txbuf_size;
+  cfg->ctrl_handle = ~0;
+  if (vtc_cfg_sync (ctrl))
     {
-      errno = EINVAL;
-      return -1;
+      vtwrn ("test cfg sync failed -- aborting!");
+      return;
     }
+  cfg->ctrl_handle = ((sock_test_cfg_t *) ctrl->rxbuf)->ctrl_handle;
+  memset (&ctrl->stats, 0, sizeof (ctrl->stats));
 
-  if (num_test_sockets < scm->num_test_sockets)
+  n_conn = cfg->num_test_sockets;
+  n_conn_per_wrk = n_conn / vcm->n_workers;
+  for (i = 0; i < vcm->n_workers; i++)
     {
-      for (i = scm->num_test_sockets - 1; i >= num_test_sockets; i--)
-	{
-	  tsock = &scm->test_socket[i];
-	  vppcom_session_close (tsock->fd);
-	  free (tsock->txbuf);
-	  free (tsock->rxbuf);
-	}
+      wrk = &vcm->workers[i];
+      wrk->wrk_index = i;
+      wrk->cfg = ctrl->cfg;
+      wrk->cfg.num_test_sockets = vtc_min (n_conn_per_wrk, n_conn);
+      n_conn -= wrk->cfg.num_test_sockets;
     }
 
-  else if (num_test_sockets > scm->num_test_sockets)
+  for (i = 1; i < vcm->n_workers; i++)
     {
-      tsock = realloc (scm->test_socket,
-		       sizeof (sock_test_socket_t) * num_test_sockets);
-      if (!tsock)
-	{
-	  errno_val = errno;
-	  perror ("ERROR in sock_test_connect_test_sockets()");
-	  fprintf (stderr, "CLIENT: ERROR: socket failed (errno = %d)!\n",
-		   errno_val);
-	  return -1;
-	}
+      wrk = &vcm->workers[i];
+      pthread_create (&wrk->thread_handle, NULL, vtc_worker_loop,
+		      (void *) wrk);
+    }
+  vtc_worker_loop (&vcm->workers[0]);
 
-      if (!scm->test_socket)
-	memset (tsock, 0, sizeof (*tsock));
+  while (vcm->active_workers > 0)
+    ;
 
-      scm->test_socket = tsock;
-      for (i = scm->num_test_sockets; i < num_test_sockets; i++)
-	{
-	  tsock = &scm->test_socket[i];
-	  tsock->fd = vppcom_session_create (ctrl->cfg.transport_udp ?
-					     VPPCOM_PROTO_UDP :
-					     VPPCOM_PROTO_TCP,
-					     1 /* is_nonblocking */ );
-	  if (tsock->fd < 0)
-	    {
-	      errno = -tsock->fd;
-	      tsock->fd = -1;
-	    }
-	  if (tsock->fd < 0)
-	    {
-	      errno_val = errno;
-	      perror ("ERROR in sock_test_connect_test_sockets()");
-	      fprintf (stderr, "CLIENT: ERROR: socket failed (errno = %d)!\n",
-		       errno_val);
-	      return tsock->fd;
-	    }
+  clock_gettime (CLOCK_REALTIME, &ctrl->stats.stop);
 
-	  rv = vppcom_session_connect (tsock->fd, &scm->server_endpt);
-	  if (rv)
-	    {
-	      errno = -rv;
-	      rv = -1;
-	    }
-	  if (rv < 0)
-	    {
-	      errno_val = errno;
-	      perror ("ERROR in sock_test_connect_test_sockets()");
-	      fprintf (stderr, "CLIENT: ERROR: connect failed "
-		       "(errno = %d)!\n", errno_val);
-	      return -1;
-	    }
-	  tsock->cfg = ctrl->cfg;
-	  sock_test_socket_buf_alloc (tsock);
-	  sock_test_cfg_sync (tsock);
+  vtinf ("(fd %d): Sending config on ctrl socket for stats...\n", ctrl->fd);
+  if (vtc_cfg_sync (ctrl))
+    {
+      vtwrn ("test cfg sync failed -- aborting!");
+      return;
+    }
+
+  for (i = 0; i < vcm->n_workers; i++)
+    vtc_accumulate_stats (&vcm->workers[i], ctrl);
+  vtc_print_stats (ctrl);
 
-	  printf ("CLIENT (fd %d): Test socket %d connected.\n",
-		  tsock->fd, i);
+  ctrl->cfg.test = SOCK_TEST_TYPE_ECHO;
+  ctrl->cfg.total_bytes = 0;
+  if (vtc_cfg_sync (ctrl))
+    vtwrn ("post-test cfg sync failed!");
+}
+
+static void
+vtc_client_exit (void)
+{
+  vcl_test_client_main_t *vcm = &vcl_client_main;
+  vcl_test_client_worker_t *wrk = &vcm->workers[0];
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
+  sock_test_socket_t *tsock;
+  int i, verbose;
+
+  verbose = ctrl->cfg.verbose;
+  for (i = 0; i < wrk->cfg.num_test_sockets; i++)
+    {
+      tsock = &wrk->sessions[i];
+      tsock->cfg.test = SOCK_TEST_TYPE_EXIT;
+
+      if (verbose)
+	{
+	  vtinf ("(fd %d): Sending exit cfg to server...", tsock->fd);
+	  sock_test_cfg_dump (&tsock->cfg, 1 /* is_client */ );
 	}
+      (void) vcl_test_write (tsock->fd, (uint8_t *) & tsock->cfg,
+			     sizeof (tsock->cfg), &tsock->stats, verbose);
     }
 
-  scm->num_test_sockets = num_test_sockets;
-  printf ("CLIENT: All sockets (%d) connected!\n", scm->num_test_sockets + 1);
-  return 0;
+  ctrl->cfg.test = SOCK_TEST_TYPE_EXIT;
+  if (verbose)
+    {
+      vtinf ("(fd %d): Sending exit cfg to server...", ctrl->fd);
+      sock_test_cfg_dump (&ctrl->cfg, 1 /* is_client */ );
+    }
+  (void) vcl_test_write (ctrl->fd, (uint8_t *) & ctrl->cfg,
+			 sizeof (ctrl->cfg), &ctrl->stats, verbose);
+  vtinf ("So long and thanks for all the fish!\n\n");
+  sleep (1);
 }
 
 static void
@@ -571,8 +563,8 @@ dump_help (void)
 static void
 cfg_txbuf_size_set (void)
 {
-  sock_client_main_t *scm = &sock_client_main;
-  sock_test_socket_t *ctrl = &scm->ctrl_socket;
+  vcl_test_client_main_t *vcm = &vcl_client_main;
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
   char *p = ctrl->txbuf + strlen (SOCK_TEST_TOKEN_TXBUF_SIZE);
   uint64_t txbuf_size = strtoull ((const char *) p, NULL, 10);
 
@@ -593,8 +585,8 @@ cfg_txbuf_size_set (void)
 static void
 cfg_num_writes_set (void)
 {
-  sock_client_main_t *scm = &sock_client_main;
-  sock_test_socket_t *ctrl = &scm->ctrl_socket;
+  vcl_test_client_main_t *vcm = &vcl_client_main;
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
   char *p = ctrl->txbuf + strlen (SOCK_TEST_TOKEN_NUM_WRITES);
   uint32_t num_writes = strtoul ((const char *) p, NULL, 10);
 
@@ -613,8 +605,8 @@ cfg_num_writes_set (void)
 static void
 cfg_num_test_sockets_set (void)
 {
-  sock_client_main_t *scm = &sock_client_main;
-  sock_test_socket_t *ctrl = &scm->ctrl_socket;
+  vcl_test_client_main_t *vcm = &vcl_client_main;
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
   char *p = ctrl->txbuf + strlen (SOCK_TEST_TOKEN_NUM_TEST_SCKTS);
   uint32_t num_test_sockets = strtoul ((const char *) p, NULL, 10);
 
@@ -622,23 +614,20 @@ cfg_num_test_sockets_set (void)
       (num_test_sockets <= SOCK_TEST_CFG_MAX_TEST_SCKTS))
     {
       ctrl->cfg.num_test_sockets = num_test_sockets;
-      sock_test_connect_test_sockets (num_test_sockets);
-
       sock_test_cfg_dump (&ctrl->cfg, 1 /* is_client */ );
     }
   else
     {
-      fprintf (stderr, "CLIENT: ERROR: invalid num test sockets: "
-	       "%u, (%d max)\n",
-	       num_test_sockets, SOCK_TEST_CFG_MAX_TEST_SCKTS);
+      vtwrn ("invalid num test sockets: %u, (%d max)",
+	     num_test_sockets, SOCK_TEST_CFG_MAX_TEST_SCKTS);
     }
 }
 
 static void
 cfg_rxbuf_size_set (void)
 {
-  sock_client_main_t *scm = &sock_client_main;
-  sock_test_socket_t *ctrl = &scm->ctrl_socket;
+  vcl_test_client_main_t *vcm = &vcl_client_main;
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
   char *p = ctrl->txbuf + strlen (SOCK_TEST_TOKEN_RXBUF_SIZE);
   uint64_t rxbuf_size = strtoull ((const char *) p, NULL, 10);
 
@@ -658,8 +647,8 @@ cfg_rxbuf_size_set (void)
 static void
 cfg_verbose_toggle (void)
 {
-  sock_client_main_t *scm = &sock_client_main;
-  sock_test_socket_t *ctrl = &scm->ctrl_socket;
+  vcl_test_client_main_t *vcm = &vcl_client_main;
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
 
   ctrl->cfg.verbose = ctrl->cfg.verbose ? 0 : 1;
   sock_test_cfg_dump (&ctrl->cfg, 1 /* is_client */ );
@@ -669,8 +658,8 @@ cfg_verbose_toggle (void)
 static sock_test_t
 parse_input ()
 {
-  sock_client_main_t *scm = &sock_client_main;
-  sock_test_socket_t *ctrl = &scm->ctrl_socket;
+  vcl_test_client_main_t *vcm = &vcl_client_main;
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
   sock_test_t rv = SOCK_TEST_TYPE_NONE;
 
   if (!strncmp (SOCK_TEST_TOKEN_EXIT, ctrl->txbuf,
@@ -683,7 +672,7 @@ parse_input ()
 
   else if (!strncmp (SOCK_TEST_TOKEN_SHOW_CFG, ctrl->txbuf,
 		     strlen (SOCK_TEST_TOKEN_SHOW_CFG)))
-    scm->dump_cfg = 1;
+    vcm->dump_cfg = 1;
 
   else if (!strncmp (SOCK_TEST_TOKEN_VERBOSE, ctrl->txbuf,
 		     strlen (SOCK_TEST_TOKEN_VERBOSE)))
@@ -741,58 +730,56 @@ print_usage_and_exit (void)
   exit (1);
 }
 
-int
-main (int argc, char **argv)
+static void
+vtc_process_opts (vcl_test_client_main_t * vcm, int argc, char **argv)
 {
-  sock_client_main_t *scm = &sock_client_main;
-  sock_test_socket_t *ctrl = &scm->ctrl_socket;
-  int c, rv, errno_val;
-  sock_test_t post_test = SOCK_TEST_TYPE_NONE;
-
-  sock_test_cfg_init (&ctrl->cfg);
-  sock_test_socket_buf_alloc (ctrl);
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
+  int c, v;
 
   opterr = 0;
   while ((c = getopt (argc, argv, "chn:w:XE:I:N:R:T:UBV6D")) != -1)
     switch (c)
       {
       case 'c':
-	scm->dump_cfg = 1;
+	vcm->dump_cfg = 1;
 	break;
 
       case 's':
 	if (sscanf (optarg, "0x%x", &ctrl->cfg.num_test_sockets) != 1)
 	  if (sscanf (optarg, "%u", &ctrl->cfg.num_test_sockets) != 1)
 	    {
-	      fprintf (stderr, "CLIENT: ERROR: Invalid value for "
-		       "option -%c!\n", c);
+	      vtwrn ("Invalid value for option -%c!", c);
 	      print_usage_and_exit ();
 	    }
 	if (!ctrl->cfg.num_test_sockets ||
 	    (ctrl->cfg.num_test_sockets > FD_SETSIZE))
 	  {
-	    fprintf (stderr, "CLIENT: ERROR: Invalid number of "
-		     "sockets (%d) specified for option -%c!\n"
-		     "       Valid range is 1 - %d\n",
-		     ctrl->cfg.num_test_sockets, c, FD_SETSIZE);
+	    vtwrn ("Invalid number of sockets (%d) specified for option -%c!"
+		   "\n       Valid range is 1 - %d",
+		   ctrl->cfg.num_test_sockets, c, FD_SETSIZE);
 	    print_usage_and_exit ();
 	  }
 	break;
 
       case 'w':
-	fprintf (stderr, "CLIENT: Writing test results to files is TBD.\n");
+	if (sscanf (optarg, "%d", &v) != 1)
+	  {
+	    vtwrn ("Invalid value for option -%c!", c);
+	    print_usage_and_exit ();
+	  }
+	if (v > 1)
+	  vcm->n_workers = v;
 	break;
 
       case 'X':
-	post_test = SOCK_TEST_TYPE_EXIT;
+	vcm->post_test = SOCK_TEST_TYPE_EXIT;
 	break;
 
       case 'E':
 	if (strlen (optarg) > ctrl->txbuf_size)
 	  {
-	    fprintf (stderr, "CLIENT: ERROR: Option -%c value "
-		     "larger than txbuf size (%d)!\n",
-		     optopt, ctrl->txbuf_size);
+	    vtwrn ("Option -%c value larger than txbuf size (%d)!",
+		   optopt, ctrl->txbuf_size);
 	    print_usage_and_exit ();
 	  }
 	strcpy (ctrl->txbuf, optarg);
@@ -803,15 +790,13 @@ main (int argc, char **argv)
 	if (sscanf (optarg, "0x%x", &ctrl->cfg.num_test_sockets) != 1)
 	  if (sscanf (optarg, "%d", &ctrl->cfg.num_test_sockets) != 1)
 	    {
-	      fprintf (stderr, "CLIENT: ERROR: Invalid value for "
-		       "option -%c!\n", c);
+	      vtwrn ("Invalid value for option -%c!", c);
 	      print_usage_and_exit ();
 	    }
 	if (ctrl->cfg.num_test_sockets > SOCK_TEST_CFG_MAX_TEST_SCKTS)
 	  {
-	    fprintf (stderr, "CLIENT: ERROR: value greater than max "
-		     "number test sockets (%d)!",
-		     SOCK_TEST_CFG_MAX_TEST_SCKTS);
+	    vtwrn ("value greater than max number test sockets (%d)!",
+		   SOCK_TEST_CFG_MAX_TEST_SCKTS);
 	    print_usage_and_exit ();
 	  }
 	break;
@@ -820,8 +805,7 @@ main (int argc, char **argv)
 	if (sscanf (optarg, "0x%lx", &ctrl->cfg.num_writes) != 1)
 	  if (sscanf (optarg, "%ld", &ctrl->cfg.num_writes) != 1)
 	    {
-	      fprintf (stderr, "CLIENT: ERROR: Invalid value for "
-		       "option -%c!\n", c);
+	      vtwrn ("Invalid value for option -%c!", c);
 	      print_usage_and_exit ();
 	    }
 	ctrl->cfg.total_bytes = ctrl->cfg.num_writes * ctrl->cfg.txbuf_size;
@@ -831,8 +815,7 @@ main (int argc, char **argv)
 	if (sscanf (optarg, "0x%lx", &ctrl->cfg.rxbuf_size) != 1)
 	  if (sscanf (optarg, "%ld", &ctrl->cfg.rxbuf_size) != 1)
 	    {
-	      fprintf (stderr, "CLIENT: ERROR: Invalid value for "
-		       "option -%c!\n", c);
+	      vtwrn ("Invalid value for option -%c!", c);
 	      print_usage_and_exit ();
 	    }
 	if (ctrl->cfg.rxbuf_size >= SOCK_TEST_CFG_BUF_SIZE_MIN)
@@ -844,9 +827,8 @@ main (int argc, char **argv)
 	  }
 	else
 	  {
-	    fprintf (stderr, "CLIENT: ERROR: rxbuf size (%lu) "
-		     "less than minumum (%u)\n",
-		     ctrl->cfg.rxbuf_size, SOCK_TEST_CFG_BUF_SIZE_MIN);
+	    vtwrn ("rxbuf size (%lu) less than minumum (%u)",
+		   ctrl->cfg.rxbuf_size, SOCK_TEST_CFG_BUF_SIZE_MIN);
 	    print_usage_and_exit ();
 	  }
 
@@ -856,8 +838,7 @@ main (int argc, char **argv)
 	if (sscanf (optarg, "0x%lx", &ctrl->cfg.txbuf_size) != 1)
 	  if (sscanf (optarg, "%ld", &ctrl->cfg.txbuf_size) != 1)
 	    {
-	      fprintf (stderr, "CLIENT: ERROR: Invalid value "
-		       "for option -%c!\n", c);
+	      vtwrn ("Invalid value for option -%c!", c);
 	      print_usage_and_exit ();
 	    }
 	if (ctrl->cfg.txbuf_size >= SOCK_TEST_CFG_BUF_SIZE_MIN)
@@ -871,9 +852,8 @@ main (int argc, char **argv)
 	  }
 	else
 	  {
-	    fprintf (stderr, "CLIENT: ERROR: txbuf size (%lu) "
-		     "less than minumum (%u)!\n",
-		     ctrl->cfg.txbuf_size, SOCK_TEST_CFG_BUF_SIZE_MIN);
+	    vtwrn ("txbuf size (%lu) less than minumum (%u)!",
+		   ctrl->cfg.txbuf_size, SOCK_TEST_CFG_BUF_SIZE_MIN);
 	    print_usage_and_exit ();
 	  }
 	break;
@@ -907,17 +887,14 @@ main (int argc, char **argv)
 	  case 'R':
 	  case 'T':
 	  case 'w':
-	    fprintf (stderr, "CLIENT: ERROR: Option -%c "
-		     "requires an argument.\n", optopt);
+	    vtwrn ("Option -%c requires an argument.", optopt);
 	    break;
 
 	  default:
 	    if (isprint (optopt))
-	      fprintf (stderr, "CLIENT: ERROR: Unknown "
-		       "option `-%c'.\n", optopt);
+	      vtwrn ("Unknown option `-%c'.", optopt);
 	    else
-	      fprintf (stderr, "CLIENT: ERROR: Unknown "
-		       "option character `\\x%x'.\n", optopt);
+	      vtwrn ("Unknown option character `\\x%x'.", optopt);
 	  }
 	/* fall thru */
       case 'h':
@@ -927,118 +904,107 @@ main (int argc, char **argv)
 
   if (argc < (optind + 2))
     {
-      fprintf (stderr, "CLIENT: ERROR: Insufficient number of arguments!\n");
+      vtwrn ("Insufficient number of arguments!");
       print_usage_and_exit ();
     }
+  vcm->proto = ctrl->cfg.transport_udp ? VPPCOM_PROTO_UDP : VPPCOM_PROTO_TCP;
 
-  ctrl->fd = vppcom_app_create ("vcl_test_client");
-  if (ctrl->fd < 0)
-    {
-      errno = -ctrl->fd;
-      ctrl->fd = -1;
-    }
-  else
-    {
-      ctrl->fd = vppcom_session_create (ctrl->cfg.transport_udp ?
-					VPPCOM_PROTO_UDP :
-					VPPCOM_PROTO_TCP,
-					0 /* is_nonblocking */ );
-      if (ctrl->fd < 0)
-	{
-	  errno = -ctrl->fd;
-	  ctrl->fd = -1;
-	}
-    }
-
-  if (ctrl->fd < 0)
-    {
-      errno_val = errno;
-      perror ("ERROR in main()");
-      fprintf (stderr, "CLIENT: ERROR: socket "
-	       "failed (errno = %d)!\n", errno_val);
-      return ctrl->fd;
-    }
-
-  memset (&scm->server_addr, 0, sizeof (scm->server_addr));
+  memset (&vcm->server_addr, 0, sizeof (vcm->server_addr));
   if (ctrl->cfg.address_ip6)
     {
-      struct sockaddr_in6 *server_addr =
-	(struct sockaddr_in6 *) &scm->server_addr;
-      scm->server_addr_size = sizeof (*server_addr);
-      server_addr->sin6_family = AF_INET6;
-      inet_pton (AF_INET6, argv[optind++], &(server_addr->sin6_addr));
-      server_addr->sin6_port = htons (atoi (argv[optind]));
+      struct sockaddr_in6 *sddr6 = (struct sockaddr_in6 *) &vcm->server_addr;
+      sddr6->sin6_family = AF_INET6;
+      inet_pton (AF_INET6, argv[optind++], &(sddr6->sin6_addr));
+      sddr6->sin6_port = htons (atoi (argv[optind]));
+
+      vcm->server_endpt.is_ip4 = 0;
+      vcm->server_endpt.ip = (uint8_t *) & sddr6->sin6_addr;
+      vcm->server_endpt.port = (uint16_t) sddr6->sin6_port;
     }
   else
     {
-      struct sockaddr_in *server_addr =
-	(struct sockaddr_in *) &scm->server_addr;
-      scm->server_addr_size = sizeof (*server_addr);
-      server_addr->sin_family = AF_INET;
-      inet_pton (AF_INET, argv[optind++], &(server_addr->sin_addr));
-      server_addr->sin_port = htons (atoi (argv[optind]));
+      struct sockaddr_in *saddr4 = (struct sockaddr_in *) &vcm->server_addr;
+      saddr4->sin_family = AF_INET;
+      inet_pton (AF_INET, argv[optind++], &(saddr4->sin_addr));
+      saddr4->sin_port = htons (atoi (argv[optind]));
+
+      vcm->server_endpt.is_ip4 = 1;
+      vcm->server_endpt.ip = (uint8_t *) & saddr4->sin_addr;
+      vcm->server_endpt.port = (uint16_t) saddr4->sin_port;
     }
+}
 
-  if (ctrl->cfg.address_ip6)
-    {
-      struct sockaddr_in6 *server_addr =
-	(struct sockaddr_in6 *) &scm->server_addr;
-      scm->server_endpt.is_ip4 = 0;
-      scm->server_endpt.ip = (uint8_t *) & server_addr->sin6_addr;
-      scm->server_endpt.port = (uint16_t) server_addr->sin6_port;
-    }
-  else
-    {
-      struct sockaddr_in *server_addr =
-	(struct sockaddr_in *) &scm->server_addr;
-      scm->server_endpt.is_ip4 = 1;
-      scm->server_endpt.ip = (uint8_t *) & server_addr->sin_addr;
-      scm->server_endpt.port = (uint16_t) server_addr->sin_port;
-    }
+static void
+vtc_read_user_input (sock_test_socket_t * ctrl)
+{
+  printf ("\nType some characters and hit <return>\n"
+	  "('" SOCK_TEST_TOKEN_HELP "' for help): ");
 
-  do
+  if (fgets (ctrl->txbuf, ctrl->txbuf_size, stdin) != NULL)
     {
-      printf ("\nCLIENT: Connecting to server...\n");
-
-      rv = vppcom_session_connect (ctrl->fd, &scm->server_endpt);
-      if (rv)
-	{
-	  errno = -rv;
-	  rv = -1;
-	}
-      if (rv < 0)
+      if (strlen (ctrl->txbuf) == 1)
 	{
-	  errno_val = errno;
-	  perror ("ERROR in main()");
-	  fprintf (stderr, "CLIENT: ERROR: connect failed (errno = %d)!\n",
-		   errno_val);
-	  return -1;
+	  printf ("\nNothing to send!  Please try again...\n");
+	  return;
 	}
+      ctrl->txbuf[strlen (ctrl->txbuf) - 1] = 0;	// chomp the newline.
 
-      sock_test_cfg_sync (ctrl);
-      printf ("CLIENT (fd %d): Control socket connected.\n", ctrl->fd);
+      /* Parse input for keywords */
+      ctrl->cfg.test = parse_input ();
     }
-  while (rv < 0);
+}
 
-  sock_test_connect_test_sockets (ctrl->cfg.num_test_sockets);
+int
+main (int argc, char **argv)
+{
+  vcl_test_client_main_t *vcm = &vcl_client_main;
+  sock_test_socket_t *ctrl = &vcm->ctrl_socket;
+  int rv, errno_val;
+
+  vcm->n_workers = 1;
+  sock_test_cfg_init (&ctrl->cfg);
+  sock_test_socket_buf_alloc (ctrl);
+  vtc_process_opts (vcm, argc, argv);
+
+  vcm->workers = calloc (vcm->n_workers, sizeof (vcl_test_client_worker_t));
+  rv = vppcom_app_create ("vcl_test_client");
+  if (rv < 0)
+    vtfail ("vppcom_app_create()", rv);
+
+  ctrl->fd = vppcom_session_create (vcm->proto, 0 /* is_nonblocking */ );
+  if (ctrl->fd < 0)
+    vtfail ("vppcom_session_create()", ctrl->fd);
+
+  vtinf ("Connecting to server...");
+  rv = vppcom_session_connect (ctrl->fd, &vcm->server_endpt);
+  if (rv)
+    vtfail ("vppcom_session_connect()", rv);
+  vtinf ("(fd %d): Control socket connected.", ctrl->fd);
+
+  rv = vtc_cfg_sync (ctrl);
+  if (rv)
+    vtfail ("vtc_cfg_sync()", rv);
+
+  ctrl->cfg.ctrl_handle = ((sock_test_cfg_t *) ctrl->rxbuf)->ctrl_handle;
+  memset (&ctrl->stats, 0, sizeof (ctrl->stats));
 
   while (ctrl->cfg.test != SOCK_TEST_TYPE_EXIT)
     {
-      if (scm->dump_cfg)
+      if (vcm->dump_cfg)
 	{
 	  sock_test_cfg_dump (&ctrl->cfg, 1 /* is_client */ );
-	  scm->dump_cfg = 0;
+	  vcm->dump_cfg = 0;
 	}
 
       switch (ctrl->cfg.test)
 	{
 	case SOCK_TEST_TYPE_ECHO:
-	  echo_test_client ();
+	  vtc_echo_client (vcm);
 	  break;
 
 	case SOCK_TEST_TYPE_UNI:
 	case SOCK_TEST_TYPE_BI:
-	  stream_test_client (ctrl->cfg.test);
+	  vtc_stream_client (vcm);
 	  break;
 
 	case SOCK_TEST_TYPE_EXIT:
@@ -1048,7 +1014,7 @@ main (int argc, char **argv)
 	default:
 	  break;
 	}
-      switch (post_test)
+      switch (vcm->post_test)
 	{
 	case SOCK_TEST_TYPE_EXIT:
 	  switch (ctrl->cfg.test)
@@ -1077,26 +1043,13 @@ main (int argc, char **argv)
       memset (ctrl->txbuf, 0, ctrl->txbuf_size);
       memset (ctrl->rxbuf, 0, ctrl->rxbuf_size);
 
-      printf ("\nCLIENT: Type some characters and hit <return>\n"
-	      "('" SOCK_TEST_TOKEN_HELP "' for help): ");
-
-      if (fgets (ctrl->txbuf, ctrl->txbuf_size, stdin) != NULL)
-	{
-	  if (strlen (ctrl->txbuf) == 1)
-	    {
-	      printf ("\nCLIENT: Nothing to send!  Please try again...\n");
-	      continue;
-	    }
-	  ctrl->txbuf[strlen (ctrl->txbuf) - 1] = 0;	// chomp the newline.
-
-	  /* Parse input for keywords */
-	  ctrl->cfg.test = parse_input ();
-	}
+      vtc_read_user_input (ctrl);
     }
 
-  exit_client ();
+  vtc_client_exit ();
   vppcom_session_close (ctrl->fd);
   vppcom_app_destroy ();
+  free (vcm->workers);
   return 0;
 }