session: segment manager improvements

- cleanup connects segment manager even if first
- fix segment manager allocation for listen sessions
- improve handling of process private segments (mheaps/main heap)
- added segment manager cli

Change-Id: Ic2ca97c3622ab2286d5fb5772aeb57680e64f769
Signed-off-by: Florin Coras <fcoras@cisco.com>
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>

12 files changed, 407 insertions, 190 deletions

diff --git a/src/svm/svm_fifo_segment.c b/src/svm/svm_fifo_segment.c
index 2094ba74..6600a423 100644
--- a/src/svm/svm_fifo_segment.c
+++ b/src/svm/svm_fifo_segment.c
@@ -188,7 +188,7 @@ svm_fifo_segment_create (svm_fifo_segment_create_args_t * a)
       return (rv);
     }
 
-  /* Note; requested_va updated due to seg base addr randomization */
+  /* Note: requested_va updated due to seg base addr randomization */
   sm->next_baseva = s->ssvm.requested_va + a->segment_size;
 
   sh = s->ssvm.sh;
@@ -200,7 +200,6 @@ svm_fifo_segment_create (svm_fifo_segment_create_args_t * a)
   sh->opaque[0] = fsh;
   s->h = fsh;
   fsh->segment_name = format (0, "%s%c", a->segment_name, 0);
-
   preallocate_fifo_pairs (fsh, a);
 
   ssvm_pop_heap (oldheap);
@@ -226,25 +225,19 @@ svm_fifo_segment_create_process_private (svm_fifo_segment_create_args_t * a)
 
   if (a->private_segment_count && a->private_segment_size)
     {
-      void *mem;
       u8 *heap;
       u32 pagesize = clib_mem_get_page_size ();
       u32 rnd_size;
+      rnd_size = (a->private_segment_size + (pagesize - 1)) & ~pagesize;
 
       for (i = 0; i < a->private_segment_count; i++)
 	{
-	  rnd_size = (a->private_segment_size + (pagesize - 1)) & ~pagesize;
-
-	  mem = mmap (0, rnd_size, PROT_READ | PROT_WRITE,
-		      MAP_PRIVATE | MAP_ANONYMOUS,
-		      -1 /* fd */ , 0 /* offset */ );
-
-	  if (mem == MAP_FAILED)
+	  heap = mheap_alloc (0, rnd_size);
+	  if (heap == 0)
 	    {
-	      clib_unix_warning ("mmap");
+	      clib_unix_warning ("mheap alloc");
 	      return -1;
 	    }
-	  heap = mheap_alloc (mem, rnd_size);
 	  heap_header = mheap_header (heap);
 	  heap_header->flags |= MHEAP_FLAG_THREAD_SAFE;
 	  vec_add1 (heaps, heap);
@@ -279,6 +272,9 @@ svm_fifo_segment_create_process_private (svm_fifo_segment_create_args_t * a)
       memset (fsh, 0, sizeof (*fsh));
       sh->opaque[0] = fsh;
       s->h = fsh;
+      fsh->flags = FIFO_SEGMENT_F_IS_PRIVATE;
+      if (!a->private_segment_count)
+	fsh->flags |= FIFO_SEGMENT_F_IS_MAIN_HEAP;
       fsh->segment_name = format (0, "%s%c", a->segment_name, 0);
 
       if (a->private_segment_count)
@@ -288,7 +284,6 @@ svm_fifo_segment_create_process_private (svm_fifo_segment_create_args_t * a)
 	  preallocate_fifo_pairs (fsh, a);
 	  clib_mem_set_heap (oldheap);
 	}
-
       sh->ready = 1;
       vec_add1 (a->new_segment_indices, s - sm->segments);
     }
@@ -336,8 +331,20 @@ void
 svm_fifo_segment_delete (svm_fifo_segment_private_t * s)
 {
   svm_fifo_segment_main_t *sm = &svm_fifo_segment_main;
-  ssvm_delete (&s->ssvm);
-  pool_put (sm->segments, s);
+  if (s->h->flags & FIFO_SEGMENT_F_IS_PRIVATE)
+    {
+      /* Don't try to free vpp's heap! */
+      if (!(s->h->flags & FIFO_SEGMENT_F_IS_MAIN_HEAP))
+	mheap_free (s->ssvm.sh->heap);
+      clib_mem_free (s->ssvm.sh);
+      clib_mem_free (s->h);
+      pool_put (sm->segments, s);
+    }
+  else
+    {
+      ssvm_delete (&s->ssvm);
+      pool_put (sm->segments, s);
+    }
 }
 
 svm_fifo_t *
@@ -352,8 +359,8 @@ svm_fifo_segment_alloc_fifo (svm_fifo_segment_private_t * s,
   int freelist_index;
 
   /*
-   * 2K minimum. It's not likely that anything good will happen
-   * with a 1K FIFO.
+   * 4K minimum. It's not likely that anything good will happen
+   * with a smaller FIFO.
    */
   if (data_size_in_bytes < FIFO_SEGMENT_MIN_FIFO_SIZE ||
       data_size_in_bytes > FIFO_SEGMENT_MAX_FIFO_SIZE)
@@ -428,6 +435,7 @@ found:
 	}
       fsh->fifos = f;
     }
+  fsh->n_active_fifos++;
 
   ssvm_pop_heap (oldheap);
   ssvm_unlock_non_recursive (sh);
@@ -489,6 +497,7 @@ svm_fifo_segment_free_fifo (svm_fifo_segment_private_t * s, svm_fifo_t * f,
       f->master_thread_index = ~0;
     }
 
+  fsh->n_active_fifos--;
   ssvm_pop_heap (oldheap);
   ssvm_unlock_non_recursive (sh);
 }
@@ -508,6 +517,25 @@ svm_fifo_segment_index (svm_fifo_segment_private_t * s)
   return s - svm_fifo_segment_main.segments;
 }
 
+/**
+ * Retrieve svm segments pool. Used only for debug purposes.
+ */
+svm_fifo_segment_private_t *
+svm_fifo_segment_segments_pool (void)
+{
+  svm_fifo_segment_main_t *sm = &svm_fifo_segment_main;
+  return sm->segments;
+}
+
+/**
+ * Get number of active fifos
+ */
+u32
+svm_fifo_segment_num_fifos (svm_fifo_segment_private_t * fifo_segment)
+{
+  return fifo_segment->h->n_active_fifos;
+}
+
 /*
  * fd.io coding-style-patch-verification: ON
  *
diff --git a/src/svm/svm_fifo_segment.h b/src/svm/svm_fifo_segment.h
index 68bb4d3d..5cc4754a 100644
--- a/src/svm/svm_fifo_segment.h
+++ b/src/svm/svm_fifo_segment.h
@@ -27,15 +27,20 @@ typedef enum
   FIFO_SEGMENT_N_FREELISTS
 } svm_fifo_segment_freelist_t;
 
-#define FIFO_SEGMENT_MIN_FIFO_SIZE 2048
+#define FIFO_SEGMENT_MIN_FIFO_SIZE 4096
 #define FIFO_SEGMENT_MAX_FIFO_SIZE (8<<20)	/* 8mb max fifo size */
-#define FIFO_SEGMENT_ALLOC_CHUNK_SIZE 32	/* allocation quantum */
+#define FIFO_SEGMENT_ALLOC_CHUNK_SIZE 32	/* Allocation quantum */
+
+#define FIFO_SEGMENT_F_IS_PRIVATE 	1 << 0	/* Private segment */
+#define FIFO_SEGMENT_F_IS_MAIN_HEAP	1 << 1	/* Segment is main heap */
 
 typedef struct
 {
   svm_fifo_t *fifos;		/**< Linked list of active RX fifos */
   u8 *segment_name;		/**< Segment name */
   svm_fifo_t **free_fifos;	/**< Freelists, by fifo size  */
+  u32 n_active_fifos;		/**< Number of active fifos */
+  u8 flags;			/**< Segment flags */
 } svm_fifo_segment_header_t;
 
 typedef struct
@@ -70,7 +75,7 @@ typedef struct
 } svm_fifo_segment_create_args_t;
 
 static inline svm_fifo_segment_private_t *
-svm_fifo_get_segment (u32 segment_index)
+svm_fifo_segment_get_segment (u32 segment_index)
 {
   svm_fifo_segment_main_t *ssm = &svm_fifo_segment_main;
   return vec_elt_at_index (ssm->segments, segment_index);
@@ -112,6 +117,8 @@ void svm_fifo_segment_free_fifo (svm_fifo_segment_private_t * s,
 				 svm_fifo_segment_freelist_t index);
 void svm_fifo_segment_init (u64 baseva, u32 timeout_in_seconds);
 u32 svm_fifo_segment_index (svm_fifo_segment_private_t * s);
+u32 svm_fifo_segment_num_fifos (svm_fifo_segment_private_t * fifo_segment);
+svm_fifo_segment_private_t *svm_fifo_segment_segments_pool (void);
 
 #endif /* __included_ssvm_fifo_segment_h__ */
 
diff --git a/src/svm/test_svm_fifo1.c b/src/svm/test_svm_fifo1.c
index 63d75845..3bdca949 100644
--- a/src/svm/test_svm_fifo1.c
+++ b/src/svm/test_svm_fifo1.c
@@ -39,7 +39,7 @@ hello_world (int verbose)
   if (rv)
     return clib_error_return (0, "svm_fifo_segment_create returned %d", rv);
 
-  sp = svm_fifo_get_segment (a->new_segment_indices[0]);
+  sp = svm_fifo_segment_get_segment (a->new_segment_indices[0]);
 
   f = svm_fifo_segment_alloc_fifo (sp, 4096, FIFO_SEGMENT_RX_FREELIST);
 
@@ -92,7 +92,7 @@ master (int verbose)
   if (rv)
     return clib_error_return (0, "svm_fifo_segment_create returned %d", rv);
 
-  sp = svm_fifo_get_segment (a->new_segment_indices[0]);
+  sp = svm_fifo_segment_get_segment (a->new_segment_indices[0]);
 
   f = svm_fifo_segment_alloc_fifo (sp, 4096, FIFO_SEGMENT_RX_FREELIST);
 
@@ -128,7 +128,7 @@ mempig (int verbose)
   if (rv)
     return clib_error_return (0, "svm_fifo_segment_create returned %d", rv);
 
-  sp = svm_fifo_get_segment (a->new_segment_indices[0]);
+  sp = svm_fifo_segment_get_segment (a->new_segment_indices[0]);
 
   for (i = 0; i < 1000; i++)
     {
@@ -186,7 +186,7 @@ offset (int verbose)
   if (rv)
     return clib_error_return (0, "svm_fifo_segment_create returned %d", rv);
 
-  sp = svm_fifo_get_segment (a->new_segment_indices[0]);
+  sp = svm_fifo_segment_get_segment (a->new_segment_indices[0]);
 
   f = svm_fifo_segment_alloc_fifo (sp, 200 << 10, FIFO_SEGMENT_RX_FREELIST);
 
@@ -246,7 +246,7 @@ slave (int verbose)
   if (rv)
     return clib_error_return (0, "svm_fifo_segment_attach returned %d", rv);
 
-  sp = svm_fifo_get_segment (a->new_segment_indices[0]);
+  sp = svm_fifo_segment_get_segment (a->new_segment_indices[0]);
   sh = sp->ssvm.sh;
   fsh = (svm_fifo_segment_header_t *) sh->opaque[0];
 
diff --git a/src/uri/vppcom.c b/src/uri/vppcom.c
index 9a4bea8f..c41acd54 100644
--- a/src/uri/vppcom.c
+++ b/src/uri/vppcom.c
@@ -1076,7 +1076,7 @@ vl_api_connect_sock_t_handler (vl_api_connect_sock_t * mp)
   session->sm_seg_index = a->new_segment_indices[0];
   vec_reset_length (a->new_segment_indices);
 
-  seg = svm_fifo_get_segment (session->sm_seg_index);
+  seg = svm_fifo_segment_get_segment (session->sm_seg_index);
   rx_fifo = session->server_rx_fifo =
     svm_fifo_segment_alloc_fifo (seg, vcm->cfg.rx_fifo_size,
 				 FIFO_SEGMENT_RX_FREELIST);
diff --git a/src/vnet/session/application.c b/src/vnet/session/application.c
index 25a4efa5..d105119c 100644
--- a/src/vnet/session/application.c
+++ b/src/vnet/session/application.c
@@ -80,8 +80,8 @@ application_new ()
   pool_get (app_pool, app);
   memset (app, 0, sizeof (*app));
   app->index = application_get_index (app);
-  app->connects_seg_manager = ~0;
-  app->first_segment_manager = ~0;
+  app->connects_seg_manager = APP_INVALID_SEGMENT_MANAGER_INDEX;
+  app->first_segment_manager = APP_INVALID_SEGMENT_MANAGER_INDEX;
   if (CLIB_DEBUG > 1)
     clib_warning ("[%d] New app (%d)", getpid (), app->index);
   return app;
@@ -104,14 +104,8 @@ application_del (application_t * app)
     clib_warning ("[%d] Delete app (%d)", getpid (), app->index);
 
   /*
-   * Cleanup segment managers
+   *  Listener cleanup
    */
-  if ((app->connects_seg_manager != (u32) ~ 0) &&
-      (app->connects_seg_manager != app->first_segment_manager))
-    {
-      sm = segment_manager_get (app->connects_seg_manager);
-      segment_manager_del (sm);
-    }
 
   /* *INDENT-OFF* */
   hash_foreach (handle, index, app->listeners_table,
@@ -120,7 +114,6 @@ application_del (application_t * app)
   }));
   /* *INDENT-ON* */
 
-  /* Actual listener cleanup */
   for (i = 0; i < vec_len (handles); i++)
     {
       a->app_index = app->index;
@@ -129,10 +122,30 @@ application_del (application_t * app)
       vnet_unbind (a);
     }
 
-  if (app->first_segment_manager != ~0)
+  /*
+   * Connects segment manager cleanup
+   */
+
+  if (app->connects_seg_manager != APP_INVALID_SEGMENT_MANAGER_INDEX)
+    {
+      sm = segment_manager_get (app->connects_seg_manager);
+      sm->app_index = SEGMENT_MANAGER_INVALID_APP_INDEX;
+      segment_manager_init_del (sm);
+    }
+
+
+  /* If first segment manager is used by a listener */
+  if (app->first_segment_manager != APP_INVALID_SEGMENT_MANAGER_INDEX
+      && app->first_segment_manager != app->connects_seg_manager)
     {
       sm = segment_manager_get (app->first_segment_manager);
-      segment_manager_first_segment_maybe_del (sm);
+      /* .. and has no fifos, e.g. it might be used for redirected sessions,
+       * remove it */
+      if (!segment_manager_has_fifos (sm))
+	{
+	  sm->app_index = SEGMENT_MANAGER_INVALID_APP_INDEX;
+	  segment_manager_del (sm);
+	}
     }
 
   application_table_del (app);
@@ -159,6 +172,7 @@ application_init (application_t * app, u32 api_client_index, u64 * options,
   segment_manager_t *sm;
   segment_manager_properties_t *props;
   u32 app_evt_queue_size, first_seg_size;
+  u32 default_rx_fifo_size = 16 << 10, default_tx_fifo_size = 16 << 10;
   int rv;
 
   app_evt_queue_size = options[APP_EVT_QUEUE_SIZE] > 0 ?
@@ -170,7 +184,11 @@ application_init (application_t * app, u32 api_client_index, u64 * options,
   props = &app->sm_properties;
   props->add_segment_size = options[SESSION_OPTIONS_ADD_SEGMENT_SIZE];
   props->rx_fifo_size = options[SESSION_OPTIONS_RX_FIFO_SIZE];
+  props->rx_fifo_size =
+    props->rx_fifo_size ? props->rx_fifo_size : default_rx_fifo_size;
   props->tx_fifo_size = options[SESSION_OPTIONS_TX_FIFO_SIZE];
+  props->tx_fifo_size =
+    props->tx_fifo_size ? props->tx_fifo_size : default_tx_fifo_size;
   props->add_segment = props->add_segment_size != 0;
   props->preallocated_fifo_pairs = options[APP_OPTIONS_PREALLOC_FIFO_PAIRS];
   props->use_private_segment = options[APP_OPTIONS_FLAGS]
@@ -181,6 +199,7 @@ application_init (application_t * app, u32 api_client_index, u64 * options,
   first_seg_size = options[SESSION_OPTIONS_SEGMENT_SIZE];
   if ((rv = segment_manager_init (sm, props, first_seg_size)))
     return rv;
+  sm->first_is_protected = 1;
 
   app->first_segment_manager = segment_manager_index (sm);
   app->api_client_index = api_client_index;
@@ -225,7 +244,8 @@ application_alloc_segment_manager (application_t * app)
 {
   segment_manager_t *sm = 0;
 
-  if (app->first_segment_manager != (u32) ~ 0
+  /* If the first segment manager is not in use, don't allocate a new one */
+  if (app->first_segment_manager != APP_INVALID_SEGMENT_MANAGER_INDEX
       && app->first_segment_manager_in_use == 0)
     {
       sm = segment_manager_get (app->first_segment_manager);
@@ -302,11 +322,15 @@ application_stop_listen (application_t * srv, u64 handle)
   ASSERT (indexp);
 
   sm = segment_manager_get (*indexp);
-  segment_manager_del (sm);
   if (srv->first_segment_manager == *indexp)
     {
+      /* Delete sessions but don't remove segment manager */
       srv->first_segment_manager_in_use = 0;
-      srv->first_segment_manager = ~0;
+      segment_manager_del_sessions (sm);
+    }
+  else
+    {
+      segment_manager_init_del (sm);
     }
   hash_unset (srv->listeners_table, handle);
   listen_session_del (listener);
@@ -379,6 +403,21 @@ application_is_proxy (application_t * app)
   return !(app->flags & APP_OPTIONS_FLAGS_IS_PROXY);
 }
 
+int
+application_add_segment_notify (u32 app_index, u32 fifo_segment_index)
+{
+  application_t *app = application_get (app_index);
+  u32 seg_size = 0;
+  u8 *seg_name;
+
+  /* Send an API message to the external app, to map new segment */
+  ASSERT (app->cb_fns.add_segment_callback);
+
+  segment_manager_get_segment_info (fifo_segment_index, &seg_name, &seg_size);
+  return app->cb_fns.add_segment_callback (app->api_client_index, seg_name,
+					   seg_size);
+}
+
 u8 *
 format_application_listener (u8 * s, va_list * args)
 {
@@ -449,7 +488,7 @@ application_format_connects (application_t * app, int verbose)
       svm_fifo_t *fifo;
       u8 *str;
 
-      fifo_segment = svm_fifo_get_segment (sm->segment_indices[j]);
+      fifo_segment = svm_fifo_segment_get_segment (sm->segment_indices[j]);
       fifo = svm_fifo_segment_get_fifo_list (fifo_segment);
       while (fifo)
 	{
diff --git a/src/vnet/session/application.h b/src/vnet/session/application.h
index 95a39c05..e030c376 100644
--- a/src/vnet/session/application.h
+++ b/src/vnet/session/application.h
@@ -82,9 +82,12 @@ typedef struct _application
    */
   u32 connects_seg_manager;
 
-  /* Lookup tables for listeners. Value is segment manager index */
+  /** Lookup tables for listeners. Value is segment manager index */
   uword *listeners_table;
 
+  /** First segment manager has in the the first segment the application's
+   * event fifo. Depending on what the app does, it may be either used for
+   * a listener or for connects. */
   u32 first_segment_manager;
   u8 first_segment_manager_in_use;
 
@@ -92,6 +95,8 @@ typedef struct _application
   segment_manager_properties_t sm_properties;
 } application_t;
 
+#define APP_INVALID_SEGMENT_MANAGER_INDEX ((u32) ~0)
+
 application_t *application_new ();
 int
 application_init (application_t * app, u32 api_client_index, u64 * options,
@@ -118,6 +123,7 @@ segment_manager_t *application_get_listen_segment_manager (application_t *
 segment_manager_t *application_get_connect_segment_manager (application_t *
 							    app);
 int application_is_proxy (application_t * app);
+int application_add_segment_notify (u32 app_index, u32 fifo_segment_index);
 
 #endif /* SRC_VNET_SESSION_APPLICATION_H_ */
 
diff --git a/src/vnet/session/segment_manager.c b/src/vnet/session/segment_manager.c
index 43977063..7cf66ee3 100644
--- a/src/vnet/session/segment_manager.c
+++ b/src/vnet/session/segment_manager.c
@@ -42,7 +42,7 @@ void
 segment_manager_get_segment_info (u32 index, u8 ** name, u32 * size)
 {
   svm_fifo_segment_private_t *s;
-  s = svm_fifo_get_segment (index);
+  s = svm_fifo_segment_get_segment (index);
   *name = s->h->segment_name;
   *size = s->ssvm.ssvm_size;
 }
@@ -56,23 +56,39 @@ session_manager_add_segment_i (segment_manager_t * sm, u32 segment_size,
 
   memset (ca, 0, sizeof (*ca));
 
-  ca->segment_name = (char *) segment_name;
-  ca->segment_size = segment_size;
-  ca->rx_fifo_size = sm->properties->rx_fifo_size;
-  ca->tx_fifo_size = sm->properties->tx_fifo_size;
-  ca->preallocated_fifo_pairs = sm->properties->preallocated_fifo_pairs;
+  if (!sm->properties->use_private_segment)
+    {
+      ca->segment_name = (char *) segment_name;
+      ca->segment_size = segment_size;
+      ca->rx_fifo_size = sm->properties->rx_fifo_size;
+      ca->tx_fifo_size = sm->properties->tx_fifo_size;
+      ca->preallocated_fifo_pairs = sm->properties->preallocated_fifo_pairs;
 
-  rv = svm_fifo_segment_create (ca);
-  if (rv)
+      rv = svm_fifo_segment_create (ca);
+      if (rv)
+	{
+	  clib_warning ("svm_fifo_segment_create ('%s', %d) failed",
+			ca->segment_name, ca->segment_size);
+	  return VNET_API_ERROR_SVM_SEGMENT_CREATE_FAIL;
+	}
+    }
+  else
     {
-      clib_warning ("svm_fifo_segment_create ('%s', %d) failed",
-		    ca->segment_name, ca->segment_size);
-      return VNET_API_ERROR_SVM_SEGMENT_CREATE_FAIL;
+      ca->segment_name = "process-private-segment";
+      ca->segment_size = ~0;
+      ca->rx_fifo_size = sm->properties->rx_fifo_size;
+      ca->tx_fifo_size = sm->properties->tx_fifo_size;
+      ca->preallocated_fifo_pairs = sm->properties->preallocated_fifo_pairs;
+      ca->private_segment_count = sm->properties->private_segment_count;
+      ca->private_segment_size = sm->properties->private_segment_size;
+
+      if (svm_fifo_segment_create_process_private (ca))
+	clib_warning ("Failed to create process private segment");
+
+      ASSERT (vec_len (ca->new_segment_indices));
     }
-
   vec_append (sm->segment_indices, ca->new_segment_indices);
   vec_free (ca->new_segment_indices);
-
   return 0;
 }
 
@@ -106,29 +122,13 @@ session_manager_add_first_segment (segment_manager_t * sm, u32 segment_size)
   return rv;
 }
 
-static void
-  segment_manager_alloc_process_private_segment
-  (segment_manager_properties_t * props)
+segment_manager_t *
+segment_manager_new ()
 {
-  svm_fifo_segment_create_args_t _a, *a = &_a;
-
-  if (private_segment_indices)
-    return;
-
-  memset (a, 0, sizeof (*a));
-  a->segment_name = "process-private-segment";
-  a->segment_size = ~0;
-  a->rx_fifo_size = props->rx_fifo_size;
-  a->tx_fifo_size = props->tx_fifo_size;
-  a->preallocated_fifo_pairs = props->preallocated_fifo_pairs;
-  a->private_segment_count = props->private_segment_count;
-  a->private_segment_size = props->private_segment_size;
-
-  if (svm_fifo_segment_create_process_private (a))
-    clib_warning ("Failed to create process private segment");
-
-  private_segment_indices = a->new_segment_indices;
-  ASSERT (vec_len (private_segment_indices));
+  segment_manager_t *sm;
+  pool_get (segment_managers, sm);
+  memset (sm, 0, sizeof (*sm));
+  return sm;
 }
 
 /**
@@ -147,65 +147,73 @@ segment_manager_init (segment_manager_t * sm,
 
   first_seg_size = first_seg_size > 0 ? first_seg_size : default_segment_size;
 
-  if (sm->properties->use_private_segment == 0)
-    {
-      rv = session_manager_add_first_segment (sm, first_seg_size);
-      if (rv)
-	{
-	  clib_warning ("Failed to allocate segment");
-	  return rv;
-	}
-    }
-  else
+  rv = session_manager_add_first_segment (sm, first_seg_size);
+  if (rv)
     {
-      if (vec_len (private_segment_indices) == 0)
-	segment_manager_alloc_process_private_segment (properties);
-      ASSERT (vec_len (private_segment_indices));
-      vec_append (sm->segment_indices, private_segment_indices);
+      clib_warning ("Failed to allocate segment");
+      return rv;
     }
 
   clib_spinlock_init (&sm->lockp);
   return 0;
 }
 
-void
-segment_manager_first_segment_maybe_del (segment_manager_t * sm)
+u8
+segment_manager_has_fifos (segment_manager_t * sm)
 {
-  svm_fifo_segment_private_t *fifo_segment;
-
-  /* If the first semgment has no fifos, then delete the 1st segment
-   */
-  fifo_segment = svm_fifo_get_segment (sm->segment_indices[0]);
-  if (!svm_fifo_segment_has_fifos (fifo_segment))
+  svm_fifo_segment_private_t *segment;
+  /* Weird, but handle it */
+  if (vec_len (sm->segment_indices) == 0)
+    return 0;
+  if (vec_len (sm->segment_indices) == 1)
     {
-      clib_spinlock_lock (&sm->lockp);
-      svm_fifo_segment_delete (fifo_segment);
-      vec_del1 (sm->segment_indices, 0);
-      clib_spinlock_unlock (&sm->lockp);
+      segment = svm_fifo_segment_get_segment (sm->segment_indices[0]);
+      if (svm_fifo_segment_num_fifos (segment) == 0)
+	return 0;
     }
+  if (CLIB_DEBUG)
+    {
+      svm_fifo_segment_private_t *segment;
+      int i;
+      for (i = 1; i < vec_len (sm->segment_indices); i++)
+	{
+	  segment = svm_fifo_segment_get_segment (sm->segment_indices[i]);
+	  if (!svm_fifo_segment_has_fifos (segment))
+	    clib_warning ("segment has no fifos!");
+	}
+    }
+  return 1;
 }
 
-  /**
- * Removes segment manager.
- *
- * Since the fifos allocated in the segment keep backpointers to the sessions
- * prior to removing the segment, we call session disconnect. This
- * subsequently propages into transport.
+static void
+segment_manager_del_segment (segment_manager_t * sm, u32 segment_index)
+{
+  svm_fifo_segment_private_t *fifo_segment;
+  u32 svm_segment_index;
+  clib_spinlock_lock (&sm->lockp);
+  svm_segment_index = sm->segment_indices[segment_index];
+  fifo_segment = svm_fifo_segment_get_segment (svm_segment_index);
+  svm_fifo_segment_delete (fifo_segment);
+  vec_del1 (sm->segment_indices, segment_index);
+  clib_spinlock_unlock (&sm->lockp);
+}
+
+/**
+ * Initiate disconnects for all sessions 'owned' by a segment manager
  */
 void
-segment_manager_del (segment_manager_t * sm)
+segment_manager_del_sessions (segment_manager_t * sm)
 {
   int j;
   svm_fifo_segment_private_t *fifo_segment;
+  svm_fifo_t *fifo;
+
   ASSERT (vec_len (sm->segment_indices));
 
   /* Across all fifo segments used by the server */
   for (j = 0; j < vec_len (sm->segment_indices); j++)
     {
-      svm_fifo_t *fifo;
-
-      /* Vector of fifos allocated in the segment */
-      fifo_segment = svm_fifo_get_segment (sm->segment_indices[j]);
+      fifo_segment = svm_fifo_segment_get_segment (sm->segment_indices[j]);
       fifo = svm_fifo_segment_get_fifo_list (fifo_segment);
 
       /*
@@ -220,14 +228,17 @@ segment_manager_del (segment_manager_t * sm)
 
 	  session_index = fifo->master_session_index;
 	  thread_index = fifo->master_thread_index;
-
 	  session = stream_session_get (session_index, thread_index);
 
 	  /* Instead of directly removing the session call disconnect */
-	  session->session_state = SESSION_STATE_CLOSED;
-	  session_send_session_evt_to_thread (stream_session_handle (session),
-					      FIFO_EVENT_DISCONNECT,
-					      thread_index);
+	  if (session->session_state != SESSION_STATE_CLOSED)
+	    {
+	      session->session_state = SESSION_STATE_CLOSED;
+	      session_send_session_evt_to_thread (stream_session_handle
+						  (session),
+						  FIFO_EVENT_DISCONNECT,
+						  thread_index);
+	    }
 	  fifo = fifo->next;
 	}
 
@@ -235,27 +246,51 @@ segment_manager_del (segment_manager_t * sm)
        * sessions if the segment can be removed.
        */
     }
+}
 
-  segment_manager_first_segment_maybe_del (sm);
+/**
+ * Removes segment manager.
+ *
+ * Since the fifos allocated in the segment keep backpointers to the sessions
+ * prior to removing the segment, we call session disconnect. This
+ * subsequently propages into transport.
+ */
+void
+segment_manager_del (segment_manager_t * sm)
+{
 
+  ASSERT (vec_len (sm->segment_indices) <= 1);
+  if (vec_len (sm->segment_indices))
+    {
+      /* The first segment in the first segment manager is not removed when
+       * all fifos are removed. It can only be removed when the manager is
+       * explicitly deleted/detached by the app. */
+      if (CLIB_DEBUG)
+	{
+	  svm_fifo_segment_private_t *fifo_segment;
+	  fifo_segment =
+	    svm_fifo_segment_get_segment (sm->segment_indices[0]);
+	  ASSERT (!svm_fifo_segment_has_fifos (fifo_segment));
+	}
+      segment_manager_del_segment (sm, 0);
+    }
   clib_spinlock_free (&sm->lockp);
+  if (CLIB_DEBUG)
+    memset (sm, 0xfe, sizeof (*sm));
   pool_put (segment_managers, sm);
 }
 
-static int
-segment_manager_notify_app_seg_add (segment_manager_t * sm,
-				    u32 fifo_segment_index)
+void
+segment_manager_init_del (segment_manager_t * sm)
 {
-  application_t *app = application_get (sm->app_index);
-  u32 seg_size = 0;
-  u8 *seg_name;
-
-  /* Send an API message to the external app, to map new segment */
-  ASSERT (app->cb_fns.add_segment_callback);
-
-  segment_manager_get_segment_info (fifo_segment_index, &seg_name, &seg_size);
-  return app->cb_fns.add_segment_callback (app->api_client_index, seg_name,
-					   seg_size);
+  if (segment_manager_has_fifos (sm))
+    segment_manager_del_sessions (sm);
+  else
+    {
+      ASSERT (!sm->first_is_protected
+	      || sm->app_index == SEGMENT_MANAGER_INVALID_APP_INDEX);
+      segment_manager_del (sm);
+    }
 }
 
 int
@@ -280,16 +315,14 @@ again:
   for (i = 0; i < vec_len (sm->segment_indices); i++)
     {
       *fifo_segment_index = sm->segment_indices[i];
-      fifo_segment = svm_fifo_get_segment (*fifo_segment_index);
+      fifo_segment = svm_fifo_segment_get_segment (*fifo_segment_index);
 
-      /* FC: cleanup, make sure sm->properties->xxx_fifo_size always set */
       fifo_size = sm->properties->rx_fifo_size;
       fifo_size = (fifo_size == 0) ? default_fifo_size : fifo_size;
       *server_rx_fifo =
 	svm_fifo_segment_alloc_fifo (fifo_segment, fifo_size,
 				     FIFO_SEGMENT_RX_FREELIST);
 
-      /* FC: cleanup, make sure sm->properties->xxx_fifo_size always set */
       fifo_size = sm->properties->tx_fifo_size;
       fifo_size = (fifo_size == 0) ? default_fifo_size : fifo_size;
       *server_tx_fifo =
@@ -327,7 +360,7 @@ again:
 	{
 	  if (added_a_segment)
 	    {
-	      clib_warning ("added a segment, still cant allocate a fifo");
+	      clib_warning ("added a segment, still can't allocate a fifo");
 	      clib_spinlock_unlock (&sm->lockp);
 	      return SESSION_ERROR_NEW_SEG_NO_SPACE;
 	    }
@@ -357,7 +390,8 @@ again:
   clib_spinlock_unlock (&sm->lockp);
 
   if (added_a_segment)
-    return segment_manager_notify_app_seg_add (sm, *fifo_segment_index);
+    return application_add_segment_notify (sm->app_index,
+					   *fifo_segment_index);
 
   return 0;
 }
@@ -368,32 +402,49 @@ segment_manager_dealloc_fifos (u32 svm_segment_index, svm_fifo_t * rx_fifo,
 {
   segment_manager_t *sm;
   svm_fifo_segment_private_t *fifo_segment;
+  u32 i, segment_index = ~0;
+  u8 is_first;
 
   sm = segment_manager_get_if_valid (rx_fifo->segment_manager);
 
   /* It's possible to have no segment manager if the session was removed
-   * as result of a detach */
+   * as result of a detach. */
   if (!sm)
     return;
 
-  fifo_segment = svm_fifo_get_segment (svm_segment_index);
+  fifo_segment = svm_fifo_segment_get_segment (svm_segment_index);
   svm_fifo_segment_free_fifo (fifo_segment, rx_fifo,
 			      FIFO_SEGMENT_RX_FREELIST);
   svm_fifo_segment_free_fifo (fifo_segment, tx_fifo,
 			      FIFO_SEGMENT_TX_FREELIST);
 
-  /* Don't try to delete process-private segments */
-  if (sm->properties->private_segment_count > 0)
-    return;
-
-  /* Remove segment only if it holds no fifos and not the first */
-  if (sm->segment_indices[0] != svm_segment_index
-      && !svm_fifo_segment_has_fifos (fifo_segment))
+  /*
+   * Try to remove svm segment if it has no fifos. This can be done only if
+   * the segment is not the first in the segment manager or if it is first
+   * and it is not protected. Moreover, if the segment is first and the app
+   * has detached from the segment manager, remove the segment manager.
+   */
+  if (!svm_fifo_segment_has_fifos (fifo_segment))
     {
-      clib_spinlock_lock (&sm->lockp);
-      svm_fifo_segment_delete (fifo_segment);
-      vec_del1 (sm->segment_indices, svm_segment_index);
-      clib_spinlock_unlock (&sm->lockp);
+      is_first = sm->segment_indices[0] == svm_segment_index;
+
+      /* Remove segment if it holds no fifos or first but not protected */
+      if (!is_first || !sm->first_is_protected)
+	{
+	  /* Find the segment manager segment index */
+	  for (i = 0; i < vec_len (sm->segment_indices); i++)
+	    if (sm->segment_indices[i] == svm_segment_index)
+	      {
+		segment_index = i;
+		break;
+	      }
+	  ASSERT (segment_index != (u32) ~ 0);
+	  segment_manager_del_segment (sm, segment_index);
+	}
+
+      /* Remove segment manager if no sessions and detached from app */
+      if (sm->app_index == SEGMENT_MANAGER_INVALID_APP_INDEX && is_first)
+	segment_manager_del (sm);
     }
 }
 
@@ -410,14 +461,14 @@ segment_manager_alloc_queue (segment_manager_t * sm, u32 queue_size)
 
   ASSERT (sm->segment_indices != 0);
 
-  segment = svm_fifo_get_segment (sm->segment_indices[0]);
+  segment = svm_fifo_segment_get_segment (sm->segment_indices[0]);
   sh = segment->ssvm.sh;
 
   oldheap = ssvm_push_heap (sh);
-  q =
-    unix_shared_memory_queue_init (queue_size, sizeof (session_fifo_event_t),
-				   0 /* consumer pid */ , 0
-				   /* signal when queue non-empty */ );
+  q = unix_shared_memory_queue_init (queue_size,
+				     sizeof (session_fifo_event_t),
+				     0 /* consumer pid */ ,
+				     0 /* signal when queue non-empty */ );
   ssvm_pop_heap (oldheap);
   return q;
 }
@@ -435,7 +486,7 @@ segment_manager_dealloc_queue (segment_manager_t * sm,
 
   ASSERT (sm->segment_indices != 0);
 
-  segment = svm_fifo_get_segment (sm->segment_indices[0]);
+  segment = svm_fifo_segment_get_segment (sm->segment_indices[0]);
   sh = segment->ssvm.sh;
 
   oldheap = ssvm_push_heap (sh);
@@ -443,6 +494,90 @@ segment_manager_dealloc_queue (segment_manager_t * sm,
   ssvm_pop_heap (oldheap);
 }
 
+static clib_error_t *
+segment_manager_show_fn (vlib_main_t * vm, unformat_input_t * input,
+			 vlib_cli_command_t * cmd)
+{
+  svm_fifo_segment_private_t *segments, *seg;
+  segment_manager_t *sm;
+  u8 show_segments = 0, verbose = 0, *name;
+  uword address;
+  u64 size;
+  u32 fifos;
+  mheap_t *heap_header;
+
+  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+    {
+      if (unformat (input, "segments"))
+	show_segments = 1;
+      else if (unformat (input, "verbose"))
+	verbose = 1;
+      else
+	return clib_error_return (0, "unknown input `%U'",
+				  format_unformat_error, input);
+    }
+  vlib_cli_output (vm, "%d segment managers allocated",
+		   pool_elts (segment_managers));
+  if (verbose && pool_elts (segment_managers))
+    {
+      vlib_cli_output (vm, "%-10s%=15s%=12s", "Index", "App Index",
+		       "Segments");
+
+      /* *INDENT-OFF* */
+      pool_foreach (sm, segment_managers, ({
+	vlib_cli_output (vm, "%-10d%=15d%=12d", segment_manager_index(sm),
+			   sm->app_index, vec_len (sm->segment_indices));
+      }));
+      /* *INDENT-ON* */
+
+    }
+  if (show_segments)
+    {
+      segments = svm_fifo_segment_segments_pool ();
+      vlib_cli_output (vm, "%d svm fifo segments allocated",
+		       pool_elts (segments));
+      vlib_cli_output (vm, "%-20s%=12s%=12s%=15s", "Name", "Size (M)",
+		       "Fifos", "Address");
+
+      /* *INDENT-OFF* */
+      pool_foreach (seg, segments, ({
+	if (seg->h->flags & FIFO_SEGMENT_F_IS_PRIVATE)
+	  {
+	    address = pointer_to_uword (seg->ssvm.sh->heap);
+	    if (seg->h->flags & FIFO_SEGMENT_F_IS_MAIN_HEAP)
+	      name = format (0, "main heap");
+	    else
+	      name = format (0, "private heap");
+	    heap_header = mheap_header (seg->ssvm.sh->heap);
+	    size = heap_header->max_size;
+	  }
+	else
+	  {
+	    address =  seg->ssvm.sh->ssvm_va;
+	    size = seg->ssvm.ssvm_size;
+	    name = seg->ssvm.sh->name;
+	  }
+	fifos = svm_fifo_segment_num_fifos (seg);
+	vlib_cli_output (vm, "%-20s%=12u%=12u%=15x", name, size << 20, fifos,
+			 address);
+	if (seg->h->flags & FIFO_SEGMENT_F_IS_PRIVATE)
+	  vec_free (name);
+      }));
+      /* *INDENT-ON* */
+
+    }
+  return 0;
+}
+
+		          /* *INDENT-OFF* */
+VLIB_CLI_COMMAND (segment_manager_show_command, static) =
+{
+  .path = "show segment-manager",
+  .short_help = "show segment-manager [segments]",
+  .function = segment_manager_show_fn,
+};
+/* *INDENT-ON* */
+
 /*
  * fd.io coding-style-patch-verification: ON
  *
diff --git a/src/vnet/session/segment_manager.h b/src/vnet/session/segment_manager.h
index 41abeb22..6e5b8989 100644
--- a/src/vnet/session/segment_manager.h
+++ b/src/vnet/session/segment_manager.h
@@ -55,24 +55,26 @@ typedef struct _segment_manager
   /** Owner app index */
   u32 app_index;
 
-  /** Pointer to manager properties. Could be shared among all of
-   * an app's segment managers s*/
+  /**
+   * Pointer to manager properties. Could be shared among all of
+   * an app's segment managers s
+   */
   segment_manager_properties_t *properties;
+
+  /**
+   * First segment should not be deleted unless segment manger is deleted.
+   * This also indicates that the segment manager is the first to have been
+   * allocated for the app.
+   */
+  u8 first_is_protected;
 } segment_manager_t;
 
+#define SEGMENT_MANAGER_INVALID_APP_INDEX ((u32) ~0)
+
 /** Pool of segment managers */
 extern segment_manager_t *segment_managers;
 
 always_inline segment_manager_t *
-segment_manager_new ()
-{
-  segment_manager_t *sm;
-  pool_get (segment_managers, sm);
-  memset (sm, 0, sizeof (*sm));
-  return sm;
-}
-
-always_inline segment_manager_t *
 segment_manager_get (u32 index)
 {
   return pool_elt_at_index (segment_managers, index);
@@ -92,6 +94,7 @@ segment_manager_index (segment_manager_t * sm)
   return sm - segment_managers;
 }
 
+segment_manager_t *segment_manager_new ();
 int
 segment_manager_init (segment_manager_t * sm,
 		      segment_manager_properties_t * properties,
@@ -100,9 +103,11 @@ segment_manager_init (segment_manager_t * sm,
 void segment_manager_get_segment_info (u32 index, u8 ** name, u32 * size);
 int
 session_manager_add_first_segment (segment_manager_t * sm, u32 segment_size);
-void segment_manager_first_segment_maybe_del (segment_manager_t * sm);
 int session_manager_add_segment (segment_manager_t * sm);
+void segment_manager_del_sessions (segment_manager_t * sm);
 void segment_manager_del (segment_manager_t * sm);
+void segment_manager_init_del (segment_manager_t * sm);
+u8 segment_manager_has_fifos (segment_manager_t * sm);
 int
 segment_manager_alloc_session_fifos (segment_manager_t * sm,
 				     svm_fifo_t ** server_rx_fifo,
diff --git a/src/vnet/session/session.c b/src/vnet/session/session.c
index 6fe99047..ee22ccbe 100644
--- a/src/vnet/session/session.c
+++ b/src/vnet/session/session.c
@@ -457,7 +457,10 @@ stream_session_connect_notify (transport_connection_t * tc, u8 is_fail)
   /* Get the app's index from the handle we stored when opening connection
    * and the opaque (api_context for external apps) from transport session
    * index*/
-  app = application_get (handle >> 32);
+  app = application_get_if_valid (handle >> 32);
+  if (!app)
+    return -1;
+
   opaque = tc->s_index;
 
   if (!is_fail)
@@ -562,9 +565,7 @@ stream_session_delete_notify (transport_connection_t * tc)
   /* App might've been removed already */
   s = stream_session_get_if_valid (tc->s_index, tc->thread_index);
   if (!s)
-    {
-      return;
-    }
+    return;
   stream_session_delete (s);
 }
 
diff --git a/src/vnet/session/session_cli.c b/src/vnet/session/session_cli.c
index f60048f1..028dc9d8 100755
--- a/src/vnet/session/session_cli.c
+++ b/src/vnet/session/session_cli.c
@@ -64,7 +64,8 @@ format_stream_session (u8 * s, va_list * args)
 		  stream_session_get_index (ss));
 
   if (ss->session_state == SESSION_STATE_READY
-      || ss->session_state == SESSION_STATE_ACCEPTING)
+      || ss->session_state == SESSION_STATE_ACCEPTING
+      || ss->session_state == SESSION_STATE_CLOSED)
     {
       s = format (s, "%U", tp_vft->format_connection, ss->connection_index,
 		  ss->thread_index, verbose);
@@ -83,16 +84,6 @@ format_stream_session (u8 * s, va_list * args)
       s = format (s, "%-40U%v", tp_vft->format_half_open,
 		  ss->connection_index, str);
     }
-  else if (ss->session_state == SESSION_STATE_CLOSED)
-    {
-      s =
-	format (s, "[CL] %U", tp_vft->format_connection, ss->connection_index,
-		ss->thread_index, verbose);
-      if (verbose == 1)
-	s = format (s, "%v", str);
-      if (verbose > 1)
-	s = format (s, "%U", format_stream_session_fifos, ss, verbose);
-    }
   else
     {
       clib_warning ("Session in state: %d!", ss->session_state);
@@ -284,9 +275,9 @@ show_session_command_fn (vlib_main_t * vm, unformat_input_t * input,
 	    {
 	      if (once_per_pool && verbose == 1)
 		{
-		  str =
-		    format (str, "%-50s%-15s%-10s%-10s%-10s", "Connection",
-			    "State", "Rx-f", "Tx-f", "S-idx");
+		  str = format (str, "%-50s%-15s%-10s%-10s%-10s",
+				"Connection", "State", "Rx-f", "Tx-f",
+				"S-idx");
 		  vlib_cli_output (vm, "%v", str);
 		  vec_reset_length (str);
 		  once_per_pool = 0;
diff --git a/src/vnet/session/session_node.c b/src/vnet/session/session_node.c
index cd52742b..dec6d13c 100644
--- a/src/vnet/session/session_node.c
+++ b/src/vnet/session/session_node.c
@@ -394,7 +394,6 @@ session_tx_fifo_dequeue_and_snd (vlib_main_t * vm, vlib_node_runtime_t * node,
 always_inline stream_session_t *
 session_event_get_session (session_fifo_event_t * e, u8 thread_index)
 {
-  ASSERT (e->fifo->master_thread_index == thread_index);
   return stream_session_get_if_valid (e->fifo->master_session_index,
 				      thread_index);
 }
@@ -643,6 +642,8 @@ skip_dequeue:
 	  break;
 	case FIFO_EVENT_BUILTIN_RX:
 	  s0 = session_event_get_session (e0, my_thread_index);
+	  if (PREDICT_FALSE (!s0))
+	    continue;
 	  svm_fifo_unset_event (s0->server_rx_fifo);
 	  app = application_get (s0->app_index);
 	  app->cb_fns.builtin_server_rx_callback (s0);
diff --git a/src/vnet/tcp/tcp.c b/src/vnet/tcp/tcp.c
index 6b2b4759..0a826a52 100644
--- a/src/vnet/tcp/tcp.c
+++ b/src/vnet/tcp/tcp.c
@@ -316,8 +316,10 @@ tcp_connection_close (tcp_connection_t * tc)
       tcp_send_fin (tc);
       tc->state = TCP_STATE_LAST_ACK;
       break;
+    case TCP_STATE_FIN_WAIT_1:
+      break;
     default:
-      clib_warning ("shouldn't be here");
+      clib_warning ("state: %u", tc->state);
     }
 
   TCP_EVT_DBG (TCP_EVT_STATE_CHANGE, tc);
@@ -836,6 +838,8 @@ format_tcp_connection (u8 * s, va_list * args)
   tcp_connection_t *tc = va_arg (*args, tcp_connection_t *);
   u32 verbose = va_arg (*args, u32);
 
+  if (!tc)
+    return s;
   s = format (s, "%-50U", format_tcp_connection_id, tc);
   if (verbose)
     {