vlmemory/svm: fix client detach from svm region

Clients cannot know at svm region detach time if the shm backing files
have been recreated (e.g., if vpp restarts) and therefore should not try
to unlink them. Otherwise, terminating clients attached to previous
instantiations of a re-allocated region end up making the new instance
un-mappable by removing its backing file.

Change-Id: Idcd0cab776e63fd75b821bc9f0fac58217b9ccbe
Signed-off-by: Florin Coras <fcoras@cisco.com>

6 files changed, 59 insertions, 13 deletions

diff --git a/src/svm/svm.c b/src/svm/svm.c
index d3e56c1a46a..16a58fa126a 100644
--- a/src/svm/svm.c
+++ b/src/svm/svm.c
@@ -1034,7 +1034,7 @@ svm_region_unlink (svm_region_t * rp)
  * a new region client showing up at the wrong moment.
  */
 void
-svm_region_unmap (void *rp_arg)
+svm_region_unmap_internal (void *rp_arg, u8 is_client)
 {
   int i, mypid = getpid ();
   int nclients_left;
@@ -1134,7 +1134,12 @@ found:
       vec_free (name);
 
       region_unlock (rp);
-      svm_region_unlink (rp);
+
+      /* If a client asks for the cleanup, don't unlink the backing
+       * file since we can't tell if it has been recreated. */
+      if (!is_client)
+	svm_region_unlink (rp);
+
       munmap ((void *) virtual_base, virtual_size);
       region_unlock (root_rp);
       svm_pop_heap (oldheap);
@@ -1147,11 +1152,23 @@ found:
   munmap ((void *) virtual_base, virtual_size);
 }
 
+void
+svm_region_unmap (void *rp_arg)
+{
+  svm_region_unmap_internal (rp_arg, 0 /* is_client */ );
+}
+
+void
+svm_region_unmap_client (void *rp_arg)
+{
+  svm_region_unmap_internal (rp_arg, 1 /* is_client */ );
+}
+
 /*
  * svm_region_exit
  */
-void
-svm_region_exit ()
+static void
+svm_region_exit_internal (u8 is_client)
 {
   void *oldheap;
   int i, mypid = getpid ();
@@ -1191,7 +1208,7 @@ svm_region_exit ()
 
 found:
 
-  if (vec_len (root_rp->client_pids) == 0)
+  if (!is_client && vec_len (root_rp->client_pids) == 0)
     svm_region_unlink (root_rp);
 
   region_unlock (root_rp);
@@ -1202,6 +1219,18 @@ found:
 }
 
 void
+svm_region_exit (void)
+{
+  svm_region_exit_internal (0 /* is_client */ );
+}
+
+void
+svm_region_exit_client (void)
+{
+  svm_region_exit_internal (1 /* is_client */ );
+}
+
+void
 svm_client_scan_this_region_nolock (svm_region_t * rp)
 {
   int j;
diff --git a/src/svm/svm_common.h b/src/svm/svm_common.h
index c650165b4cf..e523da50db6 100644
--- a/src/svm/svm_common.h
+++ b/src/svm/svm_common.h
@@ -118,7 +118,9 @@ int svm_region_init_chroot (const char *root_path);
 void svm_region_init_chroot_uid_gid (const char *root_path, int uid, int gid);
 void svm_region_init_args (svm_map_region_args_t * a);
 void svm_region_exit (void);
+void svm_region_exit_client (void);
 void svm_region_unmap (void *rp_arg);
+void svm_region_unmap_client (void *rp_arg);
 void svm_client_scan (const char *root_path);
 void svm_client_scan_this_region_nolock (svm_region_t * rp);
 u8 *shm_name_from_svm_map_region_args (svm_map_region_args_t * a);
diff --git a/src/vlibmemory/memory_client.c b/src/vlibmemory/memory_client.c
index 9f533019c87..9c8dcaa9d64 100644
--- a/src/vlibmemory/memory_client.c
+++ b/src/vlibmemory/memory_client.c
@@ -261,7 +261,7 @@ vl_api_memclnt_delete_reply_t_handler (vl_api_memclnt_delete_reply_t * mp)
   am->vl_input_queue = 0;
 }
 
-void
+int
 vl_client_disconnect (void)
 {
   vl_api_memclnt_delete_t *mp;
@@ -302,7 +302,7 @@ vl_client_disconnect (void)
 	  am->my_client_index = ~0;
 	  am->my_registration = 0;
 	  am->shmem_hdr = 0;
-	  break;
+	  return -1;
 	}
       if (svm_queue_sub (vl_input_queue, (u8 *) & rp, SVM_Q_NOWAIT, 0) < 0)
 	continue;
@@ -317,6 +317,7 @@ vl_client_disconnect (void)
       vl_msg_api_handler ((void *) rp);
       break;
     }
+  return 0;
 }
 
 /**
@@ -376,7 +377,7 @@ vl_client_api_map (const char *region_name)
 void
 vl_client_api_unmap (void)
 {
-  vl_unmap_shmem ();
+  vl_unmap_shmem_client ();
 }
 
 u8
diff --git a/src/vlibmemory/memory_client.h b/src/vlibmemory/memory_client.h
index 3fcca9e4b89..f2898f30b23 100644
--- a/src/vlibmemory/memory_client.h
+++ b/src/vlibmemory/memory_client.h
@@ -21,7 +21,7 @@
 #include <vlibmemory/memory_shared.h>
 
 int vl_client_connect (const char *name, int ctx_quota, int input_queue_size);
-void vl_client_disconnect (void);
+int vl_client_disconnect (void);
 int vl_client_api_map (const char *region_name);
 void vl_client_api_unmap (void);
 void vl_client_disconnect_from_vlib (void);
diff --git a/src/vlibmemory/memory_shared.c b/src/vlibmemory/memory_shared.c
index 488a302bbba..c8c071f9a45 100644
--- a/src/vlibmemory/memory_shared.c
+++ b/src/vlibmemory/memory_shared.c
@@ -663,8 +663,8 @@ vl_register_mapped_shmem_region (svm_region_t * rp)
   vec_add1 (am->mapped_shmem_regions, rp);
 }
 
-void
-vl_unmap_shmem (void)
+static void
+vl_unmap_shmem_internal (u8 is_client)
 {
   svm_region_t *rp;
   int i;
@@ -676,13 +676,14 @@ vl_unmap_shmem (void)
   for (i = 0; i < vec_len (am->mapped_shmem_regions); i++)
     {
       rp = am->mapped_shmem_regions[i];
-      svm_region_unmap (rp);
+      is_client ? svm_region_unmap_client (rp) : svm_region_unmap (rp);
     }
 
   vec_free (am->mapped_shmem_regions);
   am->shmem_hdr = 0;
 
-  svm_region_exit ();
+  is_client ? svm_region_exit_client () : svm_region_exit ();
+
   /* $$$ more careful cleanup, valgrind run... */
   vec_free (am->msg_handlers);
   vec_free (am->msg_endian_handlers);
@@ -690,6 +691,18 @@ vl_unmap_shmem (void)
 }
 
 void
+vl_unmap_shmem (void)
+{
+  vl_unmap_shmem_internal (0);
+}
+
+void
+vl_unmap_shmem_client (void)
+{
+  vl_unmap_shmem_internal (1);
+}
+
+void
 vl_msg_api_send_shmem (svm_queue_t * q, u8 * elem)
 {
   api_main_t *am = &api_main;
diff --git a/src/vlibmemory/memory_shared.h b/src/vlibmemory/memory_shared.h
index deaa62166ae..67ead7d2877 100644
--- a/src/vlibmemory/memory_shared.h
+++ b/src/vlibmemory/memory_shared.h
@@ -117,6 +117,7 @@ void *vl_mem_api_alloc_as_if_client_w_reg (vl_api_registration_t * reg,
 void vl_msg_api_free (void *a);
 int vl_map_shmem (const char *region_name, int is_vlib);
 void vl_unmap_shmem (void);
+void vl_unmap_shmem_client (void);
 void vl_register_mapped_shmem_region (svm_region_t * rp);
 void vl_msg_api_send_shmem (svm_queue_t * q, u8 * elem);
 void vl_msg_api_send_shmem_nolock (svm_queue_t * q, u8 * elem);