hsa: add support for SAPI in vpp_echo

Type: feature

Signed-off-by: Filip Tehlar <ftehlar@cisco.com>
Change-Id: If24a43b7f79e05092306562c192de03994dec550

4 files changed, 490 insertions, 48 deletions

diff --git a/src/plugins/hs_apps/sapi/vpp_echo.c b/src/plugins/hs_apps/sapi/vpp_echo.c
index 80d274db5b0..e07e53d194e 100644
--- a/src/plugins/hs_apps/sapi/vpp_echo.c
+++ b/src/plugins/hs_apps/sapi/vpp_echo.c
@@ -84,16 +84,19 @@ stop_signal (int signum)
   em->time_to_stop = 1;
 }
 
-int
-connect_to_vpp (char *name)
+static int
+connect_to_vpp (echo_main_t *em)
 {
-  echo_main_t *em = &echo_main;
   api_main_t *am = vlibapi_get_main ();
 
+  if (em->use_app_socket_api)
+    return echo_api_connect_app_socket (em);
+
   if (em->use_sock_api)
     {
-      if (vl_socket_client_connect ((char *) em->socket_name, name,
-				    0 /* default rx, tx buffer */ ))
+      if (vl_socket_client_connect ((char *) em->socket_name,
+				    (char *) em->app_name,
+				    0 /* default rx, tx buffer */))
 	{
 	  ECHO_FAIL (ECHO_FAIL_SOCKET_CONNECT, "socket connect failed");
 	  return -1;
@@ -107,7 +110,8 @@ connect_to_vpp (char *name)
     }
   else
     {
-      if (vl_client_connect_to_vlib ("/vpe-api", name, 32) < 0)
+      if (vl_client_connect_to_vlib ("/vpe-api", (char *) em->app_name, 32) <
+	  0)
 	{
 	  ECHO_FAIL (ECHO_FAIL_SHMEM_CONNECT, "shmem connect failed");
 	  return -1;
@@ -725,9 +729,18 @@ session_reset_handler (session_reset_msg_t * mp)
   app_send_ctrl_evt_to_vpp (s->vpp_evt_q, app_evt);
 }
 
+static int
+echo_recv_fd (echo_main_t *em, int *fds, int n_fds)
+{
+  if (em->use_app_socket_api)
+    return echo_sapi_recv_fd (em, fds, n_fds);
+  return echo_bapi_recv_fd (em, fds, n_fds);
+}
+
 static void
 add_segment_handler (session_app_add_segment_msg_t * mp)
 {
+  echo_main_t *em = &echo_main;
   fifo_segment_main_t *sm = &echo_main.segment_main;
   fifo_segment_create_args_t _a, *a = &_a;
   int *fds = 0, i;
@@ -737,10 +750,10 @@ add_segment_handler (session_app_add_segment_msg_t * mp)
   if (mp->fd_flags & SESSION_FD_F_MEMFD_SEGMENT)
     {
       vec_validate (fds, 1);
-      if (vl_socket_client_recv_fd_msg (fds, 1, 5))
+      if (echo_recv_fd (em, fds, 1))
 	{
-	  ECHO_FAIL (ECHO_FAIL_VL_API_RECV_FD_MSG,
-		     "vl_socket_client_recv_fd_msg failed");
+	  ECHO_LOG (0, "echo_recv_fd failed");
+	  em->time_to_stop = 1;
 	  goto failed;
 	}
 
@@ -1112,6 +1125,8 @@ echo_process_opts (int argc, char **argv)
 	em->test_return_packets = RETURN_PACKETS_LOG_WRONG;
       else if (unformat (a, "socket-name %s", &em->socket_name))
 	;
+      else if (unformat (a, "use-app-socket-api"))
+	em->use_app_socket_api = 1;
       else if (unformat (a, "use-svm-api"))
 	em->use_sock_api = 0;
       else if (unformat (a, "fifo-size %U", unformat_memory_size, &tmp))
@@ -1228,6 +1243,15 @@ echo_process_opts (int argc, char **argv)
     }
 }
 
+static int
+echo_needs_crypto (echo_main_t *em)
+{
+  u8 tr = em->uri_elts.transport_proto;
+  if (tr == TRANSPORT_PROTO_QUIC || tr == TRANSPORT_PROTO_TLS)
+    return 1;
+  return 0;
+}
+
 void
 echo_process_uri (echo_main_t * em)
 {
@@ -1260,13 +1284,91 @@ vpp_echo_init ()
   clib_memset (em, 0, sizeof (*em));
 }
 
+static int
+echo_detach (echo_main_t *em)
+{
+  if (em->use_app_socket_api)
+    return echo_sapi_detach (em);
+
+  echo_send_detach (em);
+  if (wait_for_state_change (em, STATE_DETACHED, TIMEOUT))
+    {
+      ECHO_FAIL (ECHO_FAIL_DETACH, "Couldn't detach from vpp");
+      return -1;
+    }
+  return 0;
+}
+
+static void
+echo_add_cert_key (echo_main_t *em)
+{
+  if (em->use_app_socket_api)
+    echo_sapi_add_cert_key (em);
+  else
+    {
+      echo_send_add_cert_key (em);
+      if (wait_for_state_change (em, STATE_ATTACHED, TIMEOUT))
+	{
+	  ECHO_FAIL (ECHO_FAIL_APP_ATTACH,
+		     "Couldn't add crypto context to vpp\n");
+	  exit (1);
+	}
+    }
+}
+
+static int
+echo_del_cert_key (echo_main_t *em)
+{
+  if (em->use_app_socket_api)
+    return echo_sapi_del_cert_key (em);
+
+  echo_send_del_cert_key (em);
+  if (wait_for_state_change (em, STATE_CLEANED_CERT_KEY, TIMEOUT))
+    {
+      ECHO_FAIL (ECHO_FAIL_DEL_CERT_KEY, "Couldn't cleanup cert and key");
+      return -1;
+    }
+  return 0;
+}
+
+static void
+echo_disconnect (echo_main_t *em)
+{
+  if (em->use_app_socket_api)
+    return;
+
+  if (em->use_sock_api)
+    vl_socket_client_disconnect ();
+  else
+    vl_client_disconnect_from_vlib ();
+}
+
+static int
+echo_attach (echo_main_t *em)
+{
+  if (em->use_app_socket_api)
+    return echo_sapi_attach (em);
+  else
+    {
+      echo_api_hookup (em);
+      echo_send_attach (em);
+      if (wait_for_state_change (em, STATE_ATTACHED_NO_CERT, TIMEOUT))
+	{
+	  ECHO_FAIL (ECHO_FAIL_ATTACH_TO_VPP,
+		     "Couldn't attach to vpp, did you run <session enable> ?");
+	  return -1;
+	}
+    }
+  return 0;
+}
+
 int
 main (int argc, char **argv)
 {
   echo_main_t *em = &echo_main;
   fifo_segment_main_t *sm = &em->segment_main;
-  char *app_name;
   u64 i;
+  int *rv;
   svm_msg_q_cfg_t _cfg, *cfg = &_cfg;
   u32 rpc_queue_size = 256 << 10;
 
@@ -1344,8 +1446,10 @@ main (int argc, char **argv)
   signal (SIGQUIT, stop_signal);
   signal (SIGTERM, stop_signal);
 
-  app_name = em->i_am_master ? "echo_server" : "echo_client";
-  if (connect_to_vpp (app_name))
+  em->app_name =
+    format (0, "%s%c", em->i_am_master ? "echo_server" : "echo_client", 0);
+
+  if (connect_to_vpp (em))
     {
       svm_region_exit ();
       ECHO_FAIL (ECHO_FAIL_CONNECT_TO_VPP, "Couldn't connect to vpp");
@@ -1355,34 +1459,22 @@ main (int argc, char **argv)
   echo_session_prealloc (em);
   echo_notify_event (em, ECHO_EVT_START);
 
-  echo_api_hookup (em);
+  if (echo_attach (em))
+    goto exit_on_error;
 
-  echo_send_attach (em);
-  if (wait_for_state_change (em, STATE_ATTACHED_NO_CERT, TIMEOUT))
+  if (echo_needs_crypto (em))
     {
-      ECHO_FAIL (ECHO_FAIL_ATTACH_TO_VPP,
-		 "Couldn't attach to vpp, did you run <session enable> ?");
-      goto exit_on_error;
+      ECHO_LOG (2, "Adding crypto context %U", echo_format_crypto_engine,
+		em->crypto_engine);
+      echo_add_cert_key (em);
     }
-
-  if (em->uri_elts.transport_proto != TRANSPORT_PROTO_QUIC
-      && em->uri_elts.transport_proto != TRANSPORT_PROTO_TLS)
-    em->state = STATE_ATTACHED;
   else
     {
-      ECHO_LOG (2, "Adding crypto context %U", echo_format_crypto_engine,
-		em->crypto_engine);
-      echo_send_add_cert_key (em);
-      if (wait_for_state_change (em, STATE_ATTACHED, TIMEOUT))
-	{
-	  ECHO_FAIL (ECHO_FAIL_APP_ATTACH,
-		     "Couldn't add crypto context to vpp\n");
-	  exit (1);
-	}
+      em->state = STATE_ATTACHED;
     }
 
-  if (pthread_create (&em->mq_thread_handle,
-		      NULL /*attr */ , echo_mq_thread_fn, 0))
+  if (pthread_create (&em->mq_thread_handle, NULL /*attr */, echo_mq_thread_fn,
+		      0))
     {
       ECHO_FAIL (ECHO_FAIL_PTHREAD_CREATE, "pthread create errored");
       goto exit_on_error;
@@ -1402,30 +1494,22 @@ main (int argc, char **argv)
     clients_run (em);
   echo_notify_event (em, ECHO_EVT_EXIT);
   echo_free_sessions (em);
-  echo_send_del_cert_key (em);
-  if (wait_for_state_change (em, STATE_CLEANED_CERT_KEY, TIMEOUT))
+  if (echo_needs_crypto (em))
     {
-      ECHO_FAIL (ECHO_FAIL_DEL_CERT_KEY, "Couldn't cleanup cert and key");
-      goto exit_on_error;
+      if (echo_del_cert_key (em))
+	goto exit_on_error;
     }
 
-  echo_send_detach (em);
-  if (wait_for_state_change (em, STATE_DETACHED, TIMEOUT))
-    {
-      ECHO_FAIL (ECHO_FAIL_DETACH, "Couldn't detach from vpp");
-      goto exit_on_error;
-    }
-  int *rv;
+  if (echo_detach (em))
+    goto exit_on_error;
+
   pthread_join (em->mq_thread_handle, (void **) &rv);
   if (rv)
     {
       ECHO_FAIL (ECHO_FAIL_MQ_PTHREAD, "mq pthread errored %d", rv);
       goto exit_on_error;
     }
-  if (em->use_sock_api)
-    vl_socket_client_disconnect ();
-  else
-    vl_client_disconnect_from_vlib ();
+  echo_disconnect (em);
   echo_assert_test_suceeded (em);
 exit_on_error:
   ECHO_LOG (1, "Test complete !\n");
diff --git a/src/plugins/hs_apps/sapi/vpp_echo_bapi.c b/src/plugins/hs_apps/sapi/vpp_echo_bapi.c
index c3b53c86ae2..e38559c255e 100644
--- a/src/plugins/hs_apps/sapi/vpp_echo_bapi.c
+++ b/src/plugins/hs_apps/sapi/vpp_echo_bapi.c
@@ -103,6 +103,19 @@ echo_send_del_cert_key (echo_main_t * em)
   vl_msg_api_send_shmem (em->vl_input_queue, (u8 *) & bmp);
 }
 
+int
+echo_bapi_recv_fd (echo_main_t *em, int *fds, int n_fds)
+{
+  clib_error_t *err;
+  err = vl_socket_client_recv_fd_msg (fds, n_fds, 5);
+  if (err)
+    {
+      clib_error_report (err);
+      return -1;
+    }
+  return 0;
+}
+
 static u8
 echo_transport_needs_crypto (transport_proto_t proto)
 {
diff --git a/src/plugins/hs_apps/sapi/vpp_echo_common.h b/src/plugins/hs_apps/sapi/vpp_echo_common.h
index 80c539ccb0f..9b2a2c677b5 100644
--- a/src/plugins/hs_apps/sapi/vpp_echo_common.h
+++ b/src/plugins/hs_apps/sapi/vpp_echo_common.h
@@ -26,6 +26,9 @@
 #define LOG_EVERY_N_IDLE_CYCLES (1e8)
 #define ECHO_MQ_SEG_HANDLE	((u64) ~0 - 1)
 
+#define ECHO_INVALID_SEGMENT_INDEX  ((u32) ~0)
+#define ECHO_INVALID_SEGMENT_HANDLE ((u64) ~0)
+
 #define foreach_echo_fail_code                                          \
   _(ECHO_FAIL_NONE, "ECHO_FAIL_NONE")                                   \
   _(ECHO_FAIL_USAGE, "ECHO_FAIL_USAGE")                                 \
@@ -269,6 +272,7 @@ typedef struct
   svm_queue_t *vl_input_queue;	/* vpe input queue */
   u32 my_client_index;		/* API client handle */
   u8 *uri;			/* The URI we're playing with */
+  u8 *app_name;
   u32 n_uris;			/* Cycle through adjacent ips */
   ip46_address_t lcl_ip;	/* Local ip for client */
   u8 lcl_ip_set;
@@ -277,6 +281,8 @@ typedef struct
   svm_msg_q_t *ctrl_mq;		/* Our control queue (towards vpp) */
   clib_time_t clib_time;	/* For deadman timers */
   u8 *socket_name;
+  u8 use_app_socket_api;
+  clib_socket_t app_api_sock;
   int i_am_master;
   u32 *listen_session_indexes;	/* vec of vpp listener sessions */
   volatile u32 listen_session_cnt;
@@ -449,6 +455,15 @@ void echo_send_disconnect_session (echo_main_t * em, void *args);
 void echo_api_hookup (echo_main_t * em);
 void echo_send_add_cert_key (echo_main_t * em);
 void echo_send_del_cert_key (echo_main_t * em);
+int echo_bapi_recv_fd (echo_main_t *em, int *fds, int n_fds);
+
+/* Session socket API */
+int echo_sapi_attach (echo_main_t *em);
+int echo_sapi_add_cert_key (echo_main_t *em);
+int echo_sapi_del_cert_key (echo_main_t *em);
+int echo_api_connect_app_socket (echo_main_t *em);
+int echo_sapi_detach (echo_main_t *em);
+int echo_sapi_recv_fd (echo_main_t *em, int *fds, int n_fds);
 
 #endif /* __included_vpp_echo_common_h__ */
 
diff --git a/src/plugins/hs_apps/sapi/vpp_echo_sapi.c b/src/plugins/hs_apps/sapi/vpp_echo_sapi.c
new file mode 100644
index 00000000000..a21fbea6183
--- /dev/null
+++ b/src/plugins/hs_apps/sapi/vpp_echo_sapi.c
@@ -0,0 +1,330 @@
+/*
+ * Copyright (c) 2022 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <hs_apps/sapi/vpp_echo_common.h>
+
+int
+echo_api_connect_app_socket (echo_main_t *em)
+{
+  clib_socket_t *cs = &em->app_api_sock;
+  clib_error_t *err;
+  int rv = 0;
+
+  cs->config = (char *) em->socket_name;
+  cs->flags =
+    CLIB_SOCKET_F_IS_CLIENT | CLIB_SOCKET_F_SEQPACKET | CLIB_SOCKET_F_BLOCKING;
+
+  if ((err = clib_socket_init (cs)))
+    {
+      clib_error_report (err);
+      rv = -1;
+    }
+
+  return rv;
+}
+
+static inline u64
+echo_vpp_worker_segment_handle (u32 wrk_index)
+{
+  return (ECHO_INVALID_SEGMENT_HANDLE - wrk_index - 1);
+}
+
+static int
+echo_segment_discover_mqs (uword segment_handle, int *fds, u32 n_fds)
+{
+  echo_main_t *em = &echo_main;
+  fifo_segment_t *fs;
+  u32 fs_index;
+
+  fs_index = echo_segment_lookup (segment_handle);
+  if (fs_index == ECHO_INVALID_SEGMENT_INDEX)
+    {
+      ECHO_LOG (0, "ERROR: mq segment %lx for is not attached!",
+		segment_handle);
+      return -1;
+    }
+
+  clib_spinlock_lock (&em->segment_handles_lock);
+
+  fs = fifo_segment_get_segment (&em->segment_main, fs_index);
+  fifo_segment_msg_qs_discover (fs, fds, n_fds);
+
+  clib_spinlock_unlock (&em->segment_handles_lock);
+
+  return 0;
+}
+
+static int
+echo_api_attach_reply_handler (app_sapi_attach_reply_msg_t *mp, int *fds)
+{
+  echo_main_t *em = &echo_main;
+  int i, rv, n_fds_used = 0;
+  u64 segment_handle;
+  u8 *segment_name;
+
+  if (mp->retval)
+    {
+      ECHO_LOG (0, "attach failed: %U", format_session_error, mp->retval);
+      goto failed;
+    }
+
+  em->my_client_index = mp->api_client_handle;
+  segment_handle = mp->segment_handle;
+  if (segment_handle == ECHO_INVALID_SEGMENT_HANDLE)
+    {
+      ECHO_LOG (0, "invalid segment handle");
+      goto failed;
+    }
+
+  if (!mp->n_fds)
+    goto failed;
+
+  if (mp->fd_flags & SESSION_FD_F_VPP_MQ_SEGMENT)
+    if (echo_segment_attach (echo_vpp_worker_segment_handle (0), "vpp-mq-seg",
+			     SSVM_SEGMENT_MEMFD, fds[n_fds_used++]))
+      goto failed;
+
+  if (mp->fd_flags & SESSION_FD_F_MEMFD_SEGMENT)
+    {
+      segment_name = format (0, "memfd-%ld%c", segment_handle, 0);
+      rv = echo_segment_attach (segment_handle, (char *) segment_name,
+				SSVM_SEGMENT_MEMFD, fds[n_fds_used++]);
+      vec_free (segment_name);
+      if (rv != 0)
+	goto failed;
+    }
+
+  echo_segment_attach_mq (segment_handle, mp->app_mq, 0, &em->app_mq);
+
+  if (mp->fd_flags & SESSION_FD_F_MQ_EVENTFD)
+    {
+      ECHO_LOG (0, "SESSION_FD_F_MQ_EVENTFD unsupported!");
+      goto failed;
+    }
+
+  echo_segment_discover_mqs (echo_vpp_worker_segment_handle (0),
+			     fds + n_fds_used, mp->n_fds - n_fds_used);
+  echo_segment_attach_mq (echo_vpp_worker_segment_handle (0), mp->vpp_ctrl_mq,
+			  mp->vpp_ctrl_mq_thread, &em->ctrl_mq);
+
+  em->state = STATE_ATTACHED_NO_CERT;
+  return 0;
+
+failed:
+
+  for (i = clib_max (n_fds_used - 1, 0); i < mp->n_fds; i++)
+    close (fds[i]);
+
+  return -1;
+}
+
+static int
+echo_api_send_attach (clib_socket_t *cs)
+{
+  echo_main_t *em = &echo_main;
+  app_sapi_msg_t msg = { 0 };
+  app_sapi_attach_msg_t *mp = &msg.attach;
+  clib_error_t *err;
+
+  clib_memcpy (&mp->name, em->app_name, vec_len (em->app_name));
+  mp->options[APP_OPTIONS_FLAGS] =
+    APP_OPTIONS_FLAGS_ACCEPT_REDIRECT | APP_OPTIONS_FLAGS_ADD_SEGMENT;
+  mp->options[APP_OPTIONS_SEGMENT_SIZE] = 256 << 20;
+  mp->options[APP_OPTIONS_ADD_SEGMENT_SIZE] = 128 << 20;
+  mp->options[APP_OPTIONS_RX_FIFO_SIZE] = em->fifo_size;
+  mp->options[APP_OPTIONS_TX_FIFO_SIZE] = em->fifo_size;
+  mp->options[APP_OPTIONS_PREALLOC_FIFO_PAIRS] = em->prealloc_fifo_pairs;
+  mp->options[APP_OPTIONS_EVT_QUEUE_SIZE] = em->evt_q_size;
+
+  msg.type = APP_SAPI_MSG_TYPE_ATTACH;
+  err = clib_socket_sendmsg (cs, &msg, sizeof (msg), 0, 0);
+  if (err)
+    {
+      clib_error_report (err);
+      return -1;
+    }
+
+  return 0;
+}
+
+int
+echo_sapi_attach (echo_main_t *em)
+{
+  app_sapi_msg_t _rmp, *rmp = &_rmp;
+  clib_error_t *err;
+  clib_socket_t *cs;
+  int fds[32];
+
+  cs = &em->app_api_sock;
+  if (echo_api_send_attach (cs))
+    return -1;
+
+  /*
+   * Wait for attach reply
+   */
+  err = clib_socket_recvmsg (cs, rmp, sizeof (*rmp), fds, ARRAY_LEN (fds));
+  if (err)
+    {
+      clib_error_report (err);
+      return -1;
+    }
+
+  if (rmp->type != APP_SAPI_MSG_TYPE_ATTACH_REPLY)
+    return -1;
+
+  return echo_api_attach_reply_handler (&rmp->attach_reply, fds);
+}
+
+int
+echo_sapi_add_cert_key (echo_main_t *em)
+{
+  u32 cert_len = test_srv_crt_rsa_len;
+  u32 key_len = test_srv_key_rsa_len;
+  u32 certkey_len = cert_len + key_len;
+  app_sapi_msg_t _msg = { 0 }, *msg = &_msg;
+  app_sapi_cert_key_add_del_msg_t *mp;
+  app_sapi_msg_t _rmp, *rmp = &_rmp;
+  clib_error_t *err;
+  clib_socket_t *cs;
+  u8 *certkey = 0;
+  int rv = -1;
+
+  msg->type = APP_SAPI_MSG_TYPE_ADD_DEL_CERT_KEY;
+  mp = &msg->cert_key_add_del;
+  mp->context = ntohl (0xfeedface);
+  mp->cert_len = cert_len;
+  mp->certkey_len = certkey_len;
+  mp->is_add = 1;
+
+  vec_validate (certkey, certkey_len - 1);
+  clib_memcpy_fast (certkey, test_srv_crt_rsa, cert_len);
+  clib_memcpy_fast (certkey + cert_len, test_srv_key_rsa, key_len);
+
+  cs = &em->app_api_sock;
+  err = clib_socket_sendmsg (cs, msg, sizeof (*msg), 0, 0);
+  if (err)
+    {
+      clib_error_report (err);
+      goto done;
+    }
+
+  err = clib_socket_sendmsg (cs, certkey, certkey_len, 0, 0);
+  if (err)
+    {
+      clib_error_report (err);
+      goto done;
+    }
+
+  /*
+   * Wait for reply and process it
+   */
+  err = clib_socket_recvmsg (cs, rmp, sizeof (*rmp), 0, 0);
+  if (err)
+    {
+      clib_error_report (err);
+      goto done;
+    }
+
+  if (rmp->type != APP_SAPI_MSG_TYPE_ADD_DEL_CERT_KEY_REPLY)
+    {
+      ECHO_LOG (0, "unexpected reply type %u", rmp->type);
+      goto done;
+    }
+
+  if (!rmp->cert_key_add_del_reply.retval)
+    rv = rmp->cert_key_add_del_reply.index;
+
+  em->state = STATE_ATTACHED;
+  em->ckpair_index = rv;
+
+done:
+
+  return rv;
+}
+
+int
+echo_sapi_recv_fd (echo_main_t *em, int *fds, int n_fds)
+{
+  app_sapi_msg_t _msg, *msg = &_msg;
+  clib_error_t *err =
+    clib_socket_recvmsg (&em->app_api_sock, msg, sizeof (*msg), fds, n_fds);
+  if (err)
+    {
+      clib_error_report (err);
+      return -1;
+    }
+  return 0;
+}
+
+int
+echo_sapi_detach (echo_main_t *em)
+{
+  clib_socket_t *cs = &em->app_api_sock;
+  clib_socket_close (cs);
+  em->state = STATE_DETACHED;
+  return 0;
+}
+
+int
+echo_sapi_del_cert_key (echo_main_t *em)
+{
+  app_sapi_msg_t _msg = { 0 }, *msg = &_msg;
+  app_sapi_cert_key_add_del_msg_t *mp;
+  app_sapi_msg_t _rmp, *rmp = &_rmp;
+  clib_error_t *err;
+  clib_socket_t *cs;
+
+  msg->type = APP_SAPI_MSG_TYPE_ADD_DEL_CERT_KEY;
+  mp = &msg->cert_key_add_del;
+  mp->index = em->ckpair_index;
+
+  cs = &em->app_api_sock;
+  err = clib_socket_sendmsg (cs, msg, sizeof (*msg), 0, 0);
+  if (err)
+    {
+      clib_error_report (err);
+      return -1;
+    }
+
+  /*
+   * Wait for reply and process it
+   */
+  err = clib_socket_recvmsg (cs, rmp, sizeof (*rmp), 0, 0);
+  if (err)
+    {
+      clib_error_report (err);
+      return -1;
+    }
+
+  if (rmp->type != APP_SAPI_MSG_TYPE_ADD_DEL_CERT_KEY_REPLY)
+    {
+      ECHO_LOG (0, "unexpected reply type %u", rmp->type);
+      return -1;
+    }
+
+  if (rmp->cert_key_add_del_reply.retval)
+    return -1;
+
+  em->state = STATE_CLEANED_CERT_KEY;
+  return 0;
+}
+
+/*
+ * fd.io coding-style-patch-verification: ON
+ *
+ * Local Variables:
+ * eval: (c-set-style "gnu")
+ * End:
+ */