From 993683150202254c6ba8dd43e087a7229edd5d4c Mon Sep 17 00:00:00 2001 From: Florin Coras Date: Thu, 2 Aug 2018 10:45:44 -0700 Subject: vcl: support for eventfd mq signaling - support eventfd based mq signaling. Based on configuration, vcl epoll/select can use either condvars or epoll on mq eventfds. - add vcl support for memfd segments - vpp explicitly registers cut-through segments with apps/vcl - if using eventfd, make ldp allow one call to libc_epoll_create. Needed for the message queue epfd - update svm_queue_t to allow blocking calls with eventfd signaling. Change-Id: I064151ac370bbe29bb16c968bf4e3659c8286bea Signed-off-by: Florin Coras --- src/vcl/ldp.c | 418 +++++++++++++++++++--------------------- src/vcl/vcl_bapi.c | 208 ++++++++++++++------ src/vcl/vcl_cfg.c | 318 ++++++++++++++++-------------- src/vcl/vcl_private.c | 151 +++++++++++++++ src/vcl/vcl_private.h | 63 +++++- src/vcl/vcl_test_server.c | 1 - src/vcl/vppcom.c | 479 ++++++++++++++++++++++++++++++---------------- src/vcl/vppcom.h | 24 +-- 8 files changed, 1063 insertions(+), 599 deletions(-) create mode 100644 src/vcl/vcl_private.c (limited to 'src/vcl') diff --git a/src/vcl/ldp.c b/src/vcl/ldp.c index c26c460bec8..b7b6b3c26b1 100644 --- a/src/vcl/ldp.c +++ b/src/vcl/ldp.c @@ -67,9 +67,17 @@ typedef struct vcl_poll_t *vcl_poll; u8 select_vcl; u8 epoll_wait_vcl; + u8 vcl_needs_real_epoll; /*< vcl needs next epoll_create to + go to libc_epoll */ + int vcl_mq_epfd; + } ldp_main_t; #define LDP_DEBUG ldp->debug +#define LDBG(_lvl, _fmt, _args...) \ + if (ldp->debug > _lvl) \ + clib_warning (_fmt, ##_args) + static ldp_main_t ldp_main = { .sid_bit_val = (1 << LDP_SID_BIT_MIN), .sid_bit_mask = (1 << LDP_SID_BIT_MIN) - 1, @@ -125,107 +133,96 @@ ldp_sid_from_fd (int fd) static inline int ldp_init (void) { - int rv = 0; + int rv; + + if (PREDICT_TRUE (ldp->init)) + return 0; + + ldp->init = 1; + ldp->vcl_needs_real_epoll = 1; + rv = vppcom_app_create (ldp_get_app_name ()); + if (rv != VPPCOM_OK) + { + fprintf (stderr, "\nLDP<%d>: ERROR: ldp_init: vppcom_app_create()" + " failed! rv = %d (%s)\n", + getpid (), rv, vppcom_retval_str (rv)); + ldp->init = 0; + return rv; + } + ldp->vcl_needs_real_epoll = 0; - if (PREDICT_FALSE (!ldp->init)) + char *env_var_str = getenv (LDP_ENV_DEBUG); + if (env_var_str) { - ldp->init = 1; - rv = vppcom_app_create (ldp_get_app_name ()); - if (rv == VPPCOM_OK) + u32 tmp; + if (sscanf (env_var_str, "%u", &tmp) != 1) + clib_warning ("LDP<%d>: WARNING: Invalid LDP debug level specified in" + " the env var " LDP_ENV_DEBUG " (%s)!", getpid (), + env_var_str); + else { - char *env_var_str = getenv (LDP_ENV_DEBUG); - if (env_var_str) - { - u32 tmp; - if (sscanf (env_var_str, "%u", &tmp) != 1) - clib_warning ("LDP<%d>: WARNING: Invalid LDP debug level " - "specified in the env var " LDP_ENV_DEBUG - " (%s)!", getpid (), env_var_str); - else - { - ldp->debug = tmp; - if (LDP_DEBUG > 0) - clib_warning ("LDP<%d>: configured LDP debug level (%u) " - "from the env var " LDP_ENV_DEBUG "!", - getpid (), ldp->debug); - } - } + ldp->debug = tmp; + LDBG (0, "LDP<%d>: configured LDP debug level (%u) from env var " + LDP_ENV_DEBUG "!", getpid (), ldp->debug); + } + } - env_var_str = getenv (LDP_ENV_APP_NAME); - if (env_var_str) - { - ldp_set_app_name (env_var_str); - if (LDP_DEBUG > 0) - clib_warning ("LDP<%d>: configured LDP app name (%s) " - "from the env var " LDP_ENV_APP_NAME "!", - getpid (), ldp->app_name); - } + env_var_str = getenv (LDP_ENV_APP_NAME); + if (env_var_str) + { + ldp_set_app_name (env_var_str); + LDBG (0, "LDP<%d>: configured LDP app name (%s) from the env var " + LDP_ENV_APP_NAME "!", getpid (), ldp->app_name); + } - env_var_str = getenv (LDP_ENV_SID_BIT); - if (env_var_str) - { - u32 sb; - if (sscanf (env_var_str, "%u", &sb) != 1) - { - clib_warning ("LDP<%d>: WARNING: Invalid LDP sid bit " - "specified in the env var " - LDP_ENV_SID_BIT " (%s)!" - "sid bit value %d (0x%x)", - getpid (), env_var_str, - ldp->sid_bit_val, ldp->sid_bit_val); - } - else if (sb < LDP_SID_BIT_MIN) - { - ldp->sid_bit_val = (1 << LDP_SID_BIT_MIN); - ldp->sid_bit_mask = ldp->sid_bit_val - 1; - - clib_warning ("LDP<%d>: WARNING: LDP sid bit (%u) " - "specified in the env var " - LDP_ENV_SID_BIT " (%s) is too small. " - "Using LDP_SID_BIT_MIN (%d)! " - "sid bit value %d (0x%x)", - getpid (), sb, env_var_str, LDP_SID_BIT_MIN, - ldp->sid_bit_val, ldp->sid_bit_val); - } - else if (sb > LDP_SID_BIT_MAX) - { - ldp->sid_bit_val = (1 << LDP_SID_BIT_MAX); - ldp->sid_bit_mask = ldp->sid_bit_val - 1; - - clib_warning ("LDP<%d>: WARNING: LDP sid bit (%u) " - "specified in the env var " - LDP_ENV_SID_BIT " (%s) is too big. " - "Using LDP_SID_BIT_MAX (%d)! " - "sid bit value %d (0x%x)", - getpid (), sb, env_var_str, LDP_SID_BIT_MAX, - ldp->sid_bit_val, ldp->sid_bit_val); - } - else - { - ldp->sid_bit_val = (1 << sb); - ldp->sid_bit_mask = ldp->sid_bit_val - 1; - - if (LDP_DEBUG > 0) - clib_warning ("LDP<%d>: configured LDP sid bit (%u) " - "from " LDP_ENV_SID_BIT - "! sid bit value %d (0x%x)", getpid (), - sb, ldp->sid_bit_val, ldp->sid_bit_val); - } - } + env_var_str = getenv (LDP_ENV_SID_BIT); + if (env_var_str) + { + u32 sb; + if (sscanf (env_var_str, "%u", &sb) != 1) + { + clib_warning ("LDP<%d>: WARNING: Invalid LDP sid bit specified in" + " the env var " LDP_ENV_SID_BIT " (%s)! sid bit " + "value %d (0x%x)", getpid (), env_var_str, + ldp->sid_bit_val, ldp->sid_bit_val); + } + else if (sb < LDP_SID_BIT_MIN) + { + ldp->sid_bit_val = (1 << LDP_SID_BIT_MIN); + ldp->sid_bit_mask = ldp->sid_bit_val - 1; - clib_time_init (&ldp->clib_time); - if (LDP_DEBUG > 0) - clib_warning ("LDP<%d>: LDP initialization: done!", getpid ()); + clib_warning ("LDP<%d>: WARNING: LDP sid bit (%u) specified in the" + " env var " LDP_ENV_SID_BIT " (%s) is too small. " + "Using LDP_SID_BIT_MIN (%d)! sid bit value %d (0x%x)", + getpid (), sb, env_var_str, LDP_SID_BIT_MIN, + ldp->sid_bit_val, ldp->sid_bit_val); + } + else if (sb > LDP_SID_BIT_MAX) + { + ldp->sid_bit_val = (1 << LDP_SID_BIT_MAX); + ldp->sid_bit_mask = ldp->sid_bit_val - 1; + + clib_warning ("LDP<%d>: WARNING: LDP sid bit (%u) specified in the" + " env var " LDP_ENV_SID_BIT " (%s) is too big. Using" + " LDP_SID_BIT_MAX (%d)! sid bit value %d (0x%x)", + getpid (), sb, env_var_str, LDP_SID_BIT_MAX, + ldp->sid_bit_val, ldp->sid_bit_val); } else { - fprintf (stderr, "\nLDP<%d>: ERROR: ldp_init: vppcom_app_create()" - " failed! rv = %d (%s)\n", - getpid (), rv, vppcom_retval_str (rv)); - ldp->init = 0; + ldp->sid_bit_val = (1 << sb); + ldp->sid_bit_mask = ldp->sid_bit_val - 1; + + LDBG (0, "LDP<%d>: configured LDP sid bit (%u) from " + LDP_ENV_SID_BIT "! sid bit value %d (0x%x)", getpid (), sb, + ldp->sid_bit_val, ldp->sid_bit_val); } } - return rv; + + clib_time_init (&ldp->clib_time); + LDBG (0, "LDP<%d>: LDP initialization: done!", getpid ()); + + return 0; } int @@ -1151,11 +1148,9 @@ socket (int domain, int type, int protocol) func_str = "vppcom_session_create"; - if (LDP_DEBUG > 0) - clib_warning ("LDP<%d>: : calling %s(): " - "proto %u (%s), is_nonblocking %u", - getpid (), func_str, proto, - vppcom_proto_str (proto), is_nonblocking); + LDBG (0, "LDP<%d>: : calling %s(): proto %u (%s), is_nonblocking %u", + getpid (), func_str, proto, vppcom_proto_str (proto), + is_nonblocking); sid = vppcom_session_create (proto, is_nonblocking); if (sid < 0) @@ -1179,8 +1174,7 @@ socket (int domain, int type, int protocol) { func_str = "libc_socket"; - if (LDP_DEBUG > 0) - clib_warning ("LDP<%d>: : calling %s()", getpid (), func_str); + LDBG (0, "LDP<%d>: : calling %s()", getpid (), func_str); rv = libc_socket (domain, type, protocol); } @@ -2765,10 +2759,8 @@ listen (int fd, int n) { func_str = "vppcom_session_listen"; - if (LDP_DEBUG > 0) - clib_warning - ("LDP<%d>: fd %d (0x%x): calling %s(): sid %u (0x%x), n %d", - getpid (), fd, fd, func_str, sid, sid, n); + LDBG (0, "LDP<%d>: fd %d (0x%x): calling %s(): sid %u (0x%x), n %d", + getpid (), fd, fd, func_str, sid, sid, n); rv = vppcom_session_listen (sid, n); if (rv != VPPCOM_OK) @@ -2781,9 +2773,8 @@ listen (int fd, int n) { func_str = "libc_listen"; - if (LDP_DEBUG > 0) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): n %d", - getpid (), fd, fd, func_str, n); + LDBG (0, "LDP<%d>: fd %d (0x%x): calling %s(): n %d", getpid (), fd, + fd, func_str, n); rv = libc_listen (fd, n); } @@ -2966,10 +2957,17 @@ epoll_create1 (int flags) if ((errno = -ldp_init ())) return -1; + if (ldp->vcl_needs_real_epoll) + { + rv = libc_epoll_create1 (flags); + ldp->vcl_needs_real_epoll = 0; + ldp->vcl_mq_epfd = rv; + LDBG (0, "LDP<%d>: created vcl epfd %u", getpid (), rv); + return rv; + } func_str = "vppcom_epoll_create"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: calling %s()", getpid (), func_str); + LDBG (1, "LDP<%d>: calling %s()", getpid (), func_str); rv = vppcom_epoll_create (); @@ -3007,121 +3005,105 @@ epoll_create (int size) int epoll_ctl (int epfd, int op, int fd, struct epoll_event *event) { - int rv; + u32 vep_idx = ldp_sid_from_fd (epfd), sid; const char *func_str; - u32 vep_idx = ldp_sid_from_fd (epfd); + int rv; if ((errno = -ldp_init ())) return -1; - if (PREDICT_TRUE (vep_idx != INVALID_SESSION_ID)) + if (PREDICT_FALSE (vep_idx == INVALID_SESSION_ID)) { - u32 sid = ldp_sid_from_fd (fd); + /* The LDP epoll_create1 always creates VCL epfd's. + * The app should never have a kernel base epoll fd unless it + * was acquired outside of the LD_PRELOAD process context. + * In any case, if we get one, punt it to libc_epoll_ctl. + */ + func_str = "libc_epoll_ctl"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: epfd %d (0x%x), vep_idx %d (0x%x), " - "sid %d (0x%x)", getpid (), epfd, epfd, - vep_idx, vep_idx, sid, sid); + LDBG (1, "LDP<%d>: epfd %d (0x%x): calling %s(): op %d, fd %d (0x%x)," + " event %p", getpid (), epfd, epfd, func_str, op, fd, fd, event); - if (sid != INVALID_SESSION_ID) - { - func_str = "vppcom_epoll_ctl"; + rv = libc_epoll_ctl (epfd, op, fd, event); + goto done; + } - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: epfd %d (0x%x): calling %s(): " - "vep_idx %d (0x%x), op %d, sid %u (0x%x), event %p", - getpid (), epfd, epfd, func_str, vep_idx, vep_idx, - sid, sid, event); + sid = ldp_sid_from_fd (fd); - rv = vppcom_epoll_ctl (vep_idx, op, sid, event); - if (rv != VPPCOM_OK) - { - errno = -rv; - rv = -1; - } + LDBG (0, "LDP<%d>: epfd %d (0x%x), vep_idx %d (0x%x), sid %d (0x%x)", + getpid (), epfd, epfd, vep_idx, vep_idx, sid, sid); + + if (sid != INVALID_SESSION_ID) + { + func_str = "vppcom_epoll_ctl"; + + LDBG (1, "LDP<%d>: epfd %d (0x%x): calling %s(): vep_idx %d (0x%x)," + " op %d, sid %u (0x%x), event %p", getpid (), epfd, epfd, + func_str, vep_idx, vep_idx, sid, sid, event); + + rv = vppcom_epoll_ctl (vep_idx, op, sid, event); + if (rv != VPPCOM_OK) + { + errno = -rv; + rv = -1; } - else + } + else + { + int libc_epfd; + u32 size = sizeof (epfd); + + func_str = "vppcom_session_attr[GET_LIBC_EPFD]"; + libc_epfd = vppcom_session_attr (vep_idx, VPPCOM_ATTR_GET_LIBC_EPFD, 0, + 0); + LDBG (1, "LDP<%d>: epfd %d (0x%x), vep_idx %d (0x%x): %s() " + "returned libc_epfd %d (0x%x)", getpid (), epfd, epfd, + vep_idx, vep_idx, func_str, libc_epfd, libc_epfd); + + if (!libc_epfd) { - int libc_epfd; - u32 size = sizeof (epfd); + func_str = "libc_epoll_create1"; - func_str = "vppcom_session_attr[GET_LIBC_EPFD]"; - libc_epfd = vppcom_session_attr (vep_idx, - VPPCOM_ATTR_GET_LIBC_EPFD, 0, 0); - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: epfd %d (0x%x), vep_idx %d (0x%x): " - "%s() returned libc_epfd %d (0x%x)", - getpid (), epfd, epfd, vep_idx, vep_idx, - func_str, libc_epfd, libc_epfd); + LDBG (1, "LDP<%d>: epfd %d (0x%x), vep_idx %d (0x%x): " + "calling %s(): EPOLL_CLOEXEC", getpid (), epfd, epfd, + vep_idx, vep_idx, func_str); - if (!libc_epfd) + libc_epfd = libc_epoll_create1 (EPOLL_CLOEXEC); + if (libc_epfd < 0) { - func_str = "libc_epoll_create1"; - - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: epfd %d (0x%x), vep_idx %d (0x%x): " - "calling %s(): EPOLL_CLOEXEC", - getpid (), epfd, epfd, vep_idx, vep_idx, - func_str); + rv = libc_epfd; + goto done; + } - libc_epfd = libc_epoll_create1 (EPOLL_CLOEXEC); - if (libc_epfd < 0) - { - rv = libc_epfd; - goto done; - } + func_str = "vppcom_session_attr[SET_LIBC_EPFD]"; + LDBG (1, "LDP<%d>: epfd %d (0x%x): calling %s(): vep_idx %d (0x%x)," + " VPPCOM_ATTR_SET_LIBC_EPFD, libc_epfd %d (0x%x), size %d", + getpid (), epfd, epfd, func_str, vep_idx, vep_idx, libc_epfd, + libc_epfd, size); - func_str = "vppcom_session_attr[SET_LIBC_EPFD]"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: epfd %d (0x%x): calling %s(): " - "vep_idx %d (0x%x), VPPCOM_ATTR_SET_LIBC_EPFD, " - "libc_epfd %d (0x%x), size %d", - getpid (), epfd, epfd, func_str, - vep_idx, vep_idx, libc_epfd, libc_epfd, size); - - rv = vppcom_session_attr (vep_idx, VPPCOM_ATTR_SET_LIBC_EPFD, - &libc_epfd, &size); - if (rv < 0) - { - errno = -rv; - rv = -1; - goto done; - } - } - else if (PREDICT_FALSE (libc_epfd < 0)) + rv = vppcom_session_attr (vep_idx, VPPCOM_ATTR_SET_LIBC_EPFD, + &libc_epfd, &size); + if (rv < 0) { - errno = -epfd; + errno = -rv; rv = -1; goto done; } - - func_str = "libc_epoll_ctl"; - - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: epfd %d (0x%x): calling %s(): " - "libc_epfd %d (0x%x), op %d, " - "fd %d (0x%x), event %p", - getpid (), epfd, epfd, func_str, - libc_epfd, libc_epfd, op, fd, fd, event); - - rv = libc_epoll_ctl (libc_epfd, op, fd, event); } - } - else - { - /* The LDP epoll_create1 always creates VCL epfd's. - * The app should never have a kernel base epoll fd unless it - * was acquired outside of the LD_PRELOAD process context. - * In any case, if we get one, punt it to libc_epoll_ctl. - */ + else if (PREDICT_FALSE (libc_epfd < 0)) + { + errno = -epfd; + rv = -1; + goto done; + } + func_str = "libc_epoll_ctl"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: epfd %d (0x%x): calling %s(): " - "op %d, fd %d (0x%x), event %p", - getpid (), epfd, epfd, func_str, op, fd, fd, event); + LDBG (1, "LDP<%d>: epfd %d (0x%x): calling %s(): libc_epfd %d (0x%x), " + "op %d, fd %d (0x%x), event %p", getpid (), epfd, epfd, func_str, + libc_epfd, libc_epfd, op, fd, fd, event); - rv = libc_epoll_ctl (epfd, op, fd, event); + rv = libc_epoll_ctl (libc_epfd, op, fd, event); } done: @@ -3144,15 +3126,13 @@ done: } static inline int -ldp_epoll_pwait (int epfd, struct epoll_event *events, - int maxevents, int timeout, const sigset_t * sigmask) +ldp_epoll_pwait (int epfd, struct epoll_event *events, int maxevents, + int timeout, const sigset_t * sigmask) { - const char *func_str; - int rv = 0; - double time_to_wait = (double) 0; - double time_out, now = 0; + double time_to_wait = (double) 0, time_out, now = 0; u32 vep_idx = ldp_sid_from_fd (epfd); - int libc_epfd; + int libc_epfd, rv = 0; + const char *func_str; if ((errno = -ldp_init ())) return -1; @@ -3163,6 +3143,9 @@ ldp_epoll_pwait (int epfd, struct epoll_event *events, return -1; } + if (epfd == ldp->vcl_mq_epfd) + return libc_epoll_pwait (epfd, events, maxevents, timeout, sigmask); + if (PREDICT_FALSE (vep_idx == INVALID_SESSION_ID)) { clib_warning ("LDP<%d>: ERROR: epfd %d (0x%x): bad vep_idx %d (0x%x)!", @@ -3183,24 +3166,19 @@ ldp_epoll_pwait (int epfd, struct epoll_event *events, goto done; } - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: epfd %d (0x%x): vep_idx %d (0x%x), " - "libc_epfd %d (0x%x), events %p, maxevents %d, " - "timeout %d, sigmask %p: time_to_wait %.02f", - getpid (), epfd, epfd, vep_idx, vep_idx, - libc_epfd, libc_epfd, events, maxevents, timeout, - sigmask, time_to_wait, time_out); + LDBG (2, "LDP<%d>: epfd %d (0x%x): vep_idx %d (0x%x), libc_epfd %d (0x%x), " + "events %p, maxevents %d, timeout %d, sigmask %p: time_to_wait %.02f", + getpid (), epfd, epfd, vep_idx, vep_idx, libc_epfd, libc_epfd, events, + maxevents, timeout, sigmask, time_to_wait, time_out); do { if (!ldp->epoll_wait_vcl) { func_str = "vppcom_epoll_wait"; - if (LDP_DEBUG > 3) - clib_warning ("LDP<%d>: epfd %d (0x%x): calling %s(): " - "vep_idx %d (0x%x), events %p, maxevents %d", - getpid (), epfd, epfd, func_str, - vep_idx, vep_idx, events, maxevents); + LDBG (3, "LDP<%d>: epfd %d (0x%x): calling %s(): vep_idx %d (0x%x)," + " events %p, maxevents %d", getpid (), epfd, epfd, func_str, + vep_idx, vep_idx, events, maxevents); rv = vppcom_epoll_wait (vep_idx, events, maxevents, 0); if (rv > 0) @@ -3222,12 +3200,10 @@ ldp_epoll_pwait (int epfd, struct epoll_event *events, { func_str = "libc_epoll_pwait"; - if (LDP_DEBUG > 3) - clib_warning ("LDP<%d>: epfd %d (0x%x): calling %s(): " - "libc_epfd %d (0x%x), events %p, " - "maxevents %d, sigmask %p", - getpid (), epfd, epfd, func_str, - libc_epfd, libc_epfd, events, maxevents, sigmask); + LDBG (3, "LDP<%d>: epfd %d (0x%x): calling %s(): libc_epfd %d " + "(0x%x), events %p, maxevents %d, sigmask %p", getpid (), + epfd, epfd, func_str, libc_epfd, libc_epfd, events, + maxevents, sigmask); rv = libc_epoll_pwait (libc_epfd, events, maxevents, 1, sigmask); if (rv != 0) diff --git a/src/vcl/vcl_bapi.c b/src/vcl/vcl_bapi.c index 311df64528c..55c38032f92 100644 --- a/src/vcl/vcl_bapi.c +++ b/src/vcl/vcl_bapi.c @@ -61,15 +61,35 @@ static void vcm->app_state = STATE_APP_ENABLED; } +static int +ssvm_segment_attach (char *name, ssvm_segment_type_t type, int fd) +{ + svm_fifo_segment_create_args_t _a, *a = &_a; + int rv; + + memset (a, 0, sizeof (*a)); + a->segment_name = (char *) name; + a->segment_type = type; + + if (type == SSVM_SEGMENT_MEMFD) + a->memfd_fd = fd; + + if ((rv = svm_fifo_segment_attach (a))) + { + clib_warning ("svm_fifo_segment_attach ('%s') failed", name); + return rv; + } + vec_reset_length (a->new_segment_indices); + return 0; +} + static void vl_api_application_attach_reply_t_handler (vl_api_application_attach_reply_t * mp) { - static svm_fifo_segment_create_args_t _a; - svm_fifo_segment_create_args_t *a = &_a; - int rv; + u32 n_fds = 0; + int *fds = 0; - memset (a, 0, sizeof (*a)); if (mp->retval) { clib_warning ("VCL<%d>: attach failed: %U", getpid (), @@ -77,30 +97,41 @@ vl_api_application_attach_reply_t_handler (vl_api_application_attach_reply_t * return; } - if (mp->segment_name_length == 0) + vcm->app_event_queue = uword_to_pointer (mp->app_event_queue_address, + svm_msg_q_t *); + if (mp->n_fds) { - clib_warning ("VCL<%d>: segment_name_length zero", getpid ()); - return; - } + vec_validate (fds, mp->n_fds); + vl_socket_client_recv_fd_msg (fds, mp->n_fds, 5); - a->segment_name = (char *) mp->segment_name; - a->segment_size = mp->segment_size; + if (mp->fd_flags & SESSION_FD_F_VPP_MQ_SEGMENT) + if (ssvm_segment_attach ("vpp-mq-seg", SSVM_SEGMENT_MEMFD, + fds[n_fds++])) + return; - ASSERT (mp->app_event_queue_address); + if (mp->fd_flags & SESSION_FD_F_MEMFD_SEGMENT) + if (ssvm_segment_attach ((char *) mp->segment_name, + SSVM_SEGMENT_MEMFD, fds[n_fds++])) + return; - /* Attach to the segment vpp created */ - rv = svm_fifo_segment_attach (a); - vec_reset_length (a->new_segment_indices); - if (PREDICT_FALSE (rv)) + if (mp->fd_flags & SESSION_FD_F_MQ_EVENTFD) + { + svm_msg_q_set_consumer_eventfd (vcm->app_event_queue, fds[n_fds]); + if (vcm->mqs_epfd < 0) + clib_unix_warning ("epoll_create() returned"); + vcl_mq_epoll_add_evfd (vcm->app_event_queue); + n_fds++; + } + + vec_free (fds); + } + else { - clib_warning ("VCL<%d>: svm_fifo_segment_attach ('%s') failed", - getpid (), mp->segment_name); - return; + if (ssvm_segment_attach ((char *) mp->segment_name, SSVM_SEGMENT_SHM, + -1)) + return; } - vcm->app_event_queue = - uword_to_pointer (mp->app_event_queue_address, svm_msg_q_t *); - vcm->app_state = STATE_APP_ATTACHED; } @@ -128,18 +159,19 @@ vl_api_disconnect_session_reply_t_handler (vl_api_disconnect_session_reply_t * static void vl_api_map_another_segment_t_handler (vl_api_map_another_segment_t * mp) { - static svm_fifo_segment_create_args_t _a; - svm_fifo_segment_create_args_t *a = &_a; - int rv; + ssvm_segment_type_t seg_type = SSVM_SEGMENT_SHM; + int fd = -1; vcm->mounting_segment = 1; - memset (a, 0, sizeof (*a)); - a->segment_name = (char *) mp->segment_name; - a->segment_size = mp->segment_size; - /* Attach to the segment vpp created */ - rv = svm_fifo_segment_attach (a); - vec_reset_length (a->new_segment_indices); - if (PREDICT_FALSE (rv)) + + if (mp->fd_flags) + { + vl_socket_client_recv_fd_msg (&fd, 1, 5); + seg_type = SSVM_SEGMENT_MEMFD; + } + + if (PREDICT_FALSE (ssvm_segment_attach ((char *) mp->segment_name, + seg_type, fd))) { clib_warning ("VCL<%d>: svm_fifo_segment_attach ('%s') failed", getpid (), mp->segment_name); @@ -163,6 +195,39 @@ vl_api_unmap_segment_t_handler (vl_api_unmap_segment_t * mp) VDBG (1, "Unmapped segment '%s'", mp->segment_name); } +static void + vl_api_app_cut_through_registration_add_t_handler + (vl_api_app_cut_through_registration_add_t * mp) +{ + vcl_cut_through_registration_t *ctr; + u32 mqc_index = ~0; + int *fds = 0; + + if (mp->n_fds) + { + ASSERT (mp->n_fds == 2); + vec_validate (fds, mp->n_fds); + vl_socket_client_recv_fd_msg (fds, mp->n_fds, 5); + } + + ctr = vcl_ct_registration_lock_and_alloc (); + ctr->mq = uword_to_pointer (mp->evt_q_address, svm_msg_q_t *); + ctr->peer_mq = uword_to_pointer (mp->peer_evt_q_address, svm_msg_q_t *); + VDBG (0, "Adding ct registration %u", vcl_ct_registration_index (ctr)); + + if (mp->fd_flags & SESSION_FD_F_MQ_EVENTFD) + { + svm_msg_q_set_consumer_eventfd (ctr->mq, fds[0]); + svm_msg_q_set_producer_eventfd (ctr->peer_mq, fds[1]); + mqc_index = vcl_mq_epoll_add_evfd (ctr->mq); + ctr->epoll_evt_conn_index = mqc_index; + vec_free (fds); + } + vcl_ct_registration_lookup_add (mp->evt_q_address, + vcl_ct_registration_index (ctr)); + vcl_ct_registration_unlock (); +} + static void vl_api_disconnect_session_t_handler (vl_api_disconnect_session_t * mp) { @@ -483,19 +548,20 @@ vl_api_accept_session_t_handler (vl_api_accept_session_t * mp) VCL_SESSION_UNLOCK (); } -#define foreach_sock_msg \ -_(SESSION_ENABLE_DISABLE_REPLY, session_enable_disable_reply) \ -_(BIND_SOCK_REPLY, bind_sock_reply) \ -_(UNBIND_SOCK_REPLY, unbind_sock_reply) \ -_(ACCEPT_SESSION, accept_session) \ -_(CONNECT_SESSION_REPLY, connect_session_reply) \ -_(DISCONNECT_SESSION, disconnect_session) \ -_(DISCONNECT_SESSION_REPLY, disconnect_session_reply) \ -_(RESET_SESSION, reset_session) \ -_(APPLICATION_ATTACH_REPLY, application_attach_reply) \ -_(APPLICATION_DETACH_REPLY, application_detach_reply) \ -_(MAP_ANOTHER_SEGMENT, map_another_segment) \ -_(UNMAP_SEGMENT, unmap_segment) +#define foreach_sock_msg \ +_(SESSION_ENABLE_DISABLE_REPLY, session_enable_disable_reply) \ +_(BIND_SOCK_REPLY, bind_sock_reply) \ +_(UNBIND_SOCK_REPLY, unbind_sock_reply) \ +_(ACCEPT_SESSION, accept_session) \ +_(CONNECT_SESSION_REPLY, connect_session_reply) \ +_(DISCONNECT_SESSION, disconnect_session) \ +_(DISCONNECT_SESSION_REPLY, disconnect_session_reply) \ +_(RESET_SESSION, reset_session) \ +_(APPLICATION_ATTACH_REPLY, application_attach_reply) \ +_(APPLICATION_DETACH_REPLY, application_detach_reply) \ +_(MAP_ANOTHER_SEGMENT, map_another_segment) \ +_(UNMAP_SEGMENT, unmap_segment) \ +_(APP_CUT_THROUGH_REGISTRATION_ADD, app_cut_through_registration_add) \ void vppcom_api_hookup (void) @@ -547,7 +613,8 @@ vppcom_app_send_attach (void) (vcm->cfg.app_scope_local ? APP_OPTIONS_FLAGS_USE_LOCAL_SCOPE : 0) | (vcm->cfg.app_scope_global ? APP_OPTIONS_FLAGS_USE_GLOBAL_SCOPE : 0) | (app_is_proxy ? APP_OPTIONS_FLAGS_IS_PROXY : 0) | - APP_OPTIONS_FLAGS_USE_MQ_FOR_CTRL_MSGS; + APP_OPTIONS_FLAGS_USE_MQ_FOR_CTRL_MSGS | + (vcm->cfg.use_mq_eventfd ? APP_OPTIONS_FLAGS_EVT_MQ_USE_EVENTFD : 0); bmp->options[APP_OPTIONS_PROXY_TRANSPORT] = (u64) ((vcm->cfg.app_proxy_transport_tcp ? 1 << TRANSPORT_PROTO_TCP : 0) | (vcm->cfg.app_proxy_transport_udp ? 1 << TRANSPORT_PROTO_UDP : 0)); @@ -709,31 +776,50 @@ vppcom_connect_to_vpp (char *app_name) { api_main_t *am = &api_main; vppcom_cfg_t *vcl_cfg = &vcm->cfg; - int rv = VPPCOM_OK; - - if (!vcl_cfg->vpp_api_filename) - vcl_cfg->vpp_api_filename = format (0, "/vpe-api%c", 0); - VDBG (0, "VCL<%d>: app (%s) connecting to VPP api (%s)...", - getpid (), app_name, vcl_cfg->vpp_api_filename); - - if (vl_client_connect_to_vlib ((char *) vcl_cfg->vpp_api_filename, app_name, - vcm->cfg.vpp_api_q_length) < 0) + if (vcl_cfg->vpp_api_socket_name) { - clib_warning ("VCL<%d>: app (%s) connect failed!", getpid (), app_name); - rv = VPPCOM_ECONNREFUSED; + if (vl_socket_client_connect ((char *) vcl_cfg->vpp_api_socket_name, + app_name, 0 /* default rx/tx buffer */ )) + { + clib_warning ("VCL<%d>: app (%s) socket connect failed!", + getpid (), app_name); + return VPPCOM_ECONNREFUSED; + } + + if (vl_socket_client_init_shm (0)) + { + clib_warning ("VCL<%d>: app (%s) init shm failed!", + getpid (), app_name); + return VPPCOM_ECONNREFUSED; + } } else { - vcm->vl_input_queue = am->shmem_hdr->vl_input_queue; - vcm->my_client_index = (u32) am->my_client_index; - vcm->app_state = STATE_APP_CONN_VPP; + if (!vcl_cfg->vpp_api_filename) + vcl_cfg->vpp_api_filename = format (0, "/vpe-api%c", 0); + + VDBG (0, "VCL<%d>: app (%s) connecting to VPP api (%s)...", getpid (), + app_name, vcl_cfg->vpp_api_filename); + + if (vl_client_connect_to_vlib ((char *) vcl_cfg->vpp_api_filename, + app_name, vcm->cfg.vpp_api_q_length) < 0) + { + clib_warning ("VCL<%d>: app (%s) connect failed!", getpid (), + app_name); + return VPPCOM_ECONNREFUSED; + } - VDBG (0, "VCL<%d>: app (%s) is connected to VPP!", getpid (), app_name); } + vcm->vl_input_queue = am->shmem_hdr->vl_input_queue; + vcm->my_client_index = (u32) am->my_client_index; + vcm->app_state = STATE_APP_CONN_VPP; + + VDBG (0, "VCL<%d>: app (%s) is connected to VPP!", getpid (), app_name); + vcl_evt (VCL_EVT_INIT, vcm); - return rv; + return VPPCOM_OK; } /* diff --git a/src/vcl/vcl_cfg.c b/src/vcl/vcl_cfg.c index ba3e53e2253..f7bddefbb3b 100644 --- a/src/vcl/vcl_cfg.c +++ b/src/vcl/vcl_cfg.c @@ -49,9 +49,11 @@ vppcom_cfg_init (vppcom_cfg_t * vcl_cfg) vcl_cfg->event_log_path = "/dev/shm"; } -#define VCL_CFG_DBG(_lvl, _fmt, _args...) \ +#define VCFG_DBG(_lvl, _fmt, _args...) \ +{ \ if (vcm->debug > _lvl) \ - fprintf (stderr, _fmt, ##_args) + fprintf (stderr, _fmt "\n", ##_args); \ +} void vppcom_cfg_heapsize (char *conf_fname) { @@ -71,17 +73,16 @@ vppcom_cfg_heapsize (char *conf_fname) fp = fopen (conf_fname, "r"); if (fp == NULL) { - VCL_CFG_DBG (0, "VCL<%d>: using default heapsize %lu (0x%lx)", - getpid (), vcl_cfg->heapsize, vcl_cfg->heapsize); + VCFG_DBG (0, "VCL<%d>: using default heapsize %lu (0x%lx)", + getpid (), vcl_cfg->heapsize, vcl_cfg->heapsize); goto defaulted; } argv = calloc (1, sizeof (char *)); if (argv == NULL) { - VCL_CFG_DBG (0, "VCL<%d>: calloc failed, using default heapsize %lu" - " (0x%lx)", getpid (), vcl_cfg->heapsize, - vcl_cfg->heapsize); + VCFG_DBG (0, "VCL<%d>: calloc failed, using default heapsize %lu" + " (0x%lx)", getpid (), vcl_cfg->heapsize, vcl_cfg->heapsize); goto defaulted; } @@ -98,18 +99,18 @@ vppcom_cfg_heapsize (char *conf_fname) char **tmp = realloc (argv, argc * sizeof (char *)); if (tmp == NULL) { - VCL_CFG_DBG (0, "VCL<%d>: realloc failed, using default " - "heapsize %lu (0x%lx)", getpid (), - vcl_cfg->heapsize, vcl_cfg->heapsize); + VCFG_DBG (0, "VCL<%d>: realloc failed, using default " + "heapsize %lu (0x%lx)", getpid (), + vcl_cfg->heapsize, vcl_cfg->heapsize); goto defaulted; } argv = tmp; arg = strndup (p, 1024); if (arg == NULL) { - VCL_CFG_DBG (0, "VCL<%d>: strndup failed, using default " - "heapsize %ld (0x%lx)", getpid (), - vcl_cfg->heapsize, vcl_cfg->heapsize); + VCFG_DBG (0, "VCL<%d>: strndup failed, using default " + "heapsize %ld (0x%lx)", getpid (), + vcl_cfg->heapsize, vcl_cfg->heapsize); goto defaulted; } argv[argc - 1] = arg; @@ -123,9 +124,8 @@ vppcom_cfg_heapsize (char *conf_fname) char **tmp = realloc (argv, (argc + 1) * sizeof (char *)); if (tmp == NULL) { - VCL_CFG_DBG (0, "VCL<%d>: realloc failed, using default heapsize %ld " - "(0x%lx)", getpid (), vcl_cfg->heapsize, - vcl_cfg->heapsize); + VCFG_DBG (0, "VCL<%d>: realloc failed, using default heapsize %ld " + "(0x%lx)", getpid (), vcl_cfg->heapsize, vcl_cfg->heapsize); goto defaulted; } argv = tmp; @@ -151,9 +151,9 @@ vppcom_cfg_heapsize (char *conf_fname) } if (size == 0) { - VCL_CFG_DBG (0, "VCL<%d>: parse error '%s %s', using default " - "heapsize %ld (0x%lx)", getpid (), argv[i], - argv[i + 1], vcl_cfg->heapsize, vcl_cfg->heapsize); + VCFG_DBG (0, "VCL<%d>: parse error '%s %s', using default " + "heapsize %ld (0x%lx)", getpid (), argv[i], + argv[i + 1], vcl_cfg->heapsize, vcl_cfg->heapsize); goto defaulted; } @@ -163,9 +163,9 @@ vppcom_cfg_heapsize (char *conf_fname) vcl_cfg->heapsize = size << 20; else { - VCL_CFG_DBG (0, "VCL<%d>: parse error '%s %s', using default " - "heapsize %ld (0x%lx)", getpid (), argv[i], - argv[i + 1], vcl_cfg->heapsize, vcl_cfg->heapsize); + VCFG_DBG (0, "VCL<%d>: parse error '%s %s', using default " + "heapsize %ld (0x%lx)", getpid (), argv[i], + argv[i + 1], vcl_cfg->heapsize, vcl_cfg->heapsize); goto defaulted; } } @@ -181,10 +181,10 @@ defaulted: MAP_SHARED | MAP_ANONYMOUS, -1, 0); if (vcl_mem == MAP_FAILED) { - VCL_CFG_DBG (0, "VCL<%d>: ERROR: mmap(0, %ld == 0x%lx, " - "PROT_READ | PROT_WRITE,MAP_SHARED | MAP_ANONYMOUS, " - "-1, 0) failed!", getpid (), vcl_cfg->heapsize, - vcl_cfg->heapsize); + VCFG_DBG (0, "VCL<%d>: ERROR: mmap(0, %ld == 0x%lx, " + "PROT_READ | PROT_WRITE,MAP_SHARED | MAP_ANONYMOUS, " + "-1, 0) failed!", getpid (), vcl_cfg->heapsize, + vcl_cfg->heapsize); ASSERT (vcl_mem != MAP_FAILED); return; } @@ -206,8 +206,8 @@ defaulted: clib_memcpy (vcl_mem, &_vppcom_main, sizeof (_vppcom_main)); vcm = vcl_mem; - VDBG (0, "VCL<%d>: allocated VCL heap = %p, size %lld (0x%llx)", getpid (), - heap, vcl_cfg->heapsize, vcl_cfg->heapsize); + VCFG_DBG (0, "VCL<%d>: allocated VCL heap = %p, size %ld (0x%lx)", + getpid (), heap, vcl_cfg->heapsize, vcl_cfg->heapsize); } void @@ -217,30 +217,29 @@ vppcom_cfg_read_file (char *conf_fname) int fd; unformat_input_t _input, *input = &_input; unformat_input_t _line_input, *line_input = &_line_input; - u8 vc_cfg_input = 0; - u8 *chroot_path; + u8 vc_cfg_input = 0, *chroot_path; struct stat s; u32 uid, gid, q_len; fd = open (conf_fname, O_RDONLY); if (fd < 0) { - VDBG (0, "VCL<%d>: using default configuration.", getpid (), - conf_fname); + VCFG_DBG (0, "VCL<%d>: using default configuration.", getpid ()); goto file_done; } if (fstat (fd, &s) < 0) { - VDBG (0, "VCL<%d>: failed to stat `%s', using default configuration", - getpid (), conf_fname); + VCFG_DBG (0, + "VCL<%d>: failed to stat `%s', using default configuration", + getpid (), conf_fname); goto file_done; } if (!(S_ISREG (s.st_mode) || S_ISLNK (s.st_mode))) { - VDBG (0, "VCL<%d>: not a regular file `%s', using default " - "configuration", getpid (), conf_fname); + VCFG_DBG (0, "VCL<%d>: not a regular file `%s', using default " + "configuration", getpid (), conf_fname); goto file_done; } @@ -262,9 +261,10 @@ vppcom_cfg_read_file (char *conf_fname) if (unformat (line_input, "heapsize %s", &chroot_path)) { vec_terminate_c_string (chroot_path); - VDBG (0, "VCL<%d>: configured heapsize %s, actual heapsize %lld" - " (0x%llx)", getpid (), chroot_path, vcl_cfg->heapsize, - vcl_cfg->heapsize); + VCFG_DBG (0, + "VCL<%d>: configured heapsize %s, actual heapsize %ld" + " (0x%lx)", getpid (), chroot_path, vcl_cfg->heapsize, + vcl_cfg->heapsize); vec_free (chroot_path); } else if (unformat (line_input, "api-prefix %s", &chroot_path)) @@ -276,176 +276,191 @@ vppcom_cfg_read_file (char *conf_fname) chroot_path, 0); vl_set_memory_root_path ((char *) chroot_path); - VDBG (0, "VCL<%d>: configured api-prefix (%s) and api filename" - " (%s)", getpid (), chroot_path, - vcl_cfg->vpp_api_filename); + VCFG_DBG (0, + "VCL<%d>: configured api-prefix (%s) and api filename" + " (%s)", getpid (), chroot_path, + vcl_cfg->vpp_api_filename); chroot_path = 0; /* Don't vec_free() it! */ } + else if (unformat (line_input, "api-socket-name %s", + &vcl_cfg->vpp_api_socket_name)) + { + vec_terminate_c_string (vcl_cfg->vpp_api_socket_name); + VCFG_DBG (0, "VCL<%d>: configured api-socket-name (%s)", + getpid (), vcl_cfg->vpp_api_socket_name); + } else if (unformat (line_input, "vpp-api-q-length %d", &q_len)) { if (q_len < vcl_cfg->vpp_api_q_length) { - clib_warning ("VCL<%d>: ERROR: configured vpp-api-q-length " - "(%u) is too small! Using default: %u ", - getpid (), q_len, vcl_cfg->vpp_api_q_length); + fprintf (stderr, + "VCL<%d>: ERROR: configured vpp-api-q-length " + "(%u) is too small! Using default: %u ", getpid (), + q_len, vcl_cfg->vpp_api_q_length); } else { vcl_cfg->vpp_api_q_length = q_len; - VDBG (0, "VCL<%d>: configured vpp-api-q-length %u", - getpid (), vcl_cfg->vpp_api_q_length); + VCFG_DBG (0, "VCL<%d>: configured vpp-api-q-length %u", + getpid (), vcl_cfg->vpp_api_q_length); } } else if (unformat (line_input, "uid %d", &uid)) { vl_set_memory_uid (uid); - VDBG (0, "VCL<%d>: configured uid %d", getpid (), uid); + VCFG_DBG (0, "VCL<%d>: configured uid %d", getpid (), uid); } else if (unformat (line_input, "gid %d", &gid)) { vl_set_memory_gid (gid); - VDBG (0, "VCL<%d>: configured gid %d", getpid (), gid); + VCFG_DBG (0, "VCL<%d>: configured gid %d", getpid (), gid); } - else if (unformat (line_input, "segment-baseva 0x%lx", + else if (unformat (line_input, "segment-baseva 0x%x", &vcl_cfg->segment_baseva)) { - VDBG (0, "VCL<%d>: configured segment_baseva 0x%lx", getpid (), - vcl_cfg->segment_baseva); + VCFG_DBG (0, "VCL<%d>: configured segment_baseva 0x%lx", + getpid (), vcl_cfg->segment_baseva); } - else if (unformat (line_input, "segment-size 0x%lx", + else if (unformat (line_input, "segment-size 0x%x", &vcl_cfg->segment_size)) { - VDBG (0, "VCL<%d>: configured segment_size 0x%lx (%ld)", - getpid (), vcl_cfg->segment_size, vcl_cfg->segment_size); + VCFG_DBG (0, "VCL<%d>: configured segment_size 0x%x (%d)", + getpid (), vcl_cfg->segment_size, + vcl_cfg->segment_size); } - else if (unformat (line_input, "segment-size %ld", + else if (unformat (line_input, "segment-size %d", &vcl_cfg->segment_size)) { - VDBG (0, "VCL<%d>: configured segment_size %ld (0x%lx)", - getpid (), vcl_cfg->segment_size, vcl_cfg->segment_size); + VCFG_DBG (0, "VCL<%d>: configured segment_size %d (0x%x)", + getpid (), vcl_cfg->segment_size, + vcl_cfg->segment_size); } - else if (unformat (line_input, "add-segment-size 0x%lx", + else if (unformat (line_input, "add-segment-size 0x%x", &vcl_cfg->add_segment_size)) { - VDBG (0, "VCL<%d>: configured add_segment_size 0x%lx (%ld)", - getpid (), vcl_cfg->add_segment_size, - vcl_cfg->add_segment_size); + VCFG_DBG (0, "VCL<%d>: configured add_segment_size 0x%x (%d)", + getpid (), vcl_cfg->add_segment_size, + vcl_cfg->add_segment_size); } - else if (unformat (line_input, "add-segment-size %ld", + else if (unformat (line_input, "add-segment-size %d", &vcl_cfg->add_segment_size)) { - VDBG (0, "VCL<%d>: configured add_segment_size %ld (0x%lx)", - getpid (), vcl_cfg->add_segment_size, - vcl_cfg->add_segment_size); + VCFG_DBG (0, "VCL<%d>: configured add_segment_size %d (0x%x)", + getpid (), vcl_cfg->add_segment_size, + vcl_cfg->add_segment_size); } else if (unformat (line_input, "preallocated-fifo-pairs %d", &vcl_cfg->preallocated_fifo_pairs)) { - VDBG (0, "VCL<%d>: configured preallocated_fifo_pairs %d " - "(0x%x)", getpid (), vcl_cfg->preallocated_fifo_pairs, - vcl_cfg->preallocated_fifo_pairs); + VCFG_DBG (0, "VCL<%d>: configured preallocated_fifo_pairs %d " + "(0x%x)", getpid (), vcl_cfg->preallocated_fifo_pairs, + vcl_cfg->preallocated_fifo_pairs); } else if (unformat (line_input, "rx-fifo-size 0x%lx", &vcl_cfg->rx_fifo_size)) { - VDBG (0, "VCL<%d>: configured rx_fifo_size 0x%lx (%ld)", - getpid (), vcl_cfg->rx_fifo_size, vcl_cfg->rx_fifo_size); + VCFG_DBG (0, "VCL<%d>: configured rx_fifo_size 0x%x (%d)", + getpid (), vcl_cfg->rx_fifo_size, + vcl_cfg->rx_fifo_size); } - else if (unformat (line_input, "rx-fifo-size %ld", + else if (unformat (line_input, "rx-fifo-size %d", &vcl_cfg->rx_fifo_size)) { - VDBG (0, "VCL<%d>: configured rx_fifo_size %ld (0x%lx)", - getpid (), vcl_cfg->rx_fifo_size, vcl_cfg->rx_fifo_size); + VCFG_DBG (0, "VCL<%d>: configured rx_fifo_size %d (0x%x)", + getpid (), vcl_cfg->rx_fifo_size, + vcl_cfg->rx_fifo_size); } else if (unformat (line_input, "tx-fifo-size 0x%lx", &vcl_cfg->tx_fifo_size)) { - VDBG (0, "VCL<%d>: configured tx_fifo_size 0x%lx (%ld)", - getpid (), vcl_cfg->tx_fifo_size, vcl_cfg->tx_fifo_size); + VCFG_DBG (0, "VCL<%d>: configured tx_fifo_size 0x%x (%d)", + getpid (), vcl_cfg->tx_fifo_size, + vcl_cfg->tx_fifo_size); } else if (unformat (line_input, "tx-fifo-size %ld", &vcl_cfg->tx_fifo_size)) { - VDBG (0, "VCL<%d>: configured tx_fifo_size %ld (0x%lx)", - getpid (), vcl_cfg->tx_fifo_size, vcl_cfg->tx_fifo_size); + VCFG_DBG (0, "VCL<%d>: configured tx_fifo_size %d (0x%x)", + getpid (), vcl_cfg->tx_fifo_size, + vcl_cfg->tx_fifo_size); } else if (unformat (line_input, "event-queue-size 0x%lx", &vcl_cfg->event_queue_size)) { - VDBG (0, "VCL<%d>: configured event_queue_size 0x%lx (%ld)", - getpid (), vcl_cfg->event_queue_size, - vcl_cfg->event_queue_size); + VCFG_DBG (0, "VCL<%d>: configured event_queue_size 0x%x (%d)", + getpid (), vcl_cfg->event_queue_size, + vcl_cfg->event_queue_size); } else if (unformat (line_input, "event-queue-size %ld", &vcl_cfg->event_queue_size)) { - VDBG (0, "VCL<%d>: configured event_queue_size %ld (0x%lx)", - getpid (), vcl_cfg->event_queue_size, - vcl_cfg->event_queue_size); + VCFG_DBG (0, "VCL<%d>: configured event_queue_size %d (0x%x)", + getpid (), vcl_cfg->event_queue_size, + vcl_cfg->event_queue_size); } else if (unformat (line_input, "listen-queue-size 0x%lx", &vcl_cfg->listen_queue_size)) { - VDBG (0, "VCL<%d>: configured listen_queue_size 0x%lx (%ld)", - getpid (), vcl_cfg->listen_queue_size, - vcl_cfg->listen_queue_size); + VCFG_DBG (0, "VCL<%d>: configured listen_queue_size 0x%x (%u)", + getpid (), vcl_cfg->listen_queue_size, + vcl_cfg->listen_queue_size); } else if (unformat (line_input, "listen-queue-size %ld", &vcl_cfg->listen_queue_size)) { - VDBG (0, "VCL<%d>: configured listen_queue_size %ld (0x%lx)", - getpid (), vcl_cfg->listen_queue_size, - vcl_cfg->listen_queue_size); + VCFG_DBG (0, "VCL<%d>: configured listen_queue_size %u (0x%x)", + getpid (), vcl_cfg->listen_queue_size, + vcl_cfg->listen_queue_size); } else if (unformat (line_input, "app-timeout %f", &vcl_cfg->app_timeout)) { - VDBG (0, "VCL<%d>: configured app_timeout %f", - getpid (), vcl_cfg->app_timeout); + VCFG_DBG (0, "VCL<%d>: configured app_timeout %f", + getpid (), vcl_cfg->app_timeout); } else if (unformat (line_input, "session-timeout %f", &vcl_cfg->session_timeout)) { - VDBG (0, "VCL<%d>: configured session_timeout %f", - getpid (), vcl_cfg->session_timeout); + VCFG_DBG (0, "VCL<%d>: configured session_timeout %f", + getpid (), vcl_cfg->session_timeout); } else if (unformat (line_input, "accept-timeout %f", &vcl_cfg->accept_timeout)) { - VDBG (0, "VCL<%d>: configured accept_timeout %f", - getpid (), vcl_cfg->accept_timeout); + VCFG_DBG (0, "VCL<%d>: configured accept_timeout %f", + getpid (), vcl_cfg->accept_timeout); } else if (unformat (line_input, "app-proxy-transport-tcp")) { vcl_cfg->app_proxy_transport_tcp = 1; - VDBG (0, "VCL<%d>: configured app_proxy_transport_tcp (%d)", - getpid (), vcl_cfg->app_proxy_transport_tcp); + VCFG_DBG (0, "VCL<%d>: configured app_proxy_transport_tcp (%d)", + getpid (), vcl_cfg->app_proxy_transport_tcp); } else if (unformat (line_input, "app-proxy-transport-udp")) { vcl_cfg->app_proxy_transport_udp = 1; - VDBG (0, "VCL<%d>: configured app_proxy_transport_udp (%d)", - getpid (), vcl_cfg->app_proxy_transport_udp); + VCFG_DBG (0, "VCL<%d>: configured app_proxy_transport_udp (%d)", + getpid (), vcl_cfg->app_proxy_transport_udp); } else if (unformat (line_input, "app-scope-local")) { vcl_cfg->app_scope_local = 1; - VDBG (0, "VCL<%d>: configured app_scope_local (%d)", - getpid (), vcl_cfg->app_scope_local); + VCFG_DBG (0, "VCL<%d>: configured app_scope_local (%d)", + getpid (), vcl_cfg->app_scope_local); } else if (unformat (line_input, "app-scope-global")) { vcl_cfg->app_scope_global = 1; - VDBG (0, "VCL<%d>: configured app_scope_global (%d)", - getpid (), vcl_cfg->app_scope_global); + VCFG_DBG (0, "VCL<%d>: configured app_scope_global (%d)", + getpid (), vcl_cfg->app_scope_global); } else if (unformat (line_input, "namespace-secret %lu", &vcl_cfg->namespace_secret)) { - VDBG (0, "VCL<%d>: configured namespace_secret %lu (0x%lx)", - getpid (), vcl_cfg->namespace_secret, - vcl_cfg->namespace_secret); + VCFG_DBG (0, "VCL<%d>: configured namespace_secret %lu (0x%lx)", + getpid (), vcl_cfg->namespace_secret, + vcl_cfg->namespace_secret); } else if (unformat (line_input, "namespace-id %v", &vcl_cfg->namespace_id)) @@ -455,19 +470,25 @@ vppcom_cfg_read_file (char *conf_fname) if (nsid_vec_len > max_nsid_vec_len) { _vec_len (vcl_cfg->namespace_id) = max_nsid_vec_len; - VDBG (0, "VCL<%d>: configured namespace_id is too long," - " truncated to %d characters!", - getpid (), max_nsid_vec_len); + VCFG_DBG (0, "VCL<%d>: configured namespace_id is too long," + " truncated to %d characters!", + getpid (), max_nsid_vec_len); } - VDBG (0, "VCL<%d>: configured namespace_id %v", - getpid (), vcl_cfg->namespace_id); + VCFG_DBG (0, "VCL<%d>: configured namespace_id %s", + getpid (), (char *) vcl_cfg->namespace_id); + } + else if (unformat (line_input, "use-mq-eventfd")) + { + vcl_cfg->use_mq_eventfd = 1; + VCFG_DBG (0, "VCL<%d>: configured with mq with eventfd", + getpid ()); } else if (unformat (line_input, "}")) { vc_cfg_input = 0; - VDBG (0, "VCL<%d>: completed parsing vppcom config!", - getpid ()); + VCFG_DBG (0, "VCL<%d>: completed parsing vppcom config!", + getpid ()); goto input_done; } else @@ -502,15 +523,15 @@ vppcom_cfg (vppcom_cfg_t * vcl_cfg) u32 tmp; if (sscanf (env_var_str, "%u", &tmp) != 1) { - VCL_CFG_DBG (0, "VCL<%d>: WARNING: Invalid debug level specified " - "in the environment variable " VPPCOM_ENV_DEBUG - " (%s)!\n", getpid (), env_var_str); + VCFG_DBG (0, "VCL<%d>: WARNING: Invalid debug level specified " + "in the environment variable " VPPCOM_ENV_DEBUG + " (%s)!\n", getpid (), env_var_str); } else { vcm->debug = tmp; - VCL_CFG_DBG (0, "VCL<%d>: configured VCL debug level (%u) from " - VPPCOM_ENV_DEBUG "!", getpid (), vcm->debug); + VCFG_DBG (0, "VCL<%d>: configured VCL debug level (%u) from " + VPPCOM_ENV_DEBUG "!", getpid (), vcm->debug); } } conf_fname = getenv (VPPCOM_ENV_CONF); @@ -527,9 +548,9 @@ vppcom_cfg (vppcom_cfg_t * vcl_cfg) vcl_cfg->vpp_api_filename = format (0, "/%s-vpe-api%c", env_var_str, 0); vl_set_memory_root_path ((char *) env_var_str); - VDBG (0, "VCL<%d>: configured api prefix (%s) and filename (%s) " - "from " VPPCOM_ENV_API_PREFIX "!", - getpid (), env_var_str, vcl_cfg->vpp_api_filename); + VCFG_DBG (0, "VCL<%d>: configured api prefix (%s) and filename (%s) " + "from " VPPCOM_ENV_API_PREFIX "!", getpid (), env_var_str, + vcl_cfg->vpp_api_filename); } env_var_str = getenv (VPPCOM_ENV_APP_NAMESPACE_ID); if (env_var_str) @@ -540,54 +561,63 @@ vppcom_cfg (vppcom_cfg_t * vcl_cfg) vec_validate (vcm->cfg.namespace_id, ns_id_vec_len - 1); clib_memcpy (vcm->cfg.namespace_id, env_var_str, ns_id_vec_len); - VDBG (0, "VCL<%d>: configured namespace_id (%v) from " - VPPCOM_ENV_APP_NAMESPACE_ID "!", getpid (), - vcm->cfg.namespace_id); + VCFG_DBG (0, "VCL<%d>: configured namespace_id (%s) from " + VPPCOM_ENV_APP_NAMESPACE_ID "!", getpid (), + (char *) vcm->cfg.namespace_id); } env_var_str = getenv (VPPCOM_ENV_APP_NAMESPACE_SECRET); if (env_var_str) { u64 tmp; if (sscanf (env_var_str, "%lu", &tmp) != 1) - clib_warning ("VCL<%d>: WARNING: Invalid namespace secret " - "specified in the environment variable " - VPPCOM_ENV_APP_NAMESPACE_SECRET - " (%s)!\n", getpid (), env_var_str); + { + VCFG_DBG (0, "VCL<%d>: WARNING: Invalid namespace secret specified" + " in the environment variable " + VPPCOM_ENV_APP_NAMESPACE_SECRET " (%s)!\n", getpid (), + env_var_str); + } else { vcm->cfg.namespace_secret = tmp; - VDBG (0, "VCL<%d>: configured namespace secret (%lu) from " - VPPCOM_ENV_APP_NAMESPACE_SECRET "!", getpid (), - vcm->cfg.namespace_secret); + VCFG_DBG (0, "VCL<%d>: configured namespace secret (%lu) from " + VPPCOM_ENV_APP_NAMESPACE_SECRET "!", getpid (), + vcm->cfg.namespace_secret); } } if (getenv (VPPCOM_ENV_APP_PROXY_TRANSPORT_TCP)) { vcm->cfg.app_proxy_transport_tcp = 1; - VDBG (0, "VCL<%d>: configured app_proxy_transport_tcp (%u) from " - VPPCOM_ENV_APP_PROXY_TRANSPORT_TCP "!", getpid (), - vcm->cfg.app_proxy_transport_tcp); + VCFG_DBG (0, "VCL<%d>: configured app_proxy_transport_tcp (%u) from " + VPPCOM_ENV_APP_PROXY_TRANSPORT_TCP "!", getpid (), + vcm->cfg.app_proxy_transport_tcp); } if (getenv (VPPCOM_ENV_APP_PROXY_TRANSPORT_UDP)) { vcm->cfg.app_proxy_transport_udp = 1; - VDBG (0, "VCL<%d>: configured app_proxy_transport_udp (%u) from " - VPPCOM_ENV_APP_PROXY_TRANSPORT_UDP "!", getpid (), - vcm->cfg.app_proxy_transport_udp); + VCFG_DBG (0, "VCL<%d>: configured app_proxy_transport_udp (%u) from " + VPPCOM_ENV_APP_PROXY_TRANSPORT_UDP "!", getpid (), + vcm->cfg.app_proxy_transport_udp); } if (getenv (VPPCOM_ENV_APP_SCOPE_LOCAL)) { vcm->cfg.app_scope_local = 1; - VDBG (0, "VCL<%d>: configured app_scope_local (%u) from " - VPPCOM_ENV_APP_SCOPE_LOCAL "!", getpid (), - vcm->cfg.app_scope_local); + VCFG_DBG (0, "VCL<%d>: configured app_scope_local (%u) from " + VPPCOM_ENV_APP_SCOPE_LOCAL "!", getpid (), + vcm->cfg.app_scope_local); } if (getenv (VPPCOM_ENV_APP_SCOPE_GLOBAL)) { vcm->cfg.app_scope_global = 1; - VDBG (0, "VCL<%d>: configured app_scope_global (%u) from " - VPPCOM_ENV_APP_SCOPE_GLOBAL "!", getpid (), - vcm->cfg.app_scope_global); + VCFG_DBG (0, "VCL<%d>: configured app_scope_global (%u) from " + VPPCOM_ENV_APP_SCOPE_GLOBAL "!", getpid (), + vcm->cfg.app_scope_global); + } + env_var_str = getenv (VPPCOM_ENV_VPP_API_SOCKET); + if (env_var_str) + { + vcm->cfg.vpp_api_socket_name = format (0, "%s%c", env_var_str, 0); + VCFG_DBG (0, "VCL<%d>: configured api-socket-name (%s)", getpid (), + vcl_cfg->vpp_api_socket_name); } } diff --git a/src/vcl/vcl_private.c b/src/vcl/vcl_private.c new file mode 100644 index 00000000000..48abbafe646 --- /dev/null +++ b/src/vcl/vcl_private.c @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2018 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +vcl_cut_through_registration_t * +vcl_ct_registration_lock_and_alloc (void) +{ + vcl_cut_through_registration_t *cr; + pool_get (vcm->cut_through_registrations, cr); + clib_spinlock_lock (&vcm->ct_registration_lock); + memset (cr, 0, sizeof (*cr)); + cr->epoll_evt_conn_index = -1; + return cr; +} + +u32 +vcl_ct_registration_index (vcl_cut_through_registration_t * ctr) +{ + return (ctr - vcm->cut_through_registrations); +} + +void +vcl_ct_registration_unlock (void) +{ + clib_spinlock_unlock (&vcm->ct_registration_lock); +} + +vcl_cut_through_registration_t * +vcl_ct_registration_get (u32 ctr_index) +{ + if (pool_is_free_index (vcm->cut_through_registrations, ctr_index)) + return 0; + return pool_elt_at_index (vcm->cut_through_registrations, ctr_index); +} + +vcl_cut_through_registration_t * +vcl_ct_registration_lock_and_lookup (uword mq_addr) +{ + uword *p; + clib_spinlock_lock (&vcm->ct_registration_lock); + p = hash_get (vcm->ct_registration_by_mq, mq_addr); + if (!p) + return 0; + return vcl_ct_registration_get (p[0]); +} + +void +vcl_ct_registration_lookup_add (uword mq_addr, u32 ctr_index) +{ + hash_set (vcm->ct_registration_by_mq, mq_addr, ctr_index); +} + +void +vcl_ct_registration_lookup_del (uword mq_addr) +{ + hash_unset (vcm->ct_registration_by_mq, mq_addr); +} + +void +vcl_ct_registration_del (vcl_cut_through_registration_t * ctr) +{ + pool_put (vcm->cut_through_registrations, ctr); +} + +vcl_mq_evt_conn_t * +vcl_mq_evt_conn_alloc (void) +{ + vcl_mq_evt_conn_t *mqc; + pool_get (vcm->mq_evt_conns, mqc); + memset (mqc, 0, sizeof (*mqc)); + return mqc; +} + +u32 +vcl_mq_evt_conn_index (vcl_mq_evt_conn_t * mqc) +{ + return (mqc - vcm->mq_evt_conns); +} + +vcl_mq_evt_conn_t * +vcl_mq_evt_conn_get (u32 mq_conn_idx) +{ + return pool_elt_at_index (vcm->mq_evt_conns, mq_conn_idx); +} + +int +vcl_mq_epoll_add_evfd (svm_msg_q_t * mq) +{ + struct epoll_event e = { 0 }; + vcl_mq_evt_conn_t *mqc; + u32 mqc_index; + int mq_fd; + + mq_fd = svm_msg_q_get_consumer_eventfd (mq); + + if (vcm->mqs_epfd < 0 || mq_fd == -1) + return -1; + + mqc = vcl_mq_evt_conn_alloc (); + mqc_index = vcl_mq_evt_conn_index (mqc); + mqc->mq_fd = mq_fd; + mqc->mq = mq; + + e.events = EPOLLIN; + e.data.u32 = mqc_index; + if (epoll_ctl (vcm->mqs_epfd, EPOLL_CTL_ADD, mq_fd, &e) < 0) + { + clib_warning ("failed to add mq eventfd to mq epoll fd"); + return -1; + } + + return mqc_index; +} + +int +vcl_mq_epoll_del_evfd (u32 mqc_index) +{ + vcl_mq_evt_conn_t *mqc; + + if (vcm->mqs_epfd || mqc_index == ~0) + return -1; + + mqc = vcl_mq_evt_conn_get (mqc_index); + if (epoll_ctl (vcm->mqs_epfd, EPOLL_CTL_DEL, mqc->mq_fd, 0) < 0) + { + clib_warning ("failed to del mq eventfd to mq epoll fd"); + return -1; + } + return 0; +} + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/src/vcl/vcl_private.h b/src/vcl/vcl_private.h index af58cdc3ac9..0506355b83c 100644 --- a/src/vcl/vcl_private.h +++ b/src/vcl/vcl_private.h @@ -141,7 +141,6 @@ typedef struct vppcom_epoll_t vep; int libc_epfd; svm_msg_q_t *our_evt_q; - u32 ct_registration; u64 options[16]; vce_event_handler_reg_t *poll_reg; vcl_session_msg_t *accept_evts_fifo; @@ -168,12 +167,14 @@ typedef struct vppcom_cfg_t_ u8 app_scope_global; u8 *namespace_id; u64 namespace_secret; + u8 use_mq_eventfd; f64 app_timeout; f64 session_timeout; f64 accept_timeout; u32 event_ring_size; char *event_log_path; u8 *vpp_api_filename; + u8 *vpp_api_socket_name; } vppcom_cfg_t; void vppcom_cfg (vppcom_cfg_t * vcl_cfg); @@ -181,9 +182,18 @@ void vppcom_cfg (vppcom_cfg_t * vcl_cfg); typedef struct vcl_cut_through_registration_ { svm_msg_q_t *mq; + svm_msg_q_t *peer_mq; u32 sid; + u32 epoll_evt_conn_index; /*< mq evt connection index part of + the mqs evtfd epoll (if used) */ } vcl_cut_through_registration_t; +typedef struct vcl_mq_evt_conn_ +{ + svm_msg_q_t *mq; + int mq_fd; +} vcl_mq_evt_conn_t; + typedef struct vppcom_main_t_ { u8 init; @@ -203,6 +213,15 @@ typedef struct vppcom_main_t_ clib_spinlock_t sessions_lockp; vcl_session_t *sessions; + /** Message queues epoll fd. Initialized only if using mqs with eventfds */ + int mqs_epfd; + + /** Pool of event message queue event connections */ + vcl_mq_evt_conn_t *mq_evt_conns; + + /** Per worker buffer for receiving mq epoll events */ + struct epoll_event *mq_events; + /* Hash table for disconnect processing */ uword *session_index_by_vpp_handles; @@ -214,6 +233,8 @@ typedef struct vppcom_main_t_ /* Our event queue */ svm_msg_q_t *app_event_queue; + svm_msg_q_t **vpp_event_queues; + /* unique segment name counter */ u32 unique_segment_index; @@ -237,6 +258,14 @@ typedef struct vppcom_main_t_ /** Pool of cut through registrations */ vcl_cut_through_registration_t *cut_through_registrations; + /** Lock for accessing ct registration pool */ + clib_spinlock_t ct_registration_lock; + + /** Cut-through registration by mq address hash table */ + uword *ct_registration_by_mq; + + svm_msg_q_msg_t *mq_msg_vector; + /** Flag indicating that a new segment is being mounted */ volatile u32 mounting_segment; @@ -284,6 +313,21 @@ do { \ #define VCL_INVALID_SESSION_INDEX ((u32)~0) +static inline vcl_session_t * +vcl_session_alloc (void) +{ + vcl_session_t *s; + pool_get (vcm->sessions, s); + memset (s, 0, sizeof (*s)); + return s; +} + +static inline void +vcl_session_free (vcl_session_t * s) +{ + pool_put (vcm->sessions, s); +} + static inline vcl_session_t * vcl_session_get (u32 session_index) { @@ -374,6 +418,23 @@ vppcom_session_table_lookup_listener (u64 listener_handle) const char *vppcom_session_state_str (session_state_t state); +/* + * Helpers + */ +vcl_cut_through_registration_t *vcl_ct_registration_lock_and_alloc (void); +void vcl_ct_registration_del (vcl_cut_through_registration_t * ctr); +u32 vcl_ct_registration_index (vcl_cut_through_registration_t * ctr); +void vcl_ct_registration_unlock (void); +vcl_cut_through_registration_t *vcl_ct_registration_get (u32 ctr_index); +vcl_cut_through_registration_t *vcl_ct_registration_lock_and_lookup (uword); +void vcl_ct_registration_lookup_add (uword mq_addr, u32 ctr_index); +void vcl_ct_registration_lookup_del (uword mq_addr); +vcl_mq_evt_conn_t *vcl_mq_evt_conn_alloc (void); +u32 vcl_mq_evt_conn_index (vcl_mq_evt_conn_t * mqc); +vcl_mq_evt_conn_t *vcl_mq_evt_conn_get (u32 mq_conn_idx); +int vcl_mq_epoll_add_evfd (svm_msg_q_t * mq); +int vcl_mq_epoll_del_evfd (u32 mqc_index); + /* * VCL Binary API */ diff --git a/src/vcl/vcl_test_server.c b/src/vcl/vcl_test_server.c index b49383eb73a..7b56d497153 100644 --- a/src/vcl/vcl_test_server.c +++ b/src/vcl/vcl_test_server.c @@ -629,7 +629,6 @@ main (int argc, char **argv) printf ("SERVER (fd %d): TX (%d bytes) - '%s'\n", conn->fd, tx_bytes, conn->buf); } - else // Extraneous read data from non-echo tests??? { xtra++; diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c index a74e55af634..8572b69aa13 100644 --- a/src/vcl/vppcom.c +++ b/src/vcl/vppcom.c @@ -248,47 +248,6 @@ vppcom_wait_for_app_state_change (app_state_t app_state) return VPPCOM_ETIMEDOUT; } -static u32 -vcl_ct_registration_add (svm_msg_q_t * mq, u32 sid) -{ - vcl_cut_through_registration_t *cr; - pool_get (vcm->cut_through_registrations, cr); - cr->mq = mq; - cr->sid = sid; - return (cr - vcm->cut_through_registrations); -} - -static void -vcl_ct_registration_del (u32 ct_index) -{ - pool_put_index (vcm->cut_through_registrations, ct_index); -} - -static vcl_session_t * -vcl_ct_session_get_from_fifo (svm_fifo_t * f, u8 type) -{ - vcl_session_t *s; - s = vcl_session_get (f->client_session_index); - if (s) - { - /* rx fifo */ - if (type == 0 && s->rx_fifo == f) - return s; - /* tx fifo */ - if (type == 1 && s->tx_fifo == f) - return s; - } - s = vcl_session_get (f->master_session_index); - if (s) - { - if (type == 0 && s->rx_fifo == f) - return s; - if (type == 1 && s->tx_fifo == f) - return s; - } - return 0; -} - static void vcl_send_session_accepted_reply (svm_msg_q_t * mq, u32 context, session_handle_t handle, int retval) @@ -303,15 +262,34 @@ vcl_send_session_accepted_reply (svm_msg_q_t * mq, u32 context, app_send_ctrl_evt_to_vpp (mq, app_evt); } +static void +vcl_send_session_disconnected_reply (svm_msg_q_t * mq, u32 context, + session_handle_t handle, int retval) +{ + app_session_evt_t _app_evt, *app_evt = &_app_evt; + session_disconnected_reply_msg_t *rmp; + app_alloc_ctrl_evt_to_vpp (mq, app_evt, + SESSION_CTRL_EVT_DISCONNECTED_REPLY); + rmp = (session_disconnected_reply_msg_t *) app_evt->evt->data; + rmp->handle = handle; + rmp->context = context; + rmp->retval = retval; + app_send_ctrl_evt_to_vpp (mq, app_evt); +} + static u32 vcl_session_accepted_handler (session_accepted_msg_t * mp) { vcl_session_t *session, *listen_session; svm_fifo_t *rx_fifo, *tx_fifo; - u32 session_index; + u32 session_index, vpp_wrk_index; + svm_msg_q_t *evt_q; VCL_SESSION_LOCK (); + session = vcl_session_alloc (); + session_index = vcl_session_index (session); + listen_session = vppcom_session_table_lookup_listener (mp->listener_handle); if (!listen_session) { @@ -322,13 +300,11 @@ vcl_session_accepted_handler (session_accepted_msg_t * mp) getpid (), mp->listener_handle); vcl_send_session_accepted_reply (evt_q, mp->context, mp->handle, VNET_API_ERROR_INVALID_ARGUMENT); + vcl_session_free (session); + VCL_SESSION_UNLOCK (); return VCL_INVALID_SESSION_INDEX; } - pool_get (vcm->sessions, session); - memset (session, 0, sizeof (*session)); - session_index = (u32) (session - vcm->sessions); - rx_fifo = uword_to_pointer (mp->server_rx_fifo, svm_fifo_t *); tx_fifo = uword_to_pointer (mp->server_tx_fifo, svm_fifo_t *); @@ -339,10 +315,11 @@ vcl_session_accepted_handler (session_accepted_msg_t * mp) session->our_evt_q = uword_to_pointer (mp->server_event_queue_address, svm_msg_q_t *); vcl_wait_for_memory (session->vpp_evt_q); - session->ct_registration = vcl_ct_registration_add (session->our_evt_q, - session_index); rx_fifo->master_session_index = session_index; tx_fifo->master_session_index = session_index; + vec_validate (vcm->vpp_event_queues, 0); + evt_q = uword_to_pointer (mp->vpp_event_queue_address, svm_msg_q_t *); + vcm->vpp_event_queues[0] = evt_q; } else { @@ -350,6 +327,10 @@ vcl_session_accepted_handler (session_accepted_msg_t * mp) svm_msg_q_t *); rx_fifo->client_session_index = session_index; tx_fifo->client_session_index = session_index; + + vpp_wrk_index = tx_fifo->master_thread_index; + vec_validate (vcm->vpp_event_queues, vpp_wrk_index); + vcm->vpp_event_queues[vpp_wrk_index] = session->vpp_evt_q; } session->vpp_handle = mp->handle; @@ -382,9 +363,10 @@ vcl_session_accepted_handler (session_accepted_msg_t * mp) static u32 vcl_session_connected_handler (session_connected_msg_t * mp) { - vcl_session_t *session = 0; - u32 session_index; + u32 session_index, vpp_wrk_index; svm_fifo_t *rx_fifo, *tx_fifo; + vcl_session_t *session = 0; + svm_msg_q_t *evt_q; int rv = VPPCOM_OK; session_index = mp->context; @@ -425,12 +407,19 @@ done: svm_msg_q_t *); session->our_evt_q = uword_to_pointer (mp->client_event_queue_address, svm_msg_q_t *); - session->ct_registration = vcl_ct_registration_add (session->our_evt_q, - session_index); + + vec_validate (vcm->vpp_event_queues, 0); + evt_q = uword_to_pointer (mp->vpp_event_queue_address, svm_msg_q_t *); + vcm->vpp_event_queues[0] = evt_q; } else - session->vpp_evt_q = uword_to_pointer (mp->vpp_event_queue_address, - svm_msg_q_t *); + { + session->vpp_evt_q = uword_to_pointer (mp->vpp_event_queue_address, + svm_msg_q_t *); + vpp_wrk_index = tx_fifo->master_thread_index; + vec_validate (vcm->vpp_event_queues, vpp_wrk_index); + vcm->vpp_event_queues[vpp_wrk_index] = session->vpp_evt_q; + } session->rx_fifo = rx_fifo; session->tx_fifo = tx_fifo; @@ -609,13 +598,23 @@ done: return rv; } +static svm_msg_q_t * +vcl_session_vpp_evt_q (vcl_session_t * s) +{ + if (vcl_session_is_ct (s)) + return vcm->vpp_event_queues[0]; + else + return vcm->vpp_event_queues[s->tx_fifo->master_thread_index]; +} + static int vppcom_session_disconnect (u32 session_index) { - int rv; + svm_msg_q_t *vpp_evt_q; vcl_session_t *session; - u64 vpp_handle; session_state_t state; + u64 vpp_handle; + int rv; VCL_SESSION_LOCK_AND_GET (session_index, &session); @@ -637,8 +636,9 @@ vppcom_session_disconnect (u32 session_index) if (state & STATE_CLOSE_ON_EMPTY) { - vppcom_send_disconnect_session_reply (vpp_handle, session_index, - 0 /* rv */ ); + vpp_evt_q = vcl_session_vpp_evt_q (session); + vcl_send_session_disconnected_reply (vpp_evt_q, vcm->my_client_index, + vpp_handle, 0); VDBG (1, "VCL<%d>: vpp handle 0x%llx, sid %u: sending disconnect " "REPLY...", getpid (), vpp_handle, session_index); } @@ -666,6 +666,11 @@ vppcom_app_create (char *app_name) { vcm->init = 1; vppcom_cfg (&vcm->cfg); + vcl_cfg = &vcm->cfg; + + vcm->mqs_epfd = -1; + if (vcl_cfg->use_mq_eventfd) + vcm->mqs_epfd = epoll_create (1); clib_spinlock_init (&vcm->session_fifo_lockp); clib_fifo_validate (vcm->client_session_index_fifo, @@ -676,11 +681,16 @@ vppcom_app_create (char *app_name) vcm->main_cpu = os_get_thread_index (); vcm->session_index_by_vpp_handles = hash_create (0, sizeof (uword)); + vcm->ct_registration_by_mq = hash_create (0, sizeof (uword)); + clib_spinlock_init (&vcm->ct_registration_lock); clib_time_init (&vcm->clib_time); vppcom_init_error_string_table (); svm_fifo_segment_main_init (vcl_cfg->segment_baseva, 20 /* timeout in secs */ ); + vec_validate (vcm->mq_events, 64); + vec_validate (vcm->mq_msg_vector, 128); + vec_reset_length (vcm->mq_msg_vector); } if (vcm->my_client_index == ~0) @@ -868,8 +878,21 @@ vppcom_session_close (uint32_t session_index) } VCL_SESSION_LOCK_AND_GET (session_index, &session); - if (session->our_evt_q) - vcl_ct_registration_del (session->ct_registration); + if (vcl_session_is_ct (session)) + { + vcl_cut_through_registration_t *ctr; + uword mq_addr; + + mq_addr = pointer_to_uword (session->our_evt_q); + ctr = vcl_ct_registration_lock_and_lookup (mq_addr); + ASSERT (ctr); + if (ctr->epoll_evt_conn_index != ~0) + vcl_mq_epoll_del_evfd (ctr->epoll_evt_conn_index); + VDBG (0, "Removing ct registration %u", + vcl_ct_registration_index (ctr)); + vcl_ct_registration_del (ctr); + vcl_ct_registration_unlock (); + } vpp_handle = session->vpp_handle; if (vpp_handle != ~0) @@ -1084,6 +1107,7 @@ vppcom_session_accept (uint32_t listen_session_index, vppcom_endpt_t * ep, handle: client_session_index = vcl_session_accepted_handler (&accepted_msg); + listen_session = vcl_session_get (listen_session_index); VCL_SESSION_LOCK_AND_GET (client_session_index, &client_session); rv = client_session_index; @@ -1115,6 +1139,7 @@ handle: svm_msg_q_t *); else vpp_evt_q = client_session->vpp_evt_q; + vcl_send_session_accepted_reply (vpp_evt_q, client_session->client_context, client_session->vpp_handle, 0); @@ -1287,23 +1312,25 @@ vppcom_session_read_internal (uint32_t session_index, void *buf, int n, VCL_SESSION_UNLOCK (); mq = vcl_session_is_ct (s) ? s->our_evt_q : vcm->app_event_queue; + svm_fifo_unset_event (rx_fifo); is_full = svm_fifo_is_full (rx_fifo); if (svm_fifo_is_empty (rx_fifo)) { - svm_fifo_unset_event (rx_fifo); if (is_nonblocking) { rv = VPPCOM_OK; goto done; } - svm_msg_q_lock (mq); while (1) { + svm_msg_q_lock (mq); if (svm_msg_q_is_empty (mq)) svm_msg_q_wait (mq); + svm_msg_q_sub_w_lock (mq, &msg); e = svm_msg_q_msg_data (mq, &msg); + svm_msg_q_unlock (mq); if (!vcl_is_rx_evt_for_session (e, session_index, s->our_evt_q != 0)) { @@ -1311,25 +1338,29 @@ vppcom_session_read_internal (uint32_t session_index, void *buf, int n, svm_msg_q_free_msg (mq, &msg); continue; } + svm_fifo_unset_event (rx_fifo); if (svm_fifo_is_empty (rx_fifo)) { svm_msg_q_free_msg (mq, &msg); continue; } svm_msg_q_free_msg (mq, &msg); - svm_msg_q_unlock (mq); break; } } if (s->is_dgram) - n_read = app_recv_dgram_raw (rx_fifo, buf, n, &s->transport, 1, peek); + n_read = app_recv_dgram_raw (rx_fifo, buf, n, &s->transport, 0, peek); else - n_read = app_recv_stream_raw (rx_fifo, buf, n, 1, peek); + n_read = app_recv_stream_raw (rx_fifo, buf, n, 0, peek); if (vcl_session_is_ct (s) && is_full) - app_send_io_evt_to_vpp (s->vpp_evt_q, rx_fifo, SESSION_IO_EVT_CT_RX, - SVM_Q_WAIT); + { + /* If the peer is not polling send notification */ + if (!svm_fifo_has_event (s->rx_fifo)) + app_send_io_evt_to_vpp (s->vpp_evt_q, s->rx_fifo, + SESSION_IO_EVT_CT_RX, SVM_Q_WAIT); + } if (VPPCOM_DEBUG > 2) { @@ -1452,18 +1483,20 @@ vppcom_session_write (uint32_t session_index, void *buf, size_t n) rv = VPPCOM_EWOULDBLOCK; goto done; } - svm_msg_q_lock (mq); while (1) { + svm_msg_q_lock (mq); if (!svm_fifo_is_full (tx_fifo)) { svm_msg_q_unlock (mq); break; } - if (svm_msg_q_is_empty (mq) && svm_msg_q_timedwait (mq, 10e-6)) - continue; + while (svm_msg_q_is_empty (mq) && svm_msg_q_timedwait (mq, 10e-6)) + ; svm_msg_q_sub_w_lock (mq, &msg); e = svm_msg_q_msg_data (mq, &msg); + svm_msg_q_unlock (mq); + if (!vcl_is_tx_evt_for_session (e, session_index, s->our_evt_q != 0)) { @@ -1477,7 +1510,6 @@ vppcom_session_write (uint32_t session_index, void *buf, size_t n) continue; } svm_msg_q_free_msg (mq, &msg); - svm_msg_q_unlock (mq); break; } } @@ -1510,6 +1542,31 @@ done: return rv; } +static vcl_session_t * +vcl_ct_session_get_from_fifo (svm_fifo_t * f, u8 type) +{ + vcl_session_t *s; + s = vcl_session_get (f->client_session_index); + if (s) + { + /* rx fifo */ + if (type == 0 && s->rx_fifo == f) + return s; + /* tx fifo */ + if (type == 1 && s->tx_fifo == f) + return s; + } + s = vcl_session_get (f->master_session_index); + if (s) + { + if (type == 0 && s->rx_fifo == f) + return s; + if (type == 1 && s->tx_fifo == f) + return s; + } + return 0; +} + static inline int vppcom_session_write_ready (vcl_session_t * session, u32 session_index) { @@ -1552,6 +1609,22 @@ vppcom_session_write_ready (vcl_session_t * session, u32 session_index) return svm_fifo_max_enqueue (session->tx_fifo); } +static inline int +vcl_mq_dequeue_batch (svm_msg_q_t * mq) +{ + svm_msg_q_msg_t *msg; + u32 n_msgs; + int i; + + n_msgs = svm_msg_q_size (mq); + for (i = 0; i < n_msgs; i++) + { + vec_add2 (vcm->mq_msg_vector, msg, 1); + svm_msg_q_sub_w_lock (mq, msg); + } + return n_msgs; +} + static int vcl_select_handle_mq (svm_msg_q_t * mq, unsigned long n_bits, unsigned long *read_map, unsigned long *write_map, @@ -1559,12 +1632,13 @@ vcl_select_handle_mq (svm_msg_q_t * mq, unsigned long n_bits, u32 * bits_set) { session_disconnected_msg_t *disconnected_msg; + session_connected_msg_t *connected_msg; session_accepted_msg_t *accepted_msg; vcl_session_msg_t *vcl_msg; vcl_session_t *session; - svm_msg_q_msg_t msg; + svm_msg_q_msg_t *msg; session_event_t *e; - u32 n_msgs, i, sid; + u32 i, sid; u64 handle; svm_msg_q_lock (mq); @@ -1594,23 +1668,20 @@ vcl_select_handle_mq (svm_msg_q_t * mq, unsigned long n_bits, } } } + vcl_mq_dequeue_batch (mq); svm_msg_q_unlock (mq); - n_msgs = svm_msg_q_size (mq); - for (i = 0; i < n_msgs; i++) + for (i = 0; i < vec_len (vcm->mq_msg_vector); i++) { - if (svm_msg_q_sub (mq, &msg, SVM_Q_WAIT, 0)) - { - clib_warning ("message queue returned"); - continue; - } - e = svm_msg_q_msg_data (mq, &msg); + msg = vec_elt_at_index (vcm->mq_msg_vector, i); + e = svm_msg_q_msg_data (mq, msg); switch (e->event_type) { case FIFO_EVENT_APP_RX: sid = e->fifo->client_session_index; session = vcl_session_get (sid); - if (!session || svm_fifo_is_empty (session->rx_fifo)) + svm_fifo_unset_event (session->rx_fifo); + if (svm_fifo_is_empty (session->rx_fifo)) break; if (sid < n_bits && read_map) { @@ -1632,7 +1703,8 @@ vcl_select_handle_mq (svm_msg_q_t * mq, unsigned long n_bits, case SESSION_IO_EVT_CT_TX: session = vcl_ct_session_get_from_fifo (e->fifo, 0); sid = vcl_session_index (session); - if (!session || svm_fifo_is_empty (session->rx_fifo)) + svm_fifo_unset_event (session->rx_fifo); + if (svm_fifo_is_empty (session->rx_fifo)) break; if (sid < n_bits && read_map) { @@ -1672,6 +1744,10 @@ vcl_select_handle_mq (svm_msg_q_t * mq, unsigned long n_bits, *bits_set += 1; } break; + case SESSION_CTRL_EVT_CONNECTED: + connected_msg = (session_connected_msg_t *) e->data; + vcl_session_connected_handler (connected_msg); + break; case SESSION_CTRL_EVT_DISCONNECTED: disconnected_msg = (session_disconnected_msg_t *) e->data; sid = vcl_session_get_index_from_handle (disconnected_msg->handle); @@ -1685,20 +1761,73 @@ vcl_select_handle_mq (svm_msg_q_t * mq, unsigned long n_bits, clib_warning ("unhandled: %u", e->event_type); break; } - svm_msg_q_free_msg (mq, &msg); + svm_msg_q_free_msg (mq, msg); } + vec_reset_length (vcm->mq_msg_vector); return *bits_set; } +static int +vppcom_select_condvar (unsigned long n_bits, unsigned long *read_map, + unsigned long *write_map, unsigned long *except_map, + double time_to_wait, u32 * bits_set) +{ + double total_wait = 0, wait_slice; + vcl_cut_through_registration_t *cr; + + time_to_wait = (time_to_wait == -1) ? 10e9 : time_to_wait; + wait_slice = vcm->cut_through_registrations ? 10e-6 : time_to_wait; + do + { + /* *INDENT-OFF* */ + pool_foreach (cr, vcm->cut_through_registrations, ({ + vcl_select_handle_mq (cr->mq, n_bits, read_map, write_map, except_map, + 0, bits_set); + })); + /* *INDENT-ON* */ + + vcl_select_handle_mq (vcm->app_event_queue, n_bits, read_map, write_map, + except_map, time_to_wait, bits_set); + total_wait += wait_slice; + if (*bits_set) + return *bits_set; + } + while (total_wait < time_to_wait); + + return 0; +} + +static int +vppcom_select_eventfd (unsigned long n_bits, unsigned long *read_map, + unsigned long *write_map, unsigned long *except_map, + double time_to_wait, u32 * bits_set) +{ + vcl_mq_evt_conn_t *mqc; + int __clib_unused n_read; + int n_mq_evts, i; + u64 buf; + + vec_validate (vcm->mq_events, pool_elts (vcm->mq_evt_conns)); + n_mq_evts = epoll_wait (vcm->mqs_epfd, vcm->mq_events, + vec_len (vcm->mq_events), time_to_wait); + for (i = 0; i < n_mq_evts; i++) + { + mqc = vcl_mq_evt_conn_get (vcm->mq_events[i].data.u32); + n_read = read (mqc->mq_fd, &buf, sizeof (buf)); + vcl_select_handle_mq (mqc->mq, n_bits, read_map, write_map, + except_map, 0, bits_set); + } + + return (n_mq_evts > 0 ? (int) *bits_set : 0); +} + int vppcom_select (unsigned long n_bits, unsigned long *read_map, unsigned long *write_map, unsigned long *except_map, double time_to_wait) { u32 sid, minbits = clib_max (n_bits, BITS (uword)), bits_set = 0; - vcl_cut_through_registration_t *cr; - double total_wait = 0, wait_slice; vcl_session_t *session = 0; int rv; @@ -1736,17 +1865,14 @@ vppcom_select (unsigned long n_bits, unsigned long *read_map, /* *INDENT-OFF* */ clib_bitmap_foreach (sid, vcm->wr_bitmap, ({ - VCL_SESSION_LOCK(); if (!(session = vcl_session_get (sid))) { - VCL_SESSION_UNLOCK(); VDBG (0, "VCL<%d>: session %d specified in write_map is closed.", getpid (), sid); return VPPCOM_EBADFD; } rv = svm_fifo_is_full (session->tx_fifo); - VCL_SESSION_UNLOCK(); if (!rv) { clib_bitmap_set_no_check (write_map, sid, 1); @@ -1757,18 +1883,16 @@ vppcom_select (unsigned long n_bits, unsigned long *read_map, check_rd: if (!read_map) goto check_mq; + clib_bitmap_foreach (sid, vcm->rd_bitmap, ({ - VCL_SESSION_LOCK(); if (!(session = vcl_session_get (sid))) { - VCL_SESSION_UNLOCK(); VDBG (0, "VCL<%d>: session %d specified in write_map is closed.", getpid (), sid); return VPPCOM_EBADFD; } rv = vppcom_session_read_ready (session); - VCL_SESSION_UNLOCK(); if (rv) { clib_bitmap_set_no_check (read_map, sid, 1); @@ -1778,23 +1902,13 @@ check_rd: /* *INDENT-ON* */ check_mq: - wait_slice = vcm->cut_through_registrations ? 10e-6 : time_to_wait; - do - { - /* *INDENT-OFF* */ - pool_foreach (cr, vcm->cut_through_registrations, ({ - vcl_select_handle_mq (cr->mq, n_bits, read_map, write_map, except_map, - 0, &bits_set); - })); - /* *INDENT-ON* */ - vcl_select_handle_mq (vcm->app_event_queue, n_bits, read_map, write_map, - except_map, time_to_wait, &bits_set); - total_wait += wait_slice; - if (bits_set) - return bits_set; - } - while (total_wait < time_to_wait); + if (vcm->cfg.use_mq_eventfd) + vppcom_select_eventfd (n_bits, read_map, write_map, except_map, + time_to_wait, &bits_set); + else + vppcom_select_condvar (n_bits, read_map, write_map, except_map, + time_to_wait, &bits_set); return (bits_set); } @@ -2130,11 +2244,11 @@ vcl_epoll_wait_handle_mq (svm_msg_q_t * mq, struct epoll_event *events, session_disconnected_msg_t *disconnected_msg; session_connected_msg_t *connected_msg; session_accepted_msg_t *accepted_msg; - u32 sid = ~0, session_events, n_msgs; u64 session_evt_data = ~0, handle; + u32 sid = ~0, session_events; vcl_session_msg_t *vcl_msg; vcl_session_t *session; - svm_msg_q_msg_t msg; + svm_msg_q_msg_t *msg; session_event_t *e; u8 add_event; int i; @@ -2153,61 +2267,58 @@ vcl_epoll_wait_handle_mq (svm_msg_q_t * mq, struct epoll_event *events, } else { - if (svm_msg_q_timedwait (mq, wait_for_time / 1e3)) + if (svm_msg_q_timedwait (mq, wait_for_time / 1e3) < 0) { svm_msg_q_unlock (mq); return 0; } } } + vcl_mq_dequeue_batch (mq); svm_msg_q_unlock (mq); - n_msgs = svm_msg_q_size (mq); - for (i = 0; i < n_msgs; i++) + for (i = 0; i < vec_len (vcm->mq_msg_vector); i++) { - if (svm_msg_q_sub (mq, &msg, SVM_Q_WAIT, 0)) - { - clib_warning ("message queue returned"); - continue; - } - e = svm_msg_q_msg_data (mq, &msg); + msg = vec_elt_at_index (vcm->mq_msg_vector, i); + e = svm_msg_q_msg_data (mq, msg); add_event = 0; switch (e->event_type) { case FIFO_EVENT_APP_RX: sid = e->fifo->client_session_index; - clib_spinlock_lock (&vcm->sessions_lockp); session = vcl_session_get (sid); session_events = session->vep.ev.events; - if ((EPOLLIN & session->vep.ev.events) - && !svm_fifo_is_empty (session->rx_fifo)) + if (!(EPOLLIN & session->vep.ev.events)) + break; + svm_fifo_unset_event (session->rx_fifo); + if (!svm_fifo_is_empty (session->rx_fifo)) { add_event = 1; events[*num_ev].events |= EPOLLIN; session_evt_data = session->vep.ev.data.u64; } - clib_spinlock_unlock (&vcm->sessions_lockp); break; case FIFO_EVENT_APP_TX: sid = e->fifo->client_session_index; - clib_spinlock_lock (&vcm->sessions_lockp); session = vcl_session_get (sid); session_events = session->vep.ev.events; - if ((EPOLLOUT & session_events) - && !svm_fifo_is_full (session->tx_fifo)) + if (!(EPOLLOUT & session_events)) + break; + if (!svm_fifo_is_full (session->tx_fifo)) { add_event = 1; events[*num_ev].events |= EPOLLOUT; session_evt_data = session->vep.ev.data.u64; } - clib_spinlock_unlock (&vcm->sessions_lockp); break; case SESSION_IO_EVT_CT_TX: session = vcl_ct_session_get_from_fifo (e->fifo, 0); sid = vcl_session_index (session); session_events = session->vep.ev.events; - if ((EPOLLIN & session->vep.ev.events) - && !svm_fifo_is_empty (session->rx_fifo)) + if (!(EPOLLIN & session->vep.ev.events)) + break; + svm_fifo_unset_event (session->rx_fifo); + if (!svm_fifo_is_empty (session->rx_fifo)) { add_event = 1; events[*num_ev].events |= EPOLLIN; @@ -2218,8 +2329,9 @@ vcl_epoll_wait_handle_mq (svm_msg_q_t * mq, struct epoll_event *events, session = vcl_ct_session_get_from_fifo (e->fifo, 1); sid = vcl_session_index (session); session_events = session->vep.ev.events; - if ((EPOLLOUT & session_events) - && !svm_fifo_is_full (session->tx_fifo)) + if (!(EPOLLOUT & session_events)) + break; + if (!svm_fifo_is_full (session->tx_fifo)) { add_event = 1; events[*num_ev].events |= EPOLLOUT; @@ -2276,11 +2388,10 @@ vcl_epoll_wait_handle_mq (svm_msg_q_t * mq, struct epoll_event *events, break; default: clib_warning ("unhandled: %u", e->event_type); - svm_msg_q_free_msg (mq, &msg); + svm_msg_q_free_msg (mq, msg); continue; } - - svm_msg_q_free_msg (mq, &msg); + svm_msg_q_free_msg (mq, msg); if (add_event) { @@ -2297,17 +2408,70 @@ vcl_epoll_wait_handle_mq (svm_msg_q_t * mq, struct epoll_event *events, break; } } + + vec_reset_length (vcm->mq_msg_vector); return *num_ev; } +static int +vppcom_epoll_wait_condvar (struct epoll_event *events, int maxevents, + double wait_for_time) +{ + vcl_cut_through_registration_t *cr; + double total_wait = 0, wait_slice; + u32 num_ev = 0; + int rv; + + wait_for_time = (wait_for_time == -1) ? (double) 10e9 : wait_for_time; + wait_slice = vcm->cut_through_registrations ? 10e-6 : wait_for_time; + + do + { + /* *INDENT-OFF* */ + pool_foreach (cr, vcm->cut_through_registrations, ({ + vcl_epoll_wait_handle_mq (cr->mq, events, maxevents, 0, &num_ev); + })); + /* *INDENT-ON* */ + + rv = vcl_epoll_wait_handle_mq (vcm->app_event_queue, events, maxevents, + num_ev ? 0 : wait_slice, &num_ev); + if (rv) + total_wait += wait_slice; + if (num_ev) + return num_ev; + } + while (total_wait < wait_for_time); + return (int) num_ev; +} + +static int +vppcom_epoll_wait_eventfd (struct epoll_event *events, int maxevents, + double wait_for_time) +{ + vcl_mq_evt_conn_t *mqc; + int __clib_unused n_read; + int n_mq_evts, i; + u32 n_evts = 0; + u64 buf; + + vec_validate (vcm->mq_events, pool_elts (vcm->mq_evt_conns)); + n_mq_evts = epoll_wait (vcm->mqs_epfd, vcm->mq_events, + vec_len (vcm->mq_events), wait_for_time); + for (i = 0; i < n_mq_evts; i++) + { + mqc = vcl_mq_evt_conn_get (vcm->mq_events[i].data.u32); + n_read = read (mqc->mq_fd, &buf, sizeof (buf)); + vcl_epoll_wait_handle_mq (mqc->mq, events, maxevents, 0, &n_evts); + } + + return (int) n_evts; +} + int vppcom_epoll_wait (uint32_t vep_idx, struct epoll_event *events, int maxevents, double wait_for_time) { - vcl_cut_through_registration_t *cr; vcl_session_t *vep_session; - double total_wait = 0, wait_slice; - u32 num_ev = 0; if (PREDICT_FALSE (maxevents <= 0)) { @@ -2328,25 +2492,11 @@ vppcom_epoll_wait (uint32_t vep_idx, struct epoll_event *events, clib_spinlock_unlock (&vcm->sessions_lockp); memset (events, 0, sizeof (*events) * maxevents); - wait_slice = vcm->cut_through_registrations ? 10e-6 : wait_for_time; - - do - { - /* *INDENT-OFF* */ - pool_foreach (cr, vcm->cut_through_registrations, ({ - vcl_epoll_wait_handle_mq (cr->mq, events, maxevents, 0, &num_ev); - })); - /* *INDENT-ON* */ - vcl_epoll_wait_handle_mq (vcm->app_event_queue, events, maxevents, - num_ev ? 0 : wait_slice, &num_ev); - total_wait += wait_slice; - if (num_ev) - return num_ev; - } - while (total_wait < wait_for_time); + if (vcm->cfg.use_mq_eventfd) + return vppcom_epoll_wait_eventfd (events, maxevents, wait_for_time); - return num_ev; + return vppcom_epoll_wait_condvar (events, maxevents, wait_for_time); } int @@ -2917,12 +3067,15 @@ vppcom_session_recvfrom (uint32_t session_index, void *buffer, return VPPCOM_EAFNOSUPPORT; } - if (session->transport.is_ip4) - clib_memcpy (ep->ip, &session->transport.rmt_ip.ip4, - sizeof (ip4_address_t)); - else - clib_memcpy (ep->ip, &session->transport.rmt_ip.ip6, - sizeof (ip6_address_t)); + if (ep) + { + if (session->transport.is_ip4) + clib_memcpy (ep->ip, &session->transport.rmt_ip.ip4, + sizeof (ip4_address_t)); + else + clib_memcpy (ep->ip, &session->transport.rmt_ip.ip6, + sizeof (ip6_address_t)); + } return rv; } @@ -3054,6 +3207,12 @@ vppcom_poll (vcl_poll_t * vp, uint32_t n_sids, double wait_for_time) return num_ev; } +int +vppcom_mq_epoll_fd (void) +{ + return vcm->mqs_epfd; +} + /* * fd.io coding-style-patch-verification: ON * diff --git a/src/vcl/vppcom.h b/src/vcl/vppcom.h index 8d414f6cc77..eff79c3a5ac 100644 --- a/src/vcl/vppcom.h +++ b/src/vcl/vppcom.h @@ -31,17 +31,18 @@ extern "C" /* * VPPCOM Public API Definitions, Enums, and Data Structures */ -#define INVALID_SESSION_ID (~0) -#define VPPCOM_CONF_DEFAULT "/etc/vpp/vcl.conf" -#define VPPCOM_ENV_CONF "VCL_CONFIG" -#define VPPCOM_ENV_DEBUG "VCL_DEBUG" -#define VPPCOM_ENV_API_PREFIX "VCL_API_PREFIX" -#define VPPCOM_ENV_APP_PROXY_TRANSPORT_TCP "VCL_APP_PROXY_TRANSPORT_TCP" -#define VPPCOM_ENV_APP_PROXY_TRANSPORT_UDP "VCL_APP_PROXY_TRANSPORT_UDP" -#define VPPCOM_ENV_APP_NAMESPACE_ID "VCL_APP_NAMESPACE_ID" -#define VPPCOM_ENV_APP_NAMESPACE_SECRET "VCL_APP_NAMESPACE_SECRET" -#define VPPCOM_ENV_APP_SCOPE_LOCAL "VCL_APP_SCOPE_LOCAL" -#define VPPCOM_ENV_APP_SCOPE_GLOBAL "VCL_APP_SCOPE_GLOBAL" +#define INVALID_SESSION_ID (~0) +#define VPPCOM_CONF_DEFAULT "/etc/vpp/vcl.conf" +#define VPPCOM_ENV_CONF "VCL_CONFIG" +#define VPPCOM_ENV_DEBUG "VCL_DEBUG" +#define VPPCOM_ENV_API_PREFIX "VCL_API_PREFIX" +#define VPPCOM_ENV_APP_PROXY_TRANSPORT_TCP "VCL_APP_PROXY_TRANSPORT_TCP" +#define VPPCOM_ENV_APP_PROXY_TRANSPORT_UDP "VCL_APP_PROXY_TRANSPORT_UDP" +#define VPPCOM_ENV_APP_NAMESPACE_ID "VCL_APP_NAMESPACE_ID" +#define VPPCOM_ENV_APP_NAMESPACE_SECRET "VCL_APP_NAMESPACE_SECRET" +#define VPPCOM_ENV_APP_SCOPE_LOCAL "VCL_APP_SCOPE_LOCAL" +#define VPPCOM_ENV_APP_SCOPE_GLOBAL "VCL_APP_SCOPE_GLOBAL" +#define VPPCOM_ENV_VPP_API_SOCKET "VCL_VPP_API_SOCKET" typedef enum { @@ -250,6 +251,7 @@ extern int vppcom_session_sendto (uint32_t session_index, void *buffer, vppcom_endpt_t * ep); extern int vppcom_poll (vcl_poll_t * vp, uint32_t n_sids, double wait_for_time); +extern int vppcom_mq_epoll_fd (void); /* * VPPCOM Event Functions -- cgit 1.2.3-korg