diff options
Diffstat (limited to 'src')
25 files changed, 590 insertions, 445 deletions
diff --git a/src/plugins/hs_apps/echo_client.c b/src/plugins/hs_apps/echo_client.c index 072d767fa6e..50d815748a5 100644 --- a/src/plugins/hs_apps/echo_client.c +++ b/src/plugins/hs_apps/echo_client.c @@ -65,9 +65,8 @@ send_data_chunk (echo_client_main_t * ecm, eclient_session_t * s) svm_fifo_t *f = s->data.tx_fifo; rv = clib_min (svm_fifo_max_enqueue_prod (f), bytes_this_chunk); svm_fifo_enqueue_nocopy (f, rv); - session_send_io_evt_to_thread_custom (&f->master_session_index, - s->thread_index, - SESSION_IO_EVT_TX); + session_send_io_evt_to_thread_custom ( + &f->shr->master_session_index, s->thread_index, SESSION_IO_EVT_TX); } else rv = app_send_stream (&s->data, test_data + test_buf_offset, @@ -101,9 +100,8 @@ send_data_chunk (echo_client_main_t * ecm, eclient_session_t * s) hdr.lcl_port = at->lcl_port; svm_fifo_enqueue (f, sizeof (hdr), (u8 *) & hdr); svm_fifo_enqueue_nocopy (f, rv); - session_send_io_evt_to_thread_custom (&f->master_session_index, - s->thread_index, - SESSION_IO_EVT_TX); + session_send_io_evt_to_thread_custom ( + &f->shr->master_session_index, s->thread_index, SESSION_IO_EVT_TX); } else { @@ -441,9 +439,9 @@ quic_echo_clients_session_connected_callback (u32 app_index, u32 api_context, session->bytes_to_send = ecm->bytes_to_send; session->bytes_to_receive = ecm->no_return ? 0ULL : ecm->bytes_to_send; session->data.rx_fifo = s->rx_fifo; - session->data.rx_fifo->client_session_index = session_index; + session->data.rx_fifo->shr->client_session_index = session_index; session->data.tx_fifo = s->tx_fifo; - session->data.tx_fifo->client_session_index = session_index; + session->data.tx_fifo->shr->client_session_index = session_index; session->data.vpp_evt_q = ecm->vpp_event_queue[thread_index]; session->vpp_session_handle = session_handle (s); @@ -508,9 +506,9 @@ echo_clients_session_connected_callback (u32 app_index, u32 api_context, session->bytes_to_send = ecm->bytes_to_send; session->bytes_to_receive = ecm->no_return ? 0ULL : ecm->bytes_to_send; session->data.rx_fifo = s->rx_fifo; - session->data.rx_fifo->client_session_index = session_index; + session->data.rx_fifo->shr->client_session_index = session_index; session->data.tx_fifo = s->tx_fifo; - session->data.tx_fifo->client_session_index = session_index; + session->data.tx_fifo->shr->client_session_index = session_index; session->data.vpp_evt_q = ecm->vpp_event_queue[thread_index]; session->vpp_session_handle = session_handle (s); @@ -589,7 +587,8 @@ echo_clients_rx_callback (session_t * s) return -1; } - sp = pool_elt_at_index (ecm->sessions, s->rx_fifo->client_session_index); + sp = + pool_elt_at_index (ecm->sessions, s->rx_fifo->shr->client_session_index); receive_data_chunk (ecm, sp); if (svm_fifo_max_dequeue_cons (s->rx_fifo)) diff --git a/src/plugins/hs_apps/proxy.c b/src/plugins/hs_apps/proxy.c index fc66286a803..834d03f8dbd 100644 --- a/src/plugins/hs_apps/proxy.c +++ b/src/plugins/hs_apps/proxy.c @@ -309,7 +309,7 @@ proxy_rx_callback (session_t * s) if (svm_fifo_set_event (ao_tx_fifo)) { u32 ao_thread_index = ao_tx_fifo->master_thread_index; - u32 ao_session_index = ao_tx_fifo->master_session_index; + u32 ao_session_index = ao_tx_fifo->shr->master_session_index; if (session_send_io_evt_to_thread_custom (&ao_session_index, ao_thread_index, SESSION_IO_EVT_TX)) @@ -475,7 +475,7 @@ active_open_connected_callback (u32 app_index, u32 opaque, * Reset the active-open tx-fifo master indices so the active-open session * will receive data, etc. */ - s->tx_fifo->master_session_index = s->session_index; + s->tx_fifo->shr->master_session_index = s->session_index; s->tx_fifo->master_thread_index = s->thread_index; /* @@ -532,7 +532,7 @@ active_open_rx_callback (session_t * s) if (svm_fifo_set_event (proxy_tx_fifo)) { u8 thread_index = proxy_tx_fifo->master_thread_index; - u32 session_index = proxy_tx_fifo->master_session_index; + u32 session_index = proxy_tx_fifo->shr->master_session_index; return session_send_io_evt_to_thread_custom (&session_index, thread_index, SESSION_IO_EVT_TX); diff --git a/src/plugins/hs_apps/sapi/vpp_echo.c b/src/plugins/hs_apps/sapi/vpp_echo.c index 4cce41edf95..a47a4d455d8 100644 --- a/src/plugins/hs_apps/sapi/vpp_echo.c +++ b/src/plugins/hs_apps/sapi/vpp_echo.c @@ -69,9 +69,9 @@ echo_session_dequeue_notify (echo_session_t * s) int rv; if (!svm_fifo_set_event (s->rx_fifo)) return; - if ((rv = - app_send_io_evt_to_vpp (s->vpp_evt_q, s->rx_fifo->master_session_index, - SESSION_IO_EVT_RX, SVM_Q_WAIT))) + if ((rv = app_send_io_evt_to_vpp (s->vpp_evt_q, + s->rx_fifo->shr->master_session_index, + SESSION_IO_EVT_RX, SVM_Q_WAIT))) ECHO_FAIL (ECHO_FAIL_SEND_IO_EVT, "app_send_io_evt_to_vpp errored %d", rv); svm_fifo_clear_deq_ntf (s->rx_fifo); @@ -542,7 +542,6 @@ session_accepted_handler (session_accepted_msg_t * mp) { app_session_evt_t _app_evt, *app_evt = &_app_evt; session_accepted_reply_msg_t *rmp; - svm_fifo_t *rx_fifo, *tx_fifo; echo_main_t *em = &echo_main; echo_session_t *session, *ls; @@ -552,24 +551,22 @@ session_accepted_handler (session_accepted_msg_t * mp) "Unknown listener handle 0x%lx", mp->listener_handle); return; } - if (echo_segment_lookup (mp->segment_handle) == ~0) + + /* Allocate local session and set it up */ + session = echo_session_new (em); + + if (echo_attach_session (mp->segment_handle, mp->server_rx_fifo, + mp->server_tx_fifo, session)) { ECHO_FAIL (ECHO_FAIL_ACCEPTED_WAIT_FOR_SEG_ALLOC, "accepted wait_for_segment_allocation errored"); return; } - /* Allocate local session and set it up */ - session = echo_session_new (em); - session->vpp_session_handle = mp->handle; - - rx_fifo = uword_to_pointer (mp->server_rx_fifo, svm_fifo_t *); - rx_fifo->client_session_index = session->session_index; - tx_fifo = uword_to_pointer (mp->server_tx_fifo, svm_fifo_t *); - tx_fifo->client_session_index = session->session_index; + session->vpp_evt_q = + uword_to_pointer (mp->vpp_event_queue_address, svm_msg_q_t *); - session->rx_fifo = rx_fifo; - session->tx_fifo = tx_fifo; + session->vpp_session_handle = mp->handle; /* session->transport needed by app_send_dgram */ clib_memcpy_fast (&session->transport.rmt_ip, &mp->rmt.ip, @@ -581,10 +578,8 @@ session_accepted_handler (session_accepted_msg_t * mp) session->transport.lcl_port = em->uri_elts.port; session->vpp_session_handle = mp->handle; - session->start = clib_time_now (&em->clib_time); - session->vpp_evt_q = uword_to_pointer (mp->vpp_event_queue_address, - svm_msg_q_t *); session->listener_index = ls->session_index; + session->start = clib_time_now (&em->clib_time); /* Add it to lookup table */ ECHO_LOG (2, "Accepted session 0x%lx S[%u] -> 0x%lx S[%u]", @@ -607,7 +602,6 @@ session_connected_handler (session_connected_msg_t * mp) echo_main_t *em = &echo_main; echo_session_t *session; u32 listener_index = htonl (mp->context); - svm_fifo_t *rx_fifo, *tx_fifo; clib_atomic_add_fetch (&em->max_sim_connects, 1); @@ -621,24 +615,18 @@ session_connected_handler (session_connected_msg_t * mp) } session = echo_session_new (em); - if (echo_segment_lookup (mp->segment_handle) == ~0) + + if (echo_attach_session (mp->segment_handle, mp->server_rx_fifo, + mp->server_tx_fifo, session)) { ECHO_FAIL (ECHO_FAIL_CONNECTED_WAIT_FOR_SEG_ALLOC, "connected wait_for_segment_allocation errored"); return; } - - rx_fifo = uword_to_pointer (mp->server_rx_fifo, svm_fifo_t *); - rx_fifo->client_session_index = session->session_index; - tx_fifo = uword_to_pointer (mp->server_tx_fifo, svm_fifo_t *); - tx_fifo->client_session_index = session->session_index; - - session->rx_fifo = rx_fifo; - session->tx_fifo = tx_fifo; - session->vpp_session_handle = mp->handle; - session->start = clib_time_now (&em->clib_time); session->vpp_evt_q = uword_to_pointer (mp->vpp_event_queue_address, svm_msg_q_t *); + session->vpp_session_handle = mp->handle; + session->start = clib_time_now (&em->clib_time); session->listener_index = listener_index; /* session->transport needed by app_send_dgram */ clib_memcpy_fast (&session->transport.lcl_ip, &mp->lcl.ip, diff --git a/src/plugins/hs_apps/sapi/vpp_echo_bapi.c b/src/plugins/hs_apps/sapi/vpp_echo_bapi.c index e71c0e9adba..c643cec2ce3 100644 --- a/src/plugins/hs_apps/sapi/vpp_echo_bapi.c +++ b/src/plugins/hs_apps/sapi/vpp_echo_bapi.c @@ -262,6 +262,39 @@ echo_segment_detach (u64 segment_handle) clib_spinlock_unlock (&em->segment_handles_lock); } +int +echo_attach_session (uword segment_handle, uword rxf_offset, uword txf_offset, + echo_session_t *s) +{ + svm_fifo_shared_t *rx_fifo, *tx_fifo; + echo_main_t *em = &echo_main; + fifo_segment_t *fs; + u32 fs_index; + + fs_index = echo_segment_lookup (segment_handle); + if (fs_index == (u32) ~0) + { + ECHO_LOG (0, "ERROR: segment for session %u is not mounted!", + s->session_index); + return -1; + } + + rx_fifo = uword_to_pointer (rxf_offset, svm_fifo_shared_t *); + tx_fifo = uword_to_pointer (txf_offset, svm_fifo_shared_t *); + rx_fifo->client_session_index = s->session_index; + tx_fifo->client_session_index = s->session_index; + + clib_spinlock_lock (&em->segment_handles_lock); + + fs = fifo_segment_get_segment (&em->segment_main, fs_index); + s->rx_fifo = fifo_segment_alloc_fifo_w_shared (fs, rx_fifo); + s->tx_fifo = fifo_segment_alloc_fifo_w_shared (fs, tx_fifo); + + clib_spinlock_unlock (&em->segment_handles_lock); + + return 0; +} + /* * * Binary API callbacks diff --git a/src/plugins/hs_apps/sapi/vpp_echo_common.h b/src/plugins/hs_apps/sapi/vpp_echo_common.h index 0def7adab36..cd2bbb6038c 100644 --- a/src/plugins/hs_apps/sapi/vpp_echo_common.h +++ b/src/plugins/hs_apps/sapi/vpp_echo_common.h @@ -443,6 +443,8 @@ int echo_segment_attach (u64 segment_handle, char *name, ssvm_segment_type_t type, int fd); u32 echo_segment_lookup (u64 segment_handle); void echo_segment_detach (u64 segment_handle); +int echo_attach_session (uword segment_handle, uword rxf_offset, + uword txf_offset, echo_session_t *s); /* Binary API */ diff --git a/src/plugins/hs_apps/sapi/vpp_echo_proto_udp.c b/src/plugins/hs_apps/sapi/vpp_echo_proto_udp.c index 456113cdd7d..9689a83d76b 100644 --- a/src/plugins/hs_apps/sapi/vpp_echo_proto_udp.c +++ b/src/plugins/hs_apps/sapi/vpp_echo_proto_udp.c @@ -126,24 +126,24 @@ udp_echo_reset_cb (session_reset_msg_t * mp, echo_session_t * s) static void udp_echo_bound_uri_cb (session_bound_msg_t * mp, echo_session_t * session) { - svm_fifo_t *rx_fifo, *tx_fifo; echo_main_t *em = &echo_main; u32 session_index = session->session_index; if (!em->i_am_master || em->uri_elts.transport_proto != TRANSPORT_PROTO_UDP) return; - rx_fifo = uword_to_pointer (mp->rx_fifo, svm_fifo_t *); - tx_fifo = uword_to_pointer (mp->tx_fifo, svm_fifo_t *); - rx_fifo->client_session_index = session_index; - tx_fifo->client_session_index = session_index; + if (echo_attach_session (mp->segment_handle, mp->rx_fifo, mp->tx_fifo, + session)) + { + ECHO_FAIL (ECHO_FAIL_ACCEPTED_WAIT_FOR_SEG_ALLOC, + "accepted wait_for_segment_allocation errored"); + return; + } - session->rx_fifo = rx_fifo; - session->tx_fifo = tx_fifo; + session->vpp_evt_q = uword_to_pointer (mp->vpp_evt_q, svm_msg_q_t *); session->transport.is_ip4 = mp->lcl_is_ip4; clib_memcpy_fast (&session->transport.lcl_ip, mp->lcl_ip, sizeof (ip46_address_t)); session->transport.lcl_port = mp->lcl_port; - session->vpp_evt_q = uword_to_pointer (mp->vpp_evt_q, svm_msg_q_t *); echo_notify_event (em, ECHO_EVT_FIRST_QCONNECT); session->session_type = ECHO_SESSION_TYPE_STREAM; diff --git a/src/plugins/unittest/segment_manager_test.c b/src/plugins/unittest/segment_manager_test.c index 2e8261e1c90..c6f52972e51 100644 --- a/src/plugins/unittest/segment_manager_test.c +++ b/src/plugins/unittest/segment_manager_test.c @@ -670,8 +670,8 @@ segment_manager_test_fifo_ops (vlib_main_t * vm, unformat_input_t * input) "fifo_segment_get_mem_status %s", states_str[rv]); /* (virtual) fifo size is still large as it is not updated */ - SEG_MGR_TEST ((rx_fifo->size == most_grown), "rx_fifo->size %u", - rx_fifo->size); + SEG_MGR_TEST ((rx_fifo->shr->size == most_grown), "rx_fifo->size %u", + rx_fifo->shr->size); vnet_app_detach_args_t detach_args = { .app_index = attach_args.app_index, diff --git a/src/plugins/unittest/session_test.c b/src/plugins/unittest/session_test.c index d3a4e091647..862d81f4acb 100644 --- a/src/plugins/unittest/session_test.c +++ b/src/plugins/unittest/session_test.c @@ -1850,7 +1850,7 @@ session_test_mq_speed (vlib_main_t * vm, unformat_input_t * input) s.rx_fifo = rx_fifo; s.tx_fifo = tx_fifo; s.session_state = SESSION_STATE_READY; - counter = (u64 *) rx_fifo->head_chunk->data; + counter = (u64 *) rx_fifo->shr->head_chunk->data; start = vlib_time_now (vm); pid = fork (); diff --git a/src/plugins/unittest/svm_fifo_test.c b/src/plugins/unittest/svm_fifo_test.c index 6adba7e8e59..978dc10ff99 100644 --- a/src/plugins/unittest/svm_fifo_test.c +++ b/src/plugins/unittest/svm_fifo_test.c @@ -256,7 +256,7 @@ sfifo_test_fifo1 (vlib_main_t * vm, unformat_input_t * input) */ rv = svm_fifo_enqueue (f, sizeof (u32), (u8 *) test_data); SFIFO_TEST ((rv == sizeof (u32)), "enqueued %d", rv); - SFIFO_TEST ((f->tail == 4), "fifo tail %u", f->tail); + SFIFO_TEST ((f->shr->tail == 4), "fifo tail %u", f->shr->tail); /* * Create 3 chunks in the future. The offsets are relative @@ -264,7 +264,7 @@ sfifo_test_fifo1 (vlib_main_t * vm, unformat_input_t * input) */ for (i = 0; i < 3; i++) { - offset = (2 * i + 1) * sizeof (u32) - f->tail; + offset = (2 * i + 1) * sizeof (u32) - f->shr->tail; data = (u8 *) (test_data + (2 * i + 1)); if (i == 0) { @@ -286,14 +286,14 @@ sfifo_test_fifo1 (vlib_main_t * vm, unformat_input_t * input) if (verbose) vlib_cli_output (vm, "fifo after odd segs: %U", format_svm_fifo, f, 1); - SFIFO_TEST ((f->tail == 8), "fifo tail %u", f->tail); + SFIFO_TEST ((f->shr->tail == 8), "fifo tail %u", f->shr->tail); SFIFO_TEST ((svm_fifo_n_ooo_segments (f) == 2), "number of ooo segments %u", svm_fifo_n_ooo_segments (f)); /* * Try adding a completely overlapped segment */ - offset = 3 * sizeof (u32) - f->tail; + offset = 3 * sizeof (u32) - f->shr->tail; data = (u8 *) (test_data + 3); rv = svm_fifo_enqueue_with_offset (f, offset, sizeof (u32), data); if (rv) @@ -319,7 +319,7 @@ sfifo_test_fifo1 (vlib_main_t * vm, unformat_input_t * input) */ for (i = 3; i > 1; i--) { - offset = (2 * i + 0) * sizeof (u32) - f->tail; + offset = (2 * i + 0) * sizeof (u32) - f->shr->tail; data = (u8 *) (test_data + (2 * i + 0)); rv = svm_fifo_enqueue_with_offset (f, offset, sizeof (u32), data); if (verbose) @@ -381,7 +381,7 @@ sfifo_test_fifo1 (vlib_main_t * vm, unformat_input_t * input) for (i = 0; i < 4; i++) { - offset = (2 * i + 1) * sizeof (u32) - f->tail; + offset = (2 * i + 1) * sizeof (u32) - f->shr->tail; data = (u8 *) (test_data + (2 * i + 1)); rv = svm_fifo_enqueue_with_offset (f, offset, sizeof (u32), data); if (verbose) @@ -394,7 +394,7 @@ sfifo_test_fifo1 (vlib_main_t * vm, unformat_input_t * input) } } - rv = svm_fifo_enqueue_with_offset (f, 8 - f->tail, 21, data); + rv = svm_fifo_enqueue_with_offset (f, 8 - f->shr->tail, 21, data); SFIFO_TEST ((rv == 0), "ooo enqueued %u", rv); SFIFO_TEST ((svm_fifo_n_ooo_segments (f) == 1), "number of ooo segments %u", svm_fifo_n_ooo_segments (f)); @@ -422,7 +422,7 @@ sfifo_test_fifo1 (vlib_main_t * vm, unformat_input_t * input) for (i = 0; i < 4; i++) { - offset = (2 * i + 1) * sizeof (u32) - f->tail; + offset = (2 * i + 1) * sizeof (u32) - f->shr->tail; data = (u8 *) (test_data + (2 * i + 1)); rv = svm_fifo_enqueue_with_offset (f, offset, sizeof (u32), data); if (verbose) @@ -500,8 +500,8 @@ sfifo_test_fifo2 (vlib_main_t * vm) { tp = vp + i; data64 = tp->offset; - svm_fifo_enqueue_with_offset (f, tp->offset - f->tail, tp->len, - (u8 *) & data64); + svm_fifo_enqueue_with_offset (f, tp->offset - f->shr->tail, tp->len, + (u8 *) &data64); } /* Expected result: one big fat chunk at offset 4 */ @@ -530,8 +530,8 @@ sfifo_test_fifo2 (vlib_main_t * vm) { tp = &test_data[i]; data64 = tp->offset; - rv = svm_fifo_enqueue_with_offset (f, tp->offset - f->tail, tp->len, - (u8 *) & data64); + rv = svm_fifo_enqueue_with_offset (f, tp->offset - f->shr->tail, tp->len, + (u8 *) &data64); if (rv) { clib_warning ("enqueue returned %d", rv); @@ -704,10 +704,9 @@ sfifo_test_fifo3 (vlib_main_t * vm, unformat_input_t * input) for (i = !randomize; i < vec_len (generate); i++) { tp = generate + i; - svm_fifo_enqueue_with_offset (f, - fifo_initial_offset + tp->offset - - f->tail, tp->len, - (u8 *) data_pattern + tp->offset); + svm_fifo_enqueue_with_offset ( + f, fifo_initial_offset + tp->offset - f->shr->tail, tp->len, + (u8 *) data_pattern + tp->offset); } /* Add the first segment in order for non random data */ @@ -823,8 +822,8 @@ sfifo_test_fifo4 (vlib_main_t * vm, unformat_input_t * input) for (i = test_n_bytes - 1; i > 0; i--) { - rv = svm_fifo_enqueue_with_offset (f, fifo_initial_offset + i - f->tail, - sizeof (u8), &test_data[i]); + rv = svm_fifo_enqueue_with_offset ( + f, fifo_initial_offset + i - f->shr->tail, sizeof (u8), &test_data[i]); if (verbose) vlib_cli_output (vm, "add [%d] [%d, %d]", i, i, i + sizeof (u8)); if (rv) @@ -1150,7 +1149,8 @@ sfifo_test_fifo7 (vlib_main_t * vm, unformat_input_t * input) if (compare_data (data_buf, test_data, 0, n_test_bytes, (u32 *) & j)) SFIFO_TEST (0, "[%d] dequeued %u expected %u", j, data_buf[j], test_data[j]); - svm_fifo_init_pointers (f, (~0 - i) % f->size, (~0 - i) % f->size); + svm_fifo_init_pointers (f, (~0 - i) % f->shr->size, + (~0 - i) % f->shr->size); } SFIFO_TEST (1, "passed multiple ooo enqueue/dequeue"); @@ -1412,13 +1412,13 @@ sfifo_test_fifo_grow (vlib_main_t * vm, unformat_input_t * input) SFIFO_TEST (rv == 2, "should have 2 chunks has %u", rv); SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane"); - c = f->head_chunk; + c = f->shr->head_chunk; SFIFO_TEST (c->start_byte == 0, "head start byte should be %u", 0); SFIFO_TEST (c->length == 4096, "head chunk length should be %u", 4096); - SFIFO_TEST (f->tail_chunk == 0, "no tail chunk"); + SFIFO_TEST (f->shr->tail_chunk == 0, "no tail chunk"); SFIFO_TEST (f->ooo_enq == 0, "should have no ooo enq chunk"); SFIFO_TEST (f->ooo_deq == 0, "should have no ooo deq chunk"); - c = f->end_chunk; + c = f->shr->end_chunk; SFIFO_TEST (c->start_byte == last_start_byte, "end chunk start byte should" " be %u", last_start_byte); SFIFO_TEST (c->length == 4096, "end chunk length should be %u", 4096); @@ -1438,8 +1438,8 @@ sfifo_test_fifo_grow (vlib_main_t * vm, unformat_input_t * input) vlib_cli_output (vm, "[%d] dequeued %u expected %u", i, data_buf[i], test_data[i]); SFIFO_TEST ((rv == 0), "dequeued compared to original returned %d", rv); - SFIFO_TEST (f->head_chunk == 0, "head chunk should be 0"); - SFIFO_TEST (f->tail_chunk == 0, "tail chunk should be 0"); + SFIFO_TEST (f->shr->head_chunk == 0, "head chunk should be 0"); + SFIFO_TEST (f->shr->tail_chunk == 0, "tail chunk should be 0"); SFIFO_TEST (f->ooo_deq == 0, "should have no ooo deq chunk"); SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane"); @@ -1461,12 +1461,12 @@ sfifo_test_fifo_grow (vlib_main_t * vm, unformat_input_t * input) SFIFO_TEST (rv == 2, "should have %u chunks has %u", 2, rv); SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane"); - SFIFO_TEST (f->head_chunk == 0, "should have no head chunk"); + SFIFO_TEST (f->shr->head_chunk == 0, "should have no head chunk"); /* When new fifo chunks are allocated, tail is initialized */ - SFIFO_TEST (f->tail_chunk != 0, "should have no tail chunk"); + SFIFO_TEST (f->shr->tail_chunk != 0, "should have no tail chunk"); SFIFO_TEST (f->ooo_enq != 0, "should have an ooo enq chunk"); - c = f->end_chunk; + c = f->shr->end_chunk; SFIFO_TEST (c->start_byte == last_start_byte, "end chunk should start at %u", last_start_byte); SFIFO_TEST (c->length == 8192, "end chunk length should be %u", 8192); @@ -1483,9 +1483,9 @@ sfifo_test_fifo_grow (vlib_main_t * vm, unformat_input_t * input) SFIFO_TEST (rv == 2, "should have %u chunks has %u", 2, rv); SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane"); - SFIFO_TEST (f->head_chunk == 0, "should have no head chunk"); + SFIFO_TEST (f->shr->head_chunk == 0, "should have no head chunk"); /* Fifo is full so tail and ooo_enq should be 0 */ - SFIFO_TEST (f->tail_chunk == 0, "should have no tail chunk"); + SFIFO_TEST (f->shr->tail_chunk == 0, "should have no tail chunk"); SFIFO_TEST (f->ooo_enq == 0, "should have no ooo enq chunk"); /* @@ -1542,8 +1542,8 @@ sfifo_test_fifo_grow (vlib_main_t * vm, unformat_input_t * input) last_start_byte += 8192; SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane"); - SFIFO_TEST (f->head_chunk == 0, "should have no head chunk"); - SFIFO_TEST (f->tail_chunk == 0, "should have no tail chunk"); + SFIFO_TEST (f->shr->head_chunk == 0, "should have no head chunk"); + SFIFO_TEST (f->shr->tail_chunk == 0, "should have no tail chunk"); /* We don't remove the last chunk even when the fifo goes empty */ rv = svm_fifo_n_chunks (f); @@ -1575,12 +1575,12 @@ sfifo_test_fifo_grow (vlib_main_t * vm, unformat_input_t * input) SFIFO_TEST (rv == 2, "should have %u chunks has %u", 2, rv); SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane"); - SFIFO_TEST (f->head_chunk == 0, "should have no head chunk"); + SFIFO_TEST (f->shr->head_chunk == 0, "should have no head chunk"); /* When new fifo chunks are allocated, tail is initialized */ - SFIFO_TEST (f->tail_chunk != 0, "should have no tail chunk"); + SFIFO_TEST (f->shr->tail_chunk != 0, "should have no tail chunk"); SFIFO_TEST (f->ooo_enq != 0, "should have an ooo enq chunk"); - c = f->end_chunk; + c = f->shr->end_chunk; SFIFO_TEST (c->start_byte == last_start_byte, "end chunk should start at %u", last_start_byte); SFIFO_TEST (c->length == 16384, "end chunk length should be %u", 16384); @@ -1623,7 +1623,7 @@ sfifo_test_fifo_grow (vlib_main_t * vm, unformat_input_t * input) SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane"); last_start_byte += 16384; - c = f->end_chunk; + c = f->shr->end_chunk; SFIFO_TEST (c->start_byte == last_start_byte, "end chunk should start at %u", last_start_byte); SFIFO_TEST (c->length == 4096, "end chunk length should be %u", 4096); @@ -1647,8 +1647,8 @@ sfifo_test_fifo_grow (vlib_main_t * vm, unformat_input_t * input) SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane"); /* fifo does not end on chunk boundary because of the - 100 */ - SFIFO_TEST (f->head_chunk != 0, "should have head chunk"); - SFIFO_TEST (f->tail_chunk != 0, "should have tail chunk"); + SFIFO_TEST (f->shr->head_chunk != 0, "should have head chunk"); + SFIFO_TEST (f->shr->tail_chunk != 0, "should have tail chunk"); /* * Enqueue and dequeue byte-by-byte ooo @@ -1673,7 +1673,7 @@ sfifo_test_fifo_grow (vlib_main_t * vm, unformat_input_t * input) SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane"); last_start_byte += 4096; - c = f->end_chunk; + c = f->shr->end_chunk; SFIFO_TEST (c->start_byte == last_start_byte, "end chunk should start at %u", last_start_byte); SFIFO_TEST (c->length == 16384, "end chunk length should be %u", 16384); @@ -1688,8 +1688,8 @@ sfifo_test_fifo_grow (vlib_main_t * vm, unformat_input_t * input) SFIFO_TEST ((rv == fifo_size), "all bytes should be dropped %u", rv); SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane"); - SFIFO_TEST (f->head_chunk != 0, "should have head chunk"); - SFIFO_TEST (f->tail_chunk != 0, "should have tail chunk"); + SFIFO_TEST (f->shr->head_chunk != 0, "should have head chunk"); + SFIFO_TEST (f->shr->tail_chunk != 0, "should have tail chunk"); /* We don't remove the last chunk even when the fifo goes empty */ rv = svm_fifo_n_chunks (f); @@ -1941,7 +1941,7 @@ sfifo_test_fifo_indirect (vlib_main_t * vm, unformat_input_t * input) svm_fifo_set_size (f, fifo_size); validate_test_and_buf_vecs (&test_data, &data_buf, fifo_size); - c = f->start_chunk; + c = f->shr->start_chunk; SFIFO_TEST (c->next == 0, "no next"); svm_fifo_fill_chunk_list (f); @@ -1958,7 +1958,7 @@ sfifo_test_fifo_indirect (vlib_main_t * vm, unformat_input_t * input) SFIFO_TEST (svm_fifo_is_sane (f), "fifo should be sane"); c = svm_fifo_tail_chunk (f); - SFIFO_TEST (c == f->end_chunk, "tail is end chunk"); + SFIFO_TEST (c == f->shr->end_chunk, "tail is end chunk"); /* Initialize head chunk */ rv = svm_fifo_max_read_chunk (f); @@ -1972,7 +1972,7 @@ sfifo_test_fifo_indirect (vlib_main_t * vm, unformat_input_t * input) SFIFO_TEST (rv == 4096, "dequeue should work"); c = svm_fifo_head_chunk (f); - SFIFO_TEST (c == f->end_chunk, "head chunk should be last"); + SFIFO_TEST (c == f->shr->end_chunk, "head chunk should be last"); rv = svm_fifo_max_read_chunk (f); SFIFO_TEST (rv == 0, "max read chunk %u", rv); @@ -2409,13 +2409,13 @@ sfifo_test_fifo_segment_slave (int verbose) svm_fifo_dequeue (f, vec_len (retrieved_data), retrieved_data); if (memcmp (retrieved_data, test_data, vec_len (retrieved_data))) { - result = (u32 *) f->head_chunk->data; + result = (u32 *) f->shr->head_chunk->data; *result = 1; _exit (0); } } - result = (u32 *) f->head_chunk->data; + result = (u32 *) f->shr->head_chunk->data; *result = 0; vec_free (test_data); @@ -2469,7 +2469,7 @@ sfifo_test_fifo_segment_master_slave (int verbose) usleep (1e3); - result = (u32 *) f->head_chunk->data; + result = (u32 *) f->shr->head_chunk->data; SFIFO_TEST (*result == 0, "slave reported no error"); vec_free (a->new_segment_indices); @@ -2597,7 +2597,7 @@ sfifo_test_fifo_segment_prealloc (int verbose) rv = fifo_segment_num_free_fifos (fs); SFIFO_TEST (rv == 50, "prealloc fifo hdrs expected %u is %u", 50, rv); rv = fifo_segment_free_bytes (fs); - free_space -= sizeof (svm_fifo_t) * 50; + free_space -= sizeof (svm_fifo_shared_t) * 50; /* Memory alloc alignment accounts for the difference */ SFIFO_TEST (approx_leq (free_space, rv, 128), "free space expected %u is %u", free_space, rv); diff --git a/src/svm/fifo_segment.c b/src/svm/fifo_segment.c index b4c3cbe6c6d..19fd0520666 100644 --- a/src/svm/fifo_segment.c +++ b/src/svm/fifo_segment.c @@ -14,6 +14,7 @@ */ #include <svm/fifo_segment.h> +#include <vppinfra/mem.h> static inline void * fsh_alloc_aligned (fifo_segment_header_t *fsh, uword size, uword align) @@ -239,7 +240,7 @@ fss_fl_chunk_bytes_sub (fifo_segment_slice_t * fss, uword size) int fifo_segment_init (fifo_segment_t * fs) { - u32 align = 8, offset = 2 * 4096, slices_sz; + u32 align = 8, offset = 2 * 4096, slices_sz, i; uword max_fifo, seg_start, seg_sz; fifo_segment_header_t *fsh; ssvm_shared_header_t *sh; @@ -270,6 +271,11 @@ fifo_segment_init (fifo_segment_t * fs) fs->max_byte_index = fsh->max_byte_index; fs->h = sh->opaque[0] = fsh; + vec_validate (fs->slices, fs->n_slices - 1); + for (i = 0; i < fs->n_slices; i++) + fs->slices[i].fifos = + clib_mem_bulk_init (sizeof (svm_fifo_t), CLIB_CACHE_LINE_BYTES, 32); + sh->ready = 1; return (0); } @@ -343,6 +349,9 @@ fifo_segment_attach (fifo_segment_main_t * sm, fifo_segment_create_args_t * a) goto done; fs->max_byte_index = fsh->max_byte_index; + vec_validate (fs->slices, 0); + fs->slices[0].fifos = + clib_mem_bulk_init (sizeof (svm_fifo_t), CLIB_CACHE_LINE_BYTES, 32); done: vec_add1 (a->new_segment_indices, fs - sm->segments); @@ -352,6 +361,7 @@ done: void fifo_segment_delete (fifo_segment_main_t * sm, fifo_segment_t * s) { + fifo_segment_cleanup (s); ssvm_delete (&s->ssvm); clib_memset (s, 0xfe, sizeof (*s)); pool_put (sm->segments, s); @@ -478,7 +488,7 @@ static int fsh_try_alloc_fifo_hdr_batch (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss, u32 batch_size) { - svm_fifo_t *f; + svm_fifo_shared_t *f; uword size; u8 *fmem; int i; @@ -494,7 +504,7 @@ fsh_try_alloc_fifo_hdr_batch (fifo_segment_header_t * fsh, /* Carve fifo hdr space */ for (i = 0; i < batch_size; i++) { - f = (svm_fifo_t *) fmem; + f = (svm_fifo_shared_t *) fmem; memset (f, 0, sizeof (*f)); f->next = fss->free_fifos; fss->free_fifos = f; @@ -555,11 +565,10 @@ fs_try_alloc_fifo_batch (fifo_segment_header_t * fsh, return fsh_try_alloc_chunk_batch (fsh, fss, fl_index, batch_size); } -static svm_fifo_t * -fsh_try_alloc_fifo_hdr (fifo_segment_header_t * fsh, - fifo_segment_slice_t * fss) +static svm_fifo_shared_t * +fsh_try_alloc_fifo_hdr (fifo_segment_header_t *fsh, fifo_segment_slice_t *fss) { - svm_fifo_t *f; + svm_fifo_shared_t *f; if (!fss->free_fifos) { @@ -642,40 +651,41 @@ done: * - single fifo allocation * - grab multiple fifo chunks from freelists */ -static svm_fifo_t * -fs_try_alloc_fifo (fifo_segment_header_t * fsh, fifo_segment_slice_t * fss, - u32 data_bytes) +static svm_fifo_shared_t * +fs_try_alloc_fifo (fifo_segment_header_t *fsh, u32 slice_index, u32 data_bytes) { + fifo_segment_slice_t *fss; u32 fl_index, min_size; svm_fifo_chunk_t *c; - svm_fifo_t *f = 0; + svm_fifo_shared_t *sf = 0; + fss = fsh_slice_get (fsh, slice_index); min_size = clib_max ((fsh->pct_first_alloc * data_bytes) / 100, 4096); fl_index = fs_freelist_for_size (min_size); if (!fss_chunk_fl_index_is_valid (fss, fl_index)) return 0; - f = fsh_try_alloc_fifo_hdr (fsh, fss); - if (!f) + sf = fsh_try_alloc_fifo_hdr (fsh, fss); + if (!sf) return 0; c = fsh_try_alloc_chunk (fsh, fss, min_size); if (!c) { - f->next = fss->free_fifos; - fss->free_fifos = f; + sf->next = fss->free_fifos; + fss->free_fifos = sf; return 0; } - f->start_chunk = c; + sf->start_chunk = c; while (c->next) c = c->next; - f->end_chunk = c; - f->size = data_bytes; - f->fs_hdr = fsh; + sf->end_chunk = c; + sf->size = data_bytes; + sf->slice_index = slice_index; - return f; + return sf; } svm_fifo_chunk_t * @@ -720,6 +730,36 @@ fsh_collect_chunks (fifo_segment_header_t * fsh, u32 slice_index, fsh_slice_collect_chunks (fsh, fss, c); } +svm_fifo_t * +fs_fifo_alloc (fifo_segment_t *fs, u32 slice_index) +{ + fifo_slice_private_t *pfss = &fs->slices[slice_index]; + svm_fifo_t *f; + + f = clib_mem_bulk_alloc (pfss->fifos); + clib_memset (f, 0, sizeof (*f)); + return f; +} + +void +fs_fifo_free (fifo_segment_t *fs, svm_fifo_t *f) +{ + u32 slice_index = f->shr->slice_index; + fifo_slice_private_t *pfss; + + pfss = &fs->slices[slice_index]; + clib_mem_bulk_free (pfss->fifos, f); +} + +void +fifo_segment_cleanup (fifo_segment_t *fs) +{ + int slice_index; + + for (slice_index = 0; slice_index < fs->n_slices; slice_index++) + clib_mem_bulk_destroy (fs->slices[slice_index].fifos); +} + /** * Allocate fifo in fifo segment */ @@ -729,6 +769,7 @@ fifo_segment_alloc_fifo_w_slice (fifo_segment_t * fs, u32 slice_index, { fifo_segment_header_t *fsh = fs->h; fifo_segment_slice_t *fss; + svm_fifo_shared_t *sf; svm_fifo_t *f = 0; ASSERT (slice_index < fs->n_slices); @@ -736,15 +777,18 @@ fifo_segment_alloc_fifo_w_slice (fifo_segment_t * fs, u32 slice_index, if (PREDICT_FALSE (data_bytes > 1 << fsh->max_log2_fifo_size)) return 0; - fss = fsh_slice_get (fsh, slice_index); - f = fs_try_alloc_fifo (fsh, fss, data_bytes); - if (!f) + sf = fs_try_alloc_fifo (fsh, slice_index, data_bytes); + if (!sf) goto done; - f->slice_index = slice_index; + f = fs_fifo_alloc (fs, slice_index); + f->fs_hdr = fsh; + f->shr = sf; svm_fifo_init (f, data_bytes); + fss = fsh_slice_get (fsh, slice_index); + /* If rx fifo type add to active fifos list. When cleaning up segment, * we need a list of active sessions that should be disconnected. Since * both rx and tx fifos keep pointers to the session, it's enough to track @@ -762,6 +806,19 @@ done: return (f); } +svm_fifo_t * +fifo_segment_alloc_fifo_w_shared (fifo_segment_t *fs, svm_fifo_shared_t *sf) +{ + svm_fifo_t *f = fs_fifo_alloc (fs, 0); + f->fs_hdr = fs->h; + f->shr = sf; + + f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX; + f->segment_index = SVM_FIFO_INVALID_INDEX; + f->refcnt = 1; + return f; +} + /** * Free fifo allocated in fifo segment */ @@ -770,13 +827,35 @@ fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f) { fifo_segment_header_t *fsh = fs->h; fifo_segment_slice_t *fss; + svm_fifo_shared_t *sf; ASSERT (f->refcnt > 0); if (--f->refcnt > 0) return; - fss = fsh_slice_get (fsh, f->slice_index); + /* + * Cleanup shared state + */ + + sf = f->shr; + fss = fsh_slice_get (fsh, sf->slice_index); + + /* Free fifo chunks */ + fsh_slice_collect_chunks (fsh, fss, sf->start_chunk); + + sf->start_chunk = sf->end_chunk = 0; + sf->head_chunk = sf->tail_chunk = 0; + + /* Add to free list */ + sf->next = fss->free_fifos; + fss->free_fifos = sf; + + fss->virtual_mem -= svm_fifo_size (f); + + /* + * Cleanup private state + */ /* Remove from active list. Only rx fifos are tracked */ if (f->flags & SVM_FIFO_F_LL_TRACKED) @@ -785,28 +864,19 @@ fifo_segment_free_fifo (fifo_segment_t * fs, svm_fifo_t * f) f->flags &= ~SVM_FIFO_F_LL_TRACKED; } - /* Free fifo chunks */ - fsh_slice_collect_chunks (fsh, fss, f->start_chunk); - - f->start_chunk = f->end_chunk = 0; - f->head_chunk = f->tail_chunk = f->ooo_enq = f->ooo_deq = 0; - - /* not allocated on segment heap */ svm_fifo_free_chunk_lookup (f); svm_fifo_free_ooo_data (f); if (CLIB_DEBUG) { - f->master_session_index = ~0; + sf->master_session_index = ~0; f->master_thread_index = ~0; } - fss->virtual_mem -= svm_fifo_size (f); - - /* Add to free list */ - f->next = fss->free_fifos; + f->ooo_enq = f->ooo_deq = 0; f->prev = 0; - fss->free_fifos = f; + + fs_fifo_free (fs, f); fsh_active_fifos_update (fsh, -1); } @@ -820,12 +890,12 @@ fifo_segment_detach_fifo (fifo_segment_t * fs, svm_fifo_t * f) ASSERT (f->refcnt == 1); - fss = fsh_slice_get (fs->h, f->slice_index); + fss = fsh_slice_get (fs->h, f->shr->slice_index); fss->virtual_mem -= svm_fifo_size (f); if (f->flags & SVM_FIFO_F_LL_TRACKED) fss_fifo_del_active_list (fss, f); - c = f->start_chunk; + c = f->shr->start_chunk; while (c) { fl_index = fs_freelist_for_size (c->length); @@ -842,13 +912,13 @@ fifo_segment_attach_fifo (fifo_segment_t * fs, svm_fifo_t * f, svm_fifo_chunk_t *c; u32 fl_index; - f->slice_index = slice_index; - fss = fsh_slice_get (fs->h, f->slice_index); + f->shr->slice_index = slice_index; + fss = fsh_slice_get (fs->h, f->shr->slice_index); fss->virtual_mem += svm_fifo_size (f); if (f->flags & SVM_FIFO_F_LL_TRACKED) fss_fifo_add_active_list (fss, f); - c = f->start_chunk; + c = f->shr->start_chunk; while (c) { fl_index = fs_freelist_for_size (c->length); @@ -966,7 +1036,7 @@ fifo_segment_num_fifos (fifo_segment_t * fs) static u32 fs_slice_num_free_fifos (fifo_segment_slice_t * fss) { - svm_fifo_t *f; + svm_fifo_shared_t *f; u32 count = 0; f = fss->free_fifos; diff --git a/src/svm/fifo_segment.h b/src/svm/fifo_segment.h index 39c94548636..006ffc40f63 100644 --- a/src/svm/fifo_segment.h +++ b/src/svm/fifo_segment.h @@ -69,6 +69,7 @@ typedef struct fifo_segment_header_t *h; /**< fifo segment data */ uword max_byte_index; u8 n_slices; /**< number of fifo segment slices */ + fifo_slice_private_t *slices; /**< private slice information */ } fifo_segment_t; typedef struct @@ -95,6 +96,7 @@ int fifo_segment_create (fifo_segment_main_t * sm, int fifo_segment_attach (fifo_segment_main_t * sm, fifo_segment_create_args_t * a); void fifo_segment_delete (fifo_segment_main_t * sm, fifo_segment_t * fs); +void fifo_segment_cleanup (fifo_segment_t *fs); fifo_segment_t *fifo_segment_get_segment (fifo_segment_main_t * sm, u32 fs_index); u32 fifo_segment_index (fifo_segment_main_t * sm, fifo_segment_t * fs); @@ -112,6 +114,8 @@ svm_fifo_t *fifo_segment_alloc_fifo_w_slice (fifo_segment_t * fs, u32 slice_index, u32 data_bytes, fifo_segment_ftype_t ftype); +svm_fifo_t *fifo_segment_alloc_fifo_w_shared (fifo_segment_t *fs, + svm_fifo_shared_t *sf); /** * Free fifo allocated in fifo segment diff --git a/src/svm/fifo_types.h b/src/svm/fifo_types.h index e839e7eaab7..434b2c38b4c 100644 --- a/src/svm/fifo_types.h +++ b/src/svm/fifo_types.h @@ -59,59 +59,65 @@ typedef struct u32 action; } svm_fifo_trace_elem_t; -typedef struct _svm_fifo +typedef struct svm_fifo_shr_ { - CLIB_CACHE_LINE_ALIGN_MARK (shared_first); - fifo_segment_header_t *fs_hdr;/**< fifo segment header for fifo */ + CLIB_CACHE_LINE_ALIGN_MARK (shared); svm_fifo_chunk_t *start_chunk;/**< first chunk in fifo chunk list */ svm_fifo_chunk_t *end_chunk; /**< end chunk in fifo chunk list */ + volatile u32 has_event; /**< non-zero if deq event exists */ u32 min_alloc; /**< min chunk alloc if space available */ u32 size; /**< size of the fifo in bytes */ - u8 flags; /**< fifo flags */ - u8 slice_index; /**< segment slice for fifo */ - - CLIB_CACHE_LINE_ALIGN_MARK (shared_second); - volatile u32 has_event; /**< non-zero if deq event exists */ u32 master_session_index; /**< session layer session index */ u32 client_session_index; /**< app session index */ - u8 master_thread_index; /**< session layer thread index */ - u8 client_thread_index; /**< app worker index */ - i8 refcnt; /**< reference count */ - u32 segment_manager; /**< session layer segment manager index */ - u32 segment_index; /**< segment index in segment manager */ - struct _svm_fifo *next; /**< next in freelist/active chain */ - struct _svm_fifo *prev; /**< prev in active chain */ - - CLIB_CACHE_LINE_ALIGN_MARK (consumer); - rb_tree_t ooo_deq_lookup; /**< rbtree for ooo deq chunk lookup */ + u8 slice_index; /**< segment slice for fifo */ + struct svm_fifo_shr_ *next; /**< next in freelist/active chain */ + + CLIB_CACHE_LINE_ALIGN_MARK (consumer); svm_fifo_chunk_t *head_chunk; /**< tracks chunk where head lands */ - svm_fifo_chunk_t *ooo_deq; /**< last chunk used for ooo dequeue */ u32 head; /**< fifo head position/byte */ volatile u32 want_deq_ntf; /**< producer wants nudge */ volatile u32 has_deq_ntf; - CLIB_CACHE_LINE_ALIGN_MARK (producer); - rb_tree_t ooo_enq_lookup; /**< rbtree for ooo enq chunk lookup */ + CLIB_CACHE_LINE_ALIGN_MARK (producer); u32 tail; /**< fifo tail position/byte */ - u32 ooos_list_head; /**< Head of out-of-order linked-list */ svm_fifo_chunk_t *tail_chunk; /**< tracks chunk where tail lands */ + volatile u8 n_subscribers; /**< Number of subscribers for io events */ + u8 subscribers[SVM_FIFO_MAX_EVT_SUBSCRIBERS]; +} svm_fifo_shared_t; + +typedef struct _svm_fifo +{ + CLIB_CACHE_LINE_ALIGN_MARK (cacheline); + svm_fifo_shared_t *shr; /**< shared fifo in fifo segment memory */ + fifo_segment_header_t *fs_hdr; /**< fifo segment header for fifo */ + rb_tree_t ooo_enq_lookup; /**< rbtree for ooo enq chunk lookup */ + rb_tree_t ooo_deq_lookup; /**< rbtree for ooo deq chunk lookup */ + svm_fifo_chunk_t *ooo_deq; /**< last chunk used for ooo dequeue */ svm_fifo_chunk_t *ooo_enq; /**< last chunk used for ooo enqueue */ ooo_segment_t *ooo_segments; /**< Pool of ooo segments */ + u32 ooos_list_head; /**< Head of out-of-order linked-list */ u32 ooos_newest; /**< Last segment to have been updated */ - volatile u8 n_subscribers; /**< Number of subscribers for io events */ - u8 subscribers[SVM_FIFO_MAX_EVT_SUBSCRIBERS]; + + u8 flags; /**< fifo flags */ + u8 master_thread_index; /**< session layer thread index */ + u8 client_thread_index; /**< app worker index */ + i8 refcnt; /**< reference count */ + u32 segment_manager; /**< session layer segment manager index */ + u32 segment_index; /**< segment index in segment manager */ + + struct _svm_fifo *next; /**< prev in active chain */ + struct _svm_fifo *prev; /**< prev in active chain */ #if SVM_FIFO_TRACE svm_fifo_trace_elem_t *trace; #endif - } svm_fifo_t; typedef struct fifo_segment_slice_ { svm_fifo_chunk_t *free_chunks[FS_CHUNK_VEC_LEN]; /**< Free chunks by size */ svm_fifo_t *fifos; /**< Linked list of active RX fifos */ - svm_fifo_t *free_fifos; /**< Freelists by fifo size */ + svm_fifo_shared_t *free_fifos; /**< Freelists of fifo shared hdrs */ uword n_fl_chunk_bytes; /**< Chunk bytes on freelist */ uword virtual_mem; /**< Slice sum of all fifo sizes */ u32 num_chunks[FS_CHUNK_VEC_LEN]; /**< Allocated chunks by chunk size */ @@ -120,6 +126,12 @@ typedef struct fifo_segment_slice_ u32 chunk_lock; } fifo_segment_slice_t; +typedef struct fifo_slice_private_ +{ + clib_mem_bulk_handle_t fifos; /**< Bulk fifo allocator */ + uword virtual_mem; /**< Slice sum of all fifo sizes */ +} fifo_slice_private_t; + struct fifo_segment_header_ { uword n_cached_bytes; /**< Cached bytes */ diff --git a/src/svm/svm_fifo.c b/src/svm/svm_fifo.c index 0c08dba7aa2..2f910e0b95f 100644 --- a/src/svm/svm_fifo.c +++ b/src/svm/svm_fifo.c @@ -348,7 +348,7 @@ ooo_segment_try_collect (svm_fifo_t * f, u32 n_bytes_enqueued, u32 * tail) } } - ASSERT (bytes <= f->size); + ASSERT (bytes <= f->shr->size); return bytes; } @@ -372,23 +372,23 @@ svm_fifo_init (svm_fifo_t * f, u32 size) svm_fifo_chunk_t *c, *prev; u32 min_alloc; - f->size = size; + f->shr->size = size; f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX; f->segment_index = SVM_FIFO_INVALID_INDEX; f->refcnt = 1; - f->head = f->tail = f->flags = 0; - f->head_chunk = f->tail_chunk = f->start_chunk; + f->shr->head = f->shr->tail = f->flags = 0; + f->shr->head_chunk = f->shr->tail_chunk = f->shr->start_chunk; f->ooo_deq = f->ooo_enq = 0; min_alloc = size > 32 << 10 ? size >> 3 : 4096; min_alloc = clib_min (min_alloc, 64 << 10); - f->min_alloc = min_alloc; + f->shr->min_alloc = min_alloc; /* * Initialize chunks */ - f->start_chunk->start_byte = 0; - prev = f->start_chunk; + f->shr->start_chunk->start_byte = 0; + prev = f->shr->start_chunk; prev->enq_rb_index = prev->deq_rb_index = RBTREE_TNIL_INDEX; c = prev->next; @@ -447,7 +447,7 @@ svm_fifo_alloc (u32 data_size_in_bytes) c->length = data_size_in_bytes; c->enq_rb_index = RBTREE_TNIL_INDEX; c->deq_rb_index = RBTREE_TNIL_INDEX; - f->start_chunk = f->end_chunk = c; + f->shr->start_chunk = f->shr->end_chunk = c; return f; } @@ -486,7 +486,7 @@ svm_fifo_find_chunk (svm_fifo_t * f, u32 pos) { svm_fifo_chunk_t *c; - c = f->start_chunk; + c = f->shr->start_chunk; while (c && !f_chunk_includes_pos (c, pos)) c = c->next; @@ -513,16 +513,17 @@ svm_fifo_max_read_chunk (svm_fifo_t * f) u32 head, tail, end_chunk; f_load_head_tail_cons (f, &head, &tail); - ASSERT (!f->head_chunk || f_chunk_includes_pos (f->head_chunk, head)); + ASSERT (!f->shr->head_chunk || + f_chunk_includes_pos (f->shr->head_chunk, head)); - if (!f->head_chunk) + if (!f->shr->head_chunk) { - f->head_chunk = svm_fifo_find_chunk (f, head); - if (PREDICT_FALSE (!f->head_chunk)) + f->shr->head_chunk = svm_fifo_find_chunk (f, head); + if (PREDICT_FALSE (!f->shr->head_chunk)) return 0; } - end_chunk = f_chunk_end (f->head_chunk); + end_chunk = f_chunk_end (f->shr->head_chunk); return f_pos_lt (end_chunk, tail) ? end_chunk - head : tail - head; } @@ -533,9 +534,10 @@ svm_fifo_max_write_chunk (svm_fifo_t * f) u32 head, tail; f_load_head_tail_prod (f, &head, &tail); - ASSERT (!f->tail_chunk || f_chunk_includes_pos (f->tail_chunk, tail)); + ASSERT (!f->shr->tail_chunk || + f_chunk_includes_pos (f->shr->tail_chunk, tail)); - return f->tail_chunk ? f_chunk_end (f->tail_chunk) - tail : 0; + return f->shr->tail_chunk ? f_chunk_end (f->shr->tail_chunk) - tail : 0; } static rb_node_t * @@ -605,13 +607,13 @@ f_update_ooo_enq (svm_fifo_t * f, u32 start_pos, u32 end_pos) /* Use linear search if rbtree is not initialized */ if (PREDICT_FALSE (!rb_tree_is_init (rt))) { - f->ooo_enq = svm_fifo_find_next_chunk (f, f->tail_chunk, start_pos); + f->ooo_enq = svm_fifo_find_next_chunk (f, f->shr->tail_chunk, start_pos); return; } if (rt->root == RBTREE_TNIL_INDEX) { - c = f->tail_chunk; + c = f->shr->tail_chunk; ASSERT (c->enq_rb_index == RBTREE_TNIL_INDEX); c->enq_rb_index = rb_tree_add_custom (rt, c->start_byte, pointer_to_uword (c), f_pos_lt); @@ -660,7 +662,7 @@ f_update_ooo_deq (svm_fifo_t * f, u32 start_pos, u32 end_pos) if (rt->root == RBTREE_TNIL_INDEX) { - c = f->start_chunk; + c = f->shr->start_chunk; ASSERT (c->deq_rb_index == RBTREE_TNIL_INDEX); c->deq_rb_index = rb_tree_add_custom (rt, c->start_byte, pointer_to_uword (c), f_pos_lt); @@ -778,14 +780,14 @@ svm_fifo_overwrite_head (svm_fifo_t * f, u8 * src, u32 len) u32 head, tail, head_idx; svm_fifo_chunk_t *c; - ASSERT (len <= f->size); + ASSERT (len <= f->shr->size); f_load_head_tail_cons (f, &head, &tail); - if (!f->head_chunk) - f->head_chunk = svm_fifo_find_chunk (f, head); + if (!f->shr->head_chunk) + f->shr->head_chunk = svm_fifo_find_chunk (f, head); - c = f->head_chunk; + c = f->shr->head_chunk; head_idx = head - c->start_byte; n_chunk = c->length - head_idx; if (len <= n_chunk) @@ -804,17 +806,17 @@ f_try_chunk_alloc (svm_fifo_t * f, u32 head, u32 tail, u32 len) svm_fifo_chunk_t *c, *cur, *prev; u32 alloc_size, free_alloced; - free_alloced = f_chunk_end (f->end_chunk) - tail; + free_alloced = f_chunk_end (f->shr->end_chunk) - tail; - alloc_size = clib_min (f->min_alloc, f->size - (tail - head)); + alloc_size = clib_min (f->shr->min_alloc, f->shr->size - (tail - head)); alloc_size = clib_max (alloc_size, len - free_alloced); - c = fsh_alloc_chunk (f->fs_hdr, f->slice_index, alloc_size); + c = fsh_alloc_chunk (f->fs_hdr, f->shr->slice_index, alloc_size); if (PREDICT_FALSE (!c)) return -1; cur = c; - prev = f->end_chunk; + prev = f->shr->end_chunk; while (cur) { @@ -827,11 +829,11 @@ f_try_chunk_alloc (svm_fifo_t * f, u32 head, u32 tail, u32 len) } prev->next = 0; - f->end_chunk->next = c; - f->end_chunk = prev; + f->shr->end_chunk->next = c; + f->shr->end_chunk = prev; - if (!f->tail_chunk) - f->tail_chunk = c; + if (!f->shr->tail_chunk) + f->shr->tail_chunk = c; return 0; } @@ -855,19 +857,20 @@ svm_fifo_enqueue (svm_fifo_t * f, u32 len, const u8 * src) /* number of bytes we're going to copy */ len = clib_min (free_count, len); - if (f_pos_gt (tail + len, f_chunk_end (f->end_chunk))) + if (f_pos_gt (tail + len, f_chunk_end (f->shr->end_chunk))) { if (PREDICT_FALSE (f_try_chunk_alloc (f, head, tail, len))) { - len = f_chunk_end (f->end_chunk) - tail; + len = f_chunk_end (f->shr->end_chunk) - tail; if (!len) return SVM_FIFO_EGROW; } } - old_tail_c = f->tail_chunk; + old_tail_c = f->shr->tail_chunk; - svm_fifo_copy_to_chunk (f, f->tail_chunk, tail, src, len, &f->tail_chunk); + svm_fifo_copy_to_chunk (f, f->shr->tail_chunk, tail, src, len, + &f->shr->tail_chunk); tail = tail + len; svm_fifo_trace_add (f, head, len, 2); @@ -877,12 +880,12 @@ svm_fifo_enqueue (svm_fifo_t * f, u32 len, const u8 * src) { len += ooo_segment_try_collect (f, len, &tail); /* Tail chunk might've changed even if nothing was collected */ - f->tail_chunk = f_lookup_clear_enq_chunks (f, old_tail_c, tail); + f->shr->tail_chunk = f_lookup_clear_enq_chunks (f, old_tail_c, tail); f->ooo_enq = 0; } /* store-rel: producer owned index (paired with load-acq in consumer) */ - clib_atomic_store_rel_n (&f->tail, tail); + clib_atomic_store_rel_n (&f->shr->tail, tail); return len; } @@ -911,7 +914,7 @@ svm_fifo_enqueue_with_offset (svm_fifo_t * f, u32 offset, u32 len, u8 * src) enq_pos = tail + offset; - if (f_pos_gt (enq_pos + len, f_chunk_end (f->end_chunk))) + if (f_pos_gt (enq_pos + len, f_chunk_end (f->shr->end_chunk))) { if (PREDICT_FALSE (f_try_chunk_alloc (f, head, tail, offset + len))) return SVM_FIFO_EGROW; @@ -938,21 +941,23 @@ svm_fifo_enqueue_nocopy (svm_fifo_t * f, u32 len) ASSERT (len <= svm_fifo_max_enqueue_prod (f)); /* load-relaxed: producer owned index */ - tail = f->tail; + tail = f->shr->tail; tail = tail + len; if (rb_tree_is_init (&f->ooo_enq_lookup)) { - f->tail_chunk = f_lookup_clear_enq_chunks (f, f->tail_chunk, tail); + f->shr->tail_chunk = + f_lookup_clear_enq_chunks (f, f->shr->tail_chunk, tail); f->ooo_enq = 0; } else { - f->tail_chunk = svm_fifo_find_next_chunk (f, f->tail_chunk, tail); + f->shr->tail_chunk = + svm_fifo_find_next_chunk (f, f->shr->tail_chunk, tail); } /* store-rel: producer owned index (paired with load-acq in consumer) */ - clib_atomic_store_rel_n (&f->tail, tail); + clib_atomic_store_rel_n (&f->shr->tail, tail); } int @@ -975,14 +980,14 @@ svm_fifo_enqueue_segments (svm_fifo_t * f, const svm_fifo_seg_t segs[], for (i = 0; i < n_segs; i++) len += segs[i].len; - old_tail_c = f->tail_chunk; + old_tail_c = f->shr->tail_chunk; if (!allow_partial) { if (PREDICT_FALSE (free_count < len)) return SVM_FIFO_EFULL; - if (f_pos_gt (tail + len, f_chunk_end (f->end_chunk))) + if (f_pos_gt (tail + len, f_chunk_end (f->shr->end_chunk))) { if (PREDICT_FALSE (f_try_chunk_alloc (f, head, tail, len))) return SVM_FIFO_EGROW; @@ -990,8 +995,8 @@ svm_fifo_enqueue_segments (svm_fifo_t * f, const svm_fifo_seg_t segs[], for (i = 0; i < n_segs; i++) { - svm_fifo_copy_to_chunk (f, f->tail_chunk, tail, segs[i].data, - segs[i].len, &f->tail_chunk); + svm_fifo_copy_to_chunk (f, f->shr->tail_chunk, tail, segs[i].data, + segs[i].len, &f->shr->tail_chunk); tail += segs[i].len; } } @@ -999,11 +1004,11 @@ svm_fifo_enqueue_segments (svm_fifo_t * f, const svm_fifo_seg_t segs[], { len = clib_min (free_count, len); - if (f_pos_gt (tail + len, f_chunk_end (f->end_chunk))) + if (f_pos_gt (tail + len, f_chunk_end (f->shr->end_chunk))) { if (PREDICT_FALSE (f_try_chunk_alloc (f, head, tail, len))) { - len = f_chunk_end (f->end_chunk) - tail; + len = f_chunk_end (f->shr->end_chunk) - tail; if (!len) return SVM_FIFO_EGROW; } @@ -1013,8 +1018,8 @@ svm_fifo_enqueue_segments (svm_fifo_t * f, const svm_fifo_seg_t segs[], while (len) { u32 to_copy = clib_min (segs[i].len, len); - svm_fifo_copy_to_chunk (f, f->tail_chunk, tail, segs[i].data, - to_copy, &f->tail_chunk); + svm_fifo_copy_to_chunk (f, f->shr->tail_chunk, tail, segs[i].data, + to_copy, &f->shr->tail_chunk); len -= to_copy; tail += to_copy; i++; @@ -1026,12 +1031,12 @@ svm_fifo_enqueue_segments (svm_fifo_t * f, const svm_fifo_seg_t segs[], { len += ooo_segment_try_collect (f, len, &tail); /* Tail chunk might've changed even if nothing was collected */ - f->tail_chunk = f_lookup_clear_enq_chunks (f, old_tail_c, tail); + f->shr->tail_chunk = f_lookup_clear_enq_chunks (f, old_tail_c, tail); f->ooo_enq = 0; } /* store-rel: producer owned index (paired with load-acq in consumer) */ - clib_atomic_store_rel_n (&f->tail, tail); + clib_atomic_store_rel_n (&f->shr->tail, tail); return len; } @@ -1043,12 +1048,12 @@ f_unlink_chunks (svm_fifo_t * f, u32 end_pos, u8 maybe_ooo) rb_tree_t *rt; rb_node_t *n; - ASSERT (!f_chunk_includes_pos (f->start_chunk, end_pos)); + ASSERT (!f_chunk_includes_pos (f->shr->start_chunk, end_pos)); if (maybe_ooo) rt = &f->ooo_deq_lookup; - c = f->start_chunk; + c = f->shr->start_chunk; do { @@ -1082,8 +1087,8 @@ f_unlink_chunks (svm_fifo_t * f, u32 end_pos, u8 maybe_ooo) return 0; prev->next = 0; - start = f->start_chunk; - f->start_chunk = c; + start = f->shr->start_chunk; + f->shr->start_chunk = c; return start; } @@ -1103,22 +1108,23 @@ svm_fifo_dequeue (svm_fifo_t * f, u32 len, u8 * dst) len = clib_min (cursize, len); - if (!f->head_chunk) - f->head_chunk = svm_fifo_find_chunk (f, head); + if (!f->shr->head_chunk) + f->shr->head_chunk = svm_fifo_find_chunk (f, head); - svm_fifo_copy_from_chunk (f, f->head_chunk, head, dst, len, &f->head_chunk); + svm_fifo_copy_from_chunk (f, f->shr->head_chunk, head, dst, len, + &f->shr->head_chunk); head = head + len; /* In order dequeues are not supported in combination with ooo peeking. * Use svm_fifo_dequeue_drop instead. */ ASSERT (rb_tree_n_nodes (&f->ooo_deq_lookup) <= 1); - if (f_pos_geq (head, f_chunk_end (f->start_chunk))) - fsh_collect_chunks (f->fs_hdr, f->slice_index, + if (f_pos_geq (head, f_chunk_end (f->shr->start_chunk))) + fsh_collect_chunks (f->fs_hdr, f->shr->slice_index, f_unlink_chunks (f, head, 0)); /* store-rel: consumer owned index (paired with load-acq in producer) */ - clib_atomic_store_rel_n (&f->head, head); + clib_atomic_store_rel_n (&f->shr->head, head); return len; } @@ -1167,16 +1173,17 @@ svm_fifo_dequeue_drop (svm_fifo_t * f, u32 len) /* move head */ head = head + total_drop_bytes; - if (f_pos_geq (head, f_chunk_end (f->start_chunk))) + if (f_pos_geq (head, f_chunk_end (f->shr->start_chunk))) { - fsh_collect_chunks (f->fs_hdr, f->slice_index, + fsh_collect_chunks (f->fs_hdr, f->shr->slice_index, f_unlink_chunks (f, head, 1)); - f->head_chunk = - f_chunk_includes_pos (f->start_chunk, head) ? f->start_chunk : 0; + f->shr->head_chunk = f_chunk_includes_pos (f->shr->start_chunk, head) ? + f->shr->start_chunk : + 0; } /* store-rel: consumer owned index (paired with load-acq in producer) */ - clib_atomic_store_rel_n (&f->head, head); + clib_atomic_store_rel_n (&f->shr->head, head); return total_drop_bytes; } @@ -1192,17 +1199,17 @@ svm_fifo_dequeue_drop_all (svm_fifo_t * f) f_load_head_tail_all_acq (f, &head, &tail); - if (!f->head_chunk || !f_chunk_includes_pos (f->head_chunk, head)) - f->head_chunk = svm_fifo_find_chunk (f, head); + if (!f->shr->head_chunk || !f_chunk_includes_pos (f->shr->head_chunk, head)) + f->shr->head_chunk = svm_fifo_find_chunk (f, head); - f->head_chunk = f_lookup_clear_deq_chunks (f, f->head_chunk, tail); + f->shr->head_chunk = f_lookup_clear_deq_chunks (f, f->shr->head_chunk, tail); - if (f_pos_geq (tail, f_chunk_end (f->start_chunk))) - fsh_collect_chunks (f->fs_hdr, f->slice_index, + if (f_pos_geq (tail, f_chunk_end (f->shr->start_chunk))) + fsh_collect_chunks (f->fs_hdr, f->shr->slice_index, f_unlink_chunks (f, tail, 0)); /* store-rel: consumer owned index (paired with load-acq in producer) */ - clib_atomic_store_rel_n (&f->head, tail); + clib_atomic_store_rel_n (&f->shr->head, tail); } int @@ -1212,10 +1219,10 @@ svm_fifo_fill_chunk_list (svm_fifo_t * f) f_load_head_tail_prod (f, &head, &tail); - if (f_chunk_end (f->end_chunk) - head >= f->size) + if (f_chunk_end (f->shr->end_chunk) - head >= f->shr->size) return 0; - if (f_try_chunk_alloc (f, head, tail, f->size - (tail - head))) + if (f_try_chunk_alloc (f, head, tail, f->shr->size - (tail - head))) return SVM_FIFO_EGROW; return 0; @@ -1233,12 +1240,12 @@ svm_fifo_provision_chunks (svm_fifo_t *f, svm_fifo_seg_t *fs, u32 n_segs, if (f_free_count (f, head, tail) < len) return SVM_FIFO_EFULL; - n_avail = f_chunk_end (f->end_chunk) - tail; + n_avail = f_chunk_end (f->shr->end_chunk) - tail; if (n_avail < len && f_try_chunk_alloc (f, head, tail, len)) return SVM_FIFO_EGROW; - c = f->tail_chunk; + c = f->shr->tail_chunk; head_pos = (tail - c->start_byte); fs[0].data = c->data + head_pos; fs[0].len = clib_min (c->length - head_pos, len); @@ -1279,10 +1286,10 @@ svm_fifo_segments (svm_fifo_t * f, u32 offset, svm_fifo_seg_t * fs, to_read = clib_min (cursize - offset, max_bytes); start = head + offset; - if (!f->head_chunk) - f->head_chunk = svm_fifo_find_chunk (f, head); + if (!f->shr->head_chunk) + f->shr->head_chunk = svm_fifo_find_chunk (f, head); - c = f->head_chunk; + c = f->shr->head_chunk; while (!f_chunk_includes_pos (c, start)) c = c->next; @@ -1320,11 +1327,12 @@ svm_fifo_clone (svm_fifo_t * df, svm_fifo_t * sf) /* Support only single chunk clones for now */ ASSERT (svm_fifo_n_chunks (sf) == 1); - clib_memcpy_fast (df->head_chunk->data, sf->head_chunk->data, sf->size); + clib_memcpy_fast (df->shr->head_chunk->data, sf->shr->head_chunk->data, + sf->shr->size); f_load_head_tail_all_acq (sf, &head, &tail); - clib_atomic_store_rel_n (&df->head, head); - clib_atomic_store_rel_n (&df->tail, tail); + clib_atomic_store_rel_n (&df->shr->head, head); + clib_atomic_store_rel_n (&df->shr->tail, tail); } u32 @@ -1347,23 +1355,23 @@ svm_fifo_init_pointers (svm_fifo_t * f, u32 head, u32 tail) { svm_fifo_chunk_t *c; - clib_atomic_store_rel_n (&f->head, head); - clib_atomic_store_rel_n (&f->tail, tail); + clib_atomic_store_rel_n (&f->shr->head, head); + clib_atomic_store_rel_n (&f->shr->tail, tail); c = svm_fifo_find_chunk (f, head); ASSERT (c != 0); - f->head_chunk = f->ooo_deq = c; + f->shr->head_chunk = f->ooo_deq = c; c = svm_fifo_find_chunk (f, tail); ASSERT (c != 0); - f->tail_chunk = f->ooo_enq = c; + f->shr->tail_chunk = f->ooo_enq = c; } void svm_fifo_add_subscriber (svm_fifo_t * f, u8 subscriber) { - if (f->n_subscribers >= SVM_FIFO_MAX_EVT_SUBSCRIBERS) + if (f->shr->n_subscribers >= SVM_FIFO_MAX_EVT_SUBSCRIBERS) return; - f->subscribers[f->n_subscribers++] = subscriber; + f->shr->subscribers[f->shr->n_subscribers++] = subscriber; } void @@ -1371,12 +1379,12 @@ svm_fifo_del_subscriber (svm_fifo_t * f, u8 subscriber) { int i; - for (i = 0; i < f->n_subscribers; i++) + for (i = 0; i < f->shr->n_subscribers; i++) { - if (f->subscribers[i] != subscriber) + if (f->shr->subscribers[i] != subscriber) continue; - f->subscribers[i] = f->subscribers[f->n_subscribers - 1]; - f->n_subscribers--; + f->shr->subscribers[i] = f->shr->subscribers[f->shr->n_subscribers - 1]; + f->shr->n_subscribers--; break; } } @@ -1386,17 +1394,20 @@ svm_fifo_is_sane (svm_fifo_t * f) { svm_fifo_chunk_t *tmp; - if (f->head_chunk && !f_chunk_includes_pos (f->head_chunk, f->head)) + if (f->shr->head_chunk && + !f_chunk_includes_pos (f->shr->head_chunk, f->shr->head)) return 0; - if (f->tail_chunk && !f_chunk_includes_pos (f->tail_chunk, f->tail)) + if (f->shr->tail_chunk && + !f_chunk_includes_pos (f->shr->tail_chunk, f->shr->tail)) return 0; if (f->ooo_deq) { if (rb_tree_is_init (&f->ooo_deq_lookup)) { - if (f_pos_lt (f->ooo_deq->start_byte, f->start_chunk->start_byte) - || f_pos_gt (f->ooo_deq->start_byte, - f_chunk_end (f->end_chunk))) + if (f_pos_lt (f->ooo_deq->start_byte, + f->shr->start_chunk->start_byte) || + f_pos_gt (f->ooo_deq->start_byte, + f_chunk_end (f->shr->end_chunk))) return 0; tmp = f_find_chunk_rbtree (&f->ooo_deq_lookup, @@ -1411,9 +1422,10 @@ svm_fifo_is_sane (svm_fifo_t * f) { if (rb_tree_is_init (&f->ooo_enq_lookup)) { - if (f_pos_lt (f->ooo_enq->start_byte, f->start_chunk->start_byte) - || f_pos_gt (f->ooo_enq->start_byte, - f_chunk_end (f->end_chunk))) + if (f_pos_lt (f->ooo_enq->start_byte, + f->shr->start_chunk->start_byte) || + f_pos_gt (f->ooo_enq->start_byte, + f_chunk_end (f->shr->end_chunk))) return 0; tmp = f_find_chunk_rbtree (&f->ooo_enq_lookup, @@ -1421,19 +1433,19 @@ svm_fifo_is_sane (svm_fifo_t * f) } else { - tmp = svm_fifo_find_next_chunk (f, f->tail_chunk, + tmp = svm_fifo_find_next_chunk (f, f->shr->tail_chunk, f->ooo_enq->start_byte); } if (tmp != f->ooo_enq) return 0; } - if (f->start_chunk->next) + if (f->shr->start_chunk->next) { svm_fifo_chunk_t *c, *prev = 0, *tmp; u32 chunks_bytes = 0; - c = f->start_chunk; + c = f->shr->start_chunk; do { tmp = svm_fifo_find_chunk (f, c->start_byte); @@ -1467,7 +1479,7 @@ svm_fifo_is_sane (svm_fifo_t * f) } while (c); - if (chunks_bytes < f->tail - f->head) + if (chunks_bytes < f->shr->tail - f->shr->head) return 0; } @@ -1480,7 +1492,7 @@ svm_fifo_n_chunks (svm_fifo_t * f) svm_fifo_chunk_t *c; int n_chunks = 0; - c = f->start_chunk; + c = f->shr->start_chunk; while (c) { n_chunks++; @@ -1544,10 +1556,10 @@ svm_fifo_replay (u8 * s, svm_fifo_t * f, u8 no_read, u8 verbose) trace_len = 0; #endif - placeholder_fifo = svm_fifo_alloc (f->size); - svm_fifo_init (f, f->size); - clib_memset (f->head_chunk->data, 0xFF, f->size); - vec_validate (data, f->size); + placeholder_fifo = svm_fifo_alloc (f->shr->size); + svm_fifo_init (f, f->shr->size); + clib_memset (f->shr->head_chunk->data, 0xFF, f->shr->size); + vec_validate (data, f->shr->size); for (i = 0; i < vec_len (data); i++) data[i] = i; @@ -1614,14 +1626,15 @@ format_svm_fifo (u8 * s, va_list * args) indent = format_get_indent (s); s = format (s, "cursize %u nitems %u has_event %d min_alloc %u\n", - svm_fifo_max_dequeue (f), f->size, f->has_event, f->min_alloc); + svm_fifo_max_dequeue (f), f->shr->size, f->shr->has_event, + f->shr->min_alloc); s = format (s, "%Uhead %u tail %u segment manager %u\n", format_white_space, - indent, f->head, f->tail, f->segment_manager); + indent, f->shr->head, f->shr->tail, f->segment_manager); if (verbose > 1) s = format (s, "%Uvpp session %d thread %d app session %d thread %d\n", - format_white_space, indent, f->master_session_index, - f->master_thread_index, f->client_session_index, + format_white_space, indent, f->shr->master_session_index, + f->master_thread_index, f->shr->client_session_index, f->client_thread_index); if (verbose) diff --git a/src/svm/svm_fifo.h b/src/svm/svm_fifo.h index 5656e63d998..5845d7042af 100644 --- a/src/svm/svm_fifo.h +++ b/src/svm/svm_fifo.h @@ -80,9 +80,9 @@ static inline void f_load_head_tail_cons (svm_fifo_t * f, u32 * head, u32 * tail) { /* load-relaxed: consumer owned index */ - *head = f->head; + *head = f->shr->head; /* load-acq: consumer foreign index (paired with store-rel in producer) */ - *tail = clib_atomic_load_acq_n (&f->tail); + *tail = clib_atomic_load_acq_n (&f->shr->tail); } /** Load head and tail optimized for producer @@ -93,9 +93,9 @@ static inline void f_load_head_tail_prod (svm_fifo_t * f, u32 * head, u32 * tail) { /* load relaxed: producer owned index */ - *tail = f->tail; + *tail = f->shr->tail; /* load-acq: producer foreign index (paired with store-rel in consumer) */ - *head = clib_atomic_load_acq_n (&f->head); + *head = clib_atomic_load_acq_n (&f->shr->head); } /** @@ -107,9 +107,9 @@ static inline void f_load_head_tail_all_acq (svm_fifo_t * f, u32 * head, u32 * tail) { /* load-acq : consumer foreign index (paired with store-rel) */ - *tail = clib_atomic_load_acq_n (&f->tail); + *tail = clib_atomic_load_acq_n (&f->shr->tail); /* load-acq : producer foriegn index (paired with store-rel) */ - *head = clib_atomic_load_acq_n (&f->head); + *head = clib_atomic_load_acq_n (&f->shr->head); } /** @@ -131,7 +131,7 @@ f_cursize (svm_fifo_t * f, u32 head, u32 tail) static inline u32 f_free_count (svm_fifo_t * f, u32 head, u32 tail) { - return (f->size - f_cursize (f, head, tail)); + return (f->shr->size - f_cursize (f, head, tail)); } always_inline u32 @@ -487,7 +487,7 @@ svm_fifo_max_dequeue (svm_fifo_t * f) static inline int svm_fifo_is_full_prod (svm_fifo_t * f) { - return (svm_fifo_max_dequeue_prod (f) == f->size); + return (svm_fifo_max_dequeue_prod (f) == f->shr->size); } /* Check if fifo is full. @@ -499,7 +499,7 @@ svm_fifo_is_full_prod (svm_fifo_t * f) static inline int svm_fifo_is_full (svm_fifo_t * f) { - return (svm_fifo_max_dequeue (f) == f->size); + return (svm_fifo_max_dequeue (f) == f->shr->size); } /** @@ -606,7 +606,7 @@ u32 svm_fifo_max_write_chunk (svm_fifo_t * f); static inline svm_fifo_chunk_t * svm_fifo_head_chunk (svm_fifo_t * f) { - return f->head_chunk; + return f->shr->head_chunk; } /** @@ -618,10 +618,11 @@ svm_fifo_head_chunk (svm_fifo_t * f) static inline u8 * svm_fifo_head (svm_fifo_t * f) { - if (!f->head_chunk) + if (!f->shr->head_chunk) return 0; /* load-relaxed: consumer owned index */ - return (f->head_chunk->data + (f->head - f->head_chunk->start_byte)); + return (f->shr->head_chunk->data + + (f->shr->head - f->shr->head_chunk->start_byte)); } /** @@ -633,7 +634,7 @@ svm_fifo_head (svm_fifo_t * f) static inline svm_fifo_chunk_t * svm_fifo_tail_chunk (svm_fifo_t * f) { - return f->tail_chunk; + return f->shr->tail_chunk; } /** @@ -646,7 +647,8 @@ static inline u8 * svm_fifo_tail (svm_fifo_t * f) { /* load-relaxed: producer owned index */ - return (f->tail_chunk->data + (f->tail - f->tail_chunk->start_byte)); + return (f->shr->tail_chunk->data + + (f->shr->tail - f->shr->tail_chunk->start_byte)); } /** @@ -658,7 +660,7 @@ svm_fifo_tail (svm_fifo_t * f) static inline u8 svm_fifo_n_subscribers (svm_fifo_t * f) { - return f->n_subscribers; + return f->shr->n_subscribers; } /** @@ -692,7 +694,7 @@ ooo_segment_offset_prod (svm_fifo_t * f, ooo_segment_t * s) { u32 tail; /* load-relaxed: producer owned index */ - tail = f->tail; + tail = f->shr->tail; return (s->start - tail); } @@ -706,7 +708,7 @@ ooo_segment_length (svm_fifo_t * f, ooo_segment_t * s) static inline u32 svm_fifo_size (svm_fifo_t * f) { - return f->size; + return f->shr->size; } static inline void @@ -714,8 +716,9 @@ svm_fifo_set_size (svm_fifo_t * f, u32 size) { if (size > (1 << f->fs_hdr->max_log2_fifo_size)) return; - fsh_virtual_mem_update (f->fs_hdr, f->slice_index, (int) f->size - size); - f->size = size; + fsh_virtual_mem_update (f->fs_hdr, f->shr->slice_index, + (int) f->shr->size - size); + f->shr->size = size; } /** @@ -727,7 +730,7 @@ svm_fifo_set_size (svm_fifo_t * f, u32 size) static inline int svm_fifo_has_event (svm_fifo_t * f) { - return f->has_event; + return f->shr->has_event; } /** @@ -741,7 +744,7 @@ svm_fifo_has_event (svm_fifo_t * f) always_inline u8 svm_fifo_set_event (svm_fifo_t * f) { - return !clib_atomic_swap_rel_n (&f->has_event, 1); + return !clib_atomic_swap_rel_n (&f->shr->has_event, 1); } /** @@ -754,7 +757,7 @@ svm_fifo_set_event (svm_fifo_t * f) always_inline void svm_fifo_unset_event (svm_fifo_t * f) { - clib_atomic_swap_acq_n (&f->has_event, 0); + clib_atomic_swap_acq_n (&f->shr->has_event, 0); } /** @@ -768,7 +771,7 @@ svm_fifo_unset_event (svm_fifo_t * f) static inline void svm_fifo_add_want_deq_ntf (svm_fifo_t * f, u8 ntf_type) { - f->want_deq_ntf |= ntf_type; + f->shr->want_deq_ntf |= ntf_type; } /** @@ -782,7 +785,7 @@ svm_fifo_add_want_deq_ntf (svm_fifo_t * f, u8 ntf_type) static inline void svm_fifo_del_want_deq_ntf (svm_fifo_t * f, u8 ntf_type) { - f->want_deq_ntf &= ~ntf_type; + f->shr->want_deq_ntf &= ~ntf_type; } /** @@ -800,7 +803,8 @@ static inline void svm_fifo_clear_deq_ntf (svm_fifo_t * f) { /* Set the flag if want_notif_if_full was the only ntf requested */ - f->has_deq_ntf = f->want_deq_ntf == SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL; + f->shr->has_deq_ntf = + f->shr->want_deq_ntf == SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL; svm_fifo_del_want_deq_ntf (f, SVM_FIFO_WANT_DEQ_NOTIF); } @@ -816,7 +820,7 @@ svm_fifo_clear_deq_ntf (svm_fifo_t * f) static inline void svm_fifo_reset_has_deq_ntf (svm_fifo_t * f) { - f->has_deq_ntf = 0; + f->shr->has_deq_ntf = 0; } /** @@ -832,7 +836,7 @@ svm_fifo_reset_has_deq_ntf (svm_fifo_t * f) static inline u8 svm_fifo_needs_deq_ntf (svm_fifo_t * f, u32 n_last_deq) { - u8 want_ntf = f->want_deq_ntf; + u8 want_ntf = f->shr->want_deq_ntf; if (PREDICT_TRUE (want_ntf == SVM_FIFO_NO_DEQ_NOTIF)) return 0; @@ -841,13 +845,14 @@ svm_fifo_needs_deq_ntf (svm_fifo_t * f, u32 n_last_deq) if (want_ntf & SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL) { u32 max_deq = svm_fifo_max_dequeue_cons (f); - u32 size = f->size; - if (!f->has_deq_ntf && max_deq < size && max_deq + n_last_deq >= size) + u32 size = f->shr->size; + if (!f->shr->has_deq_ntf && max_deq < size && + max_deq + n_last_deq >= size) return 1; } if (want_ntf & SVM_FIFO_WANT_DEQ_NOTIF_IF_EMPTY) { - if (!f->has_deq_ntf && svm_fifo_is_empty (f)) + if (!f->shr->has_deq_ntf && svm_fifo_is_empty (f)) return 1; } return 0; diff --git a/src/vcl/vcl_private.c b/src/vcl/vcl_private.c index c5dcd39038e..ea938113615 100644 --- a/src/vcl/vcl_private.c +++ b/src/vcl/vcl_private.c @@ -374,6 +374,51 @@ vcl_segment_detach (u64 segment_handle) VDBG (0, "detached segment %u handle %u", segment_index, segment_handle); } +int +vcl_segment_attach_session (uword segment_handle, uword rxf_offset, + uword txf_offset, u8 is_ct, vcl_session_t *s) +{ + svm_fifo_shared_t *rxsf, *txsf; + svm_fifo_t *rxf, *txf; + fifo_segment_t *fs; + u32 fs_index; + + fs_index = vcl_segment_table_lookup (segment_handle); + if (fs_index == VCL_INVALID_SEGMENT_INDEX) + { + VDBG (0, "ERROR: segment for session %u is not mounted!", + s->session_index); + return -1; + } + + rxsf = uword_to_pointer (rxf_offset, svm_fifo_shared_t *); + txsf = uword_to_pointer (txf_offset, svm_fifo_shared_t *); + + clib_rwlock_reader_lock (&vcm->segment_table_lock); + + fs = fifo_segment_get_segment (&vcm->segment_main, fs_index); + rxf = fifo_segment_alloc_fifo_w_shared (fs, rxsf); + txf = fifo_segment_alloc_fifo_w_shared (fs, txsf); + + clib_rwlock_reader_unlock (&vcm->segment_table_lock); + + if (!is_ct) + { + rxsf->client_session_index = s->session_index; + txsf->client_session_index = s->session_index; + rxf->client_thread_index = vcl_get_worker_index (); + txf->client_thread_index = vcl_get_worker_index (); + s->rx_fifo = rxf; + s->tx_fifo = txf; + } + else + { + s->ct_rx_fifo = rxf; + s->ct_tx_fifo = txf; + } + + return 0; +} /* * fd.io coding-style-patch-verification: ON diff --git a/src/vcl/vcl_private.h b/src/vcl/vcl_private.h index 0d26e64917c..637581b1b4b 100644 --- a/src/vcl/vcl_private.h +++ b/src/vcl/vcl_private.h @@ -685,6 +685,9 @@ int vcl_segment_attach (u64 segment_handle, char *name, void vcl_segment_detach (u64 segment_handle); void vcl_send_session_unlisten (vcl_worker_t * wrk, vcl_session_t * s); +int vcl_segment_attach_session (uword segment_handle, uword rxf_offset, + uword txf_offset, u8 is_ct, vcl_session_t *s); + /* * VCL Binary API */ diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c index 58c13c28763..734d062669e 100644 --- a/src/vcl/vppcom.c +++ b/src/vcl/vppcom.c @@ -22,21 +22,6 @@ __thread uword __vcl_worker_index = ~0; -static int -vcl_segment_is_not_mounted (vcl_worker_t * wrk, u64 segment_handle) -{ - u32 segment_index; - - if (segment_handle == VCL_INVALID_SEGMENT_HANDLE) - return 0; - - segment_index = vcl_segment_table_lookup (segment_handle); - if (segment_index != VCL_INVALID_SEGMENT_INDEX) - return 0; - - return 1; -} - static inline int vcl_mq_dequeue_batch (vcl_worker_t * wrk, svm_msg_q_t * mq, u32 n_max_msg) { @@ -377,7 +362,6 @@ vcl_session_accepted_handler (vcl_worker_t * wrk, session_accepted_msg_t * mp, u32 ls_index) { vcl_session_t *session, *listen_session; - svm_fifo_t *rx_fifo, *tx_fifo; svm_msg_q_t *evt_q; session = vcl_session_alloc (wrk); @@ -390,26 +374,17 @@ vcl_session_accepted_handler (vcl_worker_t * wrk, session_accepted_msg_t * mp, goto error; } - if (vcl_segment_is_not_mounted (wrk, mp->segment_handle)) + session->vpp_evt_q = + uword_to_pointer (mp->vpp_event_queue_address, svm_msg_q_t *); + + if (vcl_segment_attach_session (mp->segment_handle, mp->server_rx_fifo, + mp->server_tx_fifo, 0, session)) { - VDBG (0, "ERROR: segment for session %u is not mounted!", - session->session_index); + VDBG (0, "failed to attach fifos for %u", session->session_index); goto error; } - rx_fifo = uword_to_pointer (mp->server_rx_fifo, svm_fifo_t *); - tx_fifo = uword_to_pointer (mp->server_tx_fifo, svm_fifo_t *); - session->vpp_evt_q = uword_to_pointer (mp->vpp_event_queue_address, - svm_msg_q_t *); - rx_fifo->client_session_index = session->session_index; - tx_fifo->client_session_index = session->session_index; - rx_fifo->client_thread_index = vcl_get_worker_index (); - tx_fifo->client_thread_index = vcl_get_worker_index (); - session->vpp_handle = mp->handle; - session->rx_fifo = rx_fifo; - session->tx_fifo = tx_fifo; - session->session_state = VCL_STATE_READY; session->transport.rmt_port = mp->rmt.port; session->transport.is_ip4 = mp->rmt.is_ip4; @@ -448,7 +423,6 @@ static u32 vcl_session_connected_handler (vcl_worker_t * wrk, session_connected_msg_t * mp) { - svm_fifo_t *rx_fifo, *tx_fifo; vcl_session_t *session = 0; u32 session_index; @@ -472,38 +446,28 @@ vcl_session_connected_handler (vcl_worker_t * wrk, session->vpp_handle = mp->handle; session->vpp_evt_q = uword_to_pointer (mp->vpp_event_queue_address, svm_msg_q_t *); - rx_fifo = uword_to_pointer (mp->server_rx_fifo, svm_fifo_t *); - tx_fifo = uword_to_pointer (mp->server_tx_fifo, svm_fifo_t *); - if (vcl_segment_is_not_mounted (wrk, mp->segment_handle)) + + if (vcl_segment_attach_session (mp->segment_handle, mp->server_rx_fifo, + mp->server_tx_fifo, 0, session)) { - VDBG (0, "segment for session %u is not mounted!", - session->session_index); + VDBG (0, "failed to attach fifos for %u", session->session_index); session->session_state = VCL_STATE_DETACHED; vcl_send_session_disconnect (wrk, session); return session_index; } - rx_fifo->client_session_index = session_index; - tx_fifo->client_session_index = session_index; - rx_fifo->client_thread_index = vcl_get_worker_index (); - tx_fifo->client_thread_index = vcl_get_worker_index (); - if (mp->ct_rx_fifo) { - session->ct_rx_fifo = uword_to_pointer (mp->ct_rx_fifo, svm_fifo_t *); - session->ct_tx_fifo = uword_to_pointer (mp->ct_tx_fifo, svm_fifo_t *); - if (vcl_segment_is_not_mounted (wrk, mp->ct_segment_handle)) + if (vcl_segment_attach_session (mp->ct_segment_handle, mp->ct_rx_fifo, + mp->ct_tx_fifo, 1, session)) { - VDBG (0, "ct segment for session %u is not mounted!", - session->session_index); + VDBG (0, "failed to attach ct fifos for %u", session->session_index); session->session_state = VCL_STATE_DETACHED; vcl_send_session_disconnect (wrk, session); return session_index; } } - session->rx_fifo = rx_fifo; - session->tx_fifo = tx_fifo; session->transport.is_ip4 = mp->lcl.is_ip4; clib_memcpy_fast (&session->transport.lcl_ip, &mp->lcl.ip, sizeof (session->transport.lcl_ip)); @@ -611,14 +575,13 @@ vcl_session_bound_handler (vcl_worker_t * wrk, session_bound_msg_t * mp) if (vcl_session_is_cl (session)) { - svm_fifo_t *rx_fifo, *tx_fifo; - session->vpp_evt_q = uword_to_pointer (mp->vpp_evt_q, svm_msg_q_t *); - rx_fifo = uword_to_pointer (mp->rx_fifo, svm_fifo_t *); - rx_fifo->client_session_index = sid; - tx_fifo = uword_to_pointer (mp->tx_fifo, svm_fifo_t *); - tx_fifo->client_session_index = sid; - session->rx_fifo = rx_fifo; - session->tx_fifo = tx_fifo; + if (vcl_segment_attach_session (mp->segment_handle, mp->rx_fifo, + mp->tx_fifo, 0, session)) + { + VDBG (0, "failed to attach fifos for %u", session->session_index); + session->session_state = VCL_STATE_DETACHED; + return VCL_INVALID_SESSION_INDEX; + } } VDBG (0, "session %u [0x%llx]: listen succeeded!", sid, mp->handle); @@ -664,6 +627,7 @@ vcl_session_migrated_handler (vcl_worker_t * wrk, void *data) { session_migrated_msg_t *mp = (session_migrated_msg_t *) data; vcl_session_t *s; + u32 fs_index; s = vcl_session_get_w_vpp_handle (wrk, mp->handle); if (!s) @@ -672,6 +636,14 @@ vcl_session_migrated_handler (vcl_worker_t * wrk, void *data) return; } + fs_index = vcl_segment_table_lookup (mp->segment_handle); + if (fs_index == VCL_INVALID_SEGMENT_INDEX) + { + VDBG (0, "segment for session %u is not mounted!", s->session_index); + s->session_state = VCL_STATE_DETACHED; + return; + } + s->vpp_handle = mp->new_handle; s->vpp_evt_q = uword_to_pointer (mp->vpp_evt_q, svm_msg_q_t *); @@ -680,7 +652,8 @@ vcl_session_migrated_handler (vcl_worker_t * wrk, void *data) /* Generate new tx event if we have outstanding data */ if (svm_fifo_has_event (s->tx_fifo)) - app_send_io_evt_to_vpp (s->vpp_evt_q, s->tx_fifo->master_session_index, + app_send_io_evt_to_vpp (s->vpp_evt_q, + s->tx_fifo->shr->master_session_index, SESSION_IO_EVT_TX, SVM_Q_WAIT); VDBG (0, "Migrated 0x%lx to thread %u 0x%lx", mp->handle, @@ -879,21 +852,15 @@ vcl_session_worker_update_reply_handler (vcl_worker_t * wrk, void *data) VDBG (0, "unknown handle 0x%llx", msg->handle); return; } - if (vcl_segment_is_not_mounted (wrk, msg->segment_handle)) - { - clib_warning ("segment for session %u is not mounted!", - s->session_index); - return; - } if (s->rx_fifo) { - s->rx_fifo = uword_to_pointer (msg->rx_fifo, svm_fifo_t *); - s->tx_fifo = uword_to_pointer (msg->tx_fifo, svm_fifo_t *); - s->rx_fifo->client_session_index = s->session_index; - s->tx_fifo->client_session_index = s->session_index; - s->rx_fifo->client_thread_index = wrk->wrk_index; - s->tx_fifo->client_thread_index = wrk->wrk_index; + if (vcl_segment_attach_session (msg->segment_handle, msg->rx_fifo, + msg->tx_fifo, 0, s)) + { + VDBG (0, "failed to attach fifos for %u", s->session_index); + return; + } } s->session_state = VCL_STATE_UPDATED; @@ -1921,7 +1888,8 @@ read_again: if (PREDICT_FALSE (svm_fifo_needs_deq_ntf (rx_fifo, n_read))) { svm_fifo_clear_deq_ntf (rx_fifo); - app_send_io_evt_to_vpp (s->vpp_evt_q, s->rx_fifo->master_session_index, + app_send_io_evt_to_vpp (s->vpp_evt_q, + s->rx_fifo->shr->master_session_index, SESSION_IO_EVT_RX, SVM_Q_WAIT); } @@ -2132,8 +2100,8 @@ vppcom_session_write_inline (vcl_worker_t * wrk, vcl_session_t * s, void *buf, 0 /* do_evt */ , SVM_Q_WAIT); if (svm_fifo_set_event (s->tx_fifo)) - app_send_io_evt_to_vpp (s->vpp_evt_q, s->tx_fifo->master_session_index, - et, SVM_Q_WAIT); + app_send_io_evt_to_vpp ( + s->vpp_evt_q, s->tx_fifo->shr->master_session_index, et, SVM_Q_WAIT); /* The underlying fifo segment can run out of memory */ if (PREDICT_FALSE (n_write < 0)) diff --git a/src/vnet/session/application_interface.h b/src/vnet/session/application_interface.h index 79c6f288e93..096af1efa15 100644 --- a/src/vnet/session/application_interface.h +++ b/src/vnet/session/application_interface.h @@ -343,9 +343,7 @@ typedef struct session_bound_msg_ uword rx_fifo; uword tx_fifo; uword vpp_evt_q; - u32 segment_size; - u8 segment_name_length; - u8 segment_name[128]; + u64 segment_handle; } __clib_packed session_bound_msg_t; typedef struct session_unlisten_msg_ @@ -519,6 +517,7 @@ typedef struct session_migrate_msg_ uword vpp_evt_q; session_handle_t handle; session_handle_t new_handle; + u64 segment_handle; u32 vpp_thread_index; } __clib_packed session_migrated_msg_t; @@ -640,8 +639,8 @@ app_send_dgram_raw (svm_fifo_t * f, app_session_transport_t * at, if (do_evt) { if (svm_fifo_set_event (f)) - app_send_io_evt_to_vpp (vpp_evt_q, f->master_session_index, evt_type, - noblock); + app_send_io_evt_to_vpp (vpp_evt_q, f->shr->master_session_index, + evt_type, noblock); } return len; } @@ -664,8 +663,8 @@ app_send_stream_raw (svm_fifo_t * f, svm_msg_q_t * vpp_evt_q, u8 * data, if (do_evt) { if (rv > 0 && svm_fifo_set_event (f)) - app_send_io_evt_to_vpp (vpp_evt_q, f->master_session_index, evt_type, - noblock); + app_send_io_evt_to_vpp (vpp_evt_q, f->shr->master_session_index, + evt_type, noblock); } return rv; } diff --git a/src/vnet/session/application_local.c b/src/vnet/session/application_local.c index 6e0aee2aae3..18e26f2c1cc 100644 --- a/src/vnet/session/application_local.c +++ b/src/vnet/session/application_local.c @@ -208,8 +208,8 @@ ct_init_accepted_session (app_worker_t * server_wrk, } sm_index = segment_manager_index (sm); - ls->rx_fifo->master_session_index = ls->session_index; - ls->tx_fifo->master_session_index = ls->session_index; + ls->rx_fifo->shr->master_session_index = ls->session_index; + ls->tx_fifo->shr->master_session_index = ls->session_index; ls->rx_fifo->master_thread_index = ls->thread_index; ls->tx_fifo->master_thread_index = ls->thread_index; ls->rx_fifo->segment_manager = sm_index; diff --git a/src/vnet/session/application_worker.c b/src/vnet/session/application_worker.c index dd159c6693c..d44f52c5340 100644 --- a/src/vnet/session/application_worker.c +++ b/src/vnet/session/application_worker.c @@ -200,10 +200,10 @@ app_worker_alloc_session_fifos (segment_manager_t * sm, session_t * s) &rx_fifo, &tx_fifo))) return rv; - rx_fifo->master_session_index = s->session_index; + rx_fifo->shr->master_session_index = s->session_index; rx_fifo->master_thread_index = s->thread_index; - tx_fifo->master_session_index = s->session_index; + tx_fifo->shr->master_session_index = s->session_index; tx_fifo->master_thread_index = s->thread_index; s->rx_fifo = rx_fifo; @@ -711,7 +711,7 @@ app_send_io_evt_rx (app_worker_t * app_wrk, session_t * s) msg = svm_msg_q_alloc_msg_w_ring (mq, SESSION_MQ_IO_EVT_RING); evt = (session_event_t *) svm_msg_q_msg_data (mq, &msg); - evt->session_index = s->rx_fifo->client_session_index; + evt->session_index = s->rx_fifo->shr->client_session_index; evt->event_type = SESSION_IO_EVT_RX; (void) svm_fifo_set_event (s->rx_fifo); @@ -750,7 +750,7 @@ app_send_io_evt_tx (app_worker_t * app_wrk, session_t * s) msg = svm_msg_q_alloc_msg_w_ring (mq, SESSION_MQ_IO_EVT_RING); evt = (session_event_t *) svm_msg_q_msg_data (mq, &msg); evt->event_type = SESSION_IO_EVT_TX; - evt->session_index = s->tx_fifo->client_session_index; + evt->session_index = s->tx_fifo->shr->client_session_index; svm_msg_q_add_and_unlock (mq, &msg); return 0; diff --git a/src/vnet/session/segment_manager.c b/src/vnet/session/segment_manager.c index bb3d7adc379..7683760fde9 100644 --- a/src/vnet/session/segment_manager.c +++ b/src/vnet/session/segment_manager.c @@ -211,6 +211,7 @@ segment_manager_del_segment (segment_manager_t * sm, fifo_segment_t * fs) } } + fifo_segment_cleanup (fs); ssvm_delete (&fs->ssvm); if (CLIB_DEBUG) @@ -612,12 +613,12 @@ segment_manager_del_sessions (segment_manager_t * sm) */ while (f) { - session = session_get_if_valid (f->master_session_index, - f->master_thread_index); - if (session) - vec_add1 (handles, session_handle (session)); - f = f->next; - } + session = session_get_if_valid (f->shr->master_session_index, + f->master_thread_index); + if (session) + vec_add1 (handles, session_handle (session)); + f = f->next; + } } /* Instead of removing the segment, test when cleaning up disconnected @@ -841,7 +842,7 @@ segment_manager_attach_fifo (segment_manager_t * sm, svm_fifo_t * f, fifo_segment_attach_fifo (fs, f, s->thread_index); segment_manager_segment_reader_unlock (sm); - f->master_session_index = s->session_index; + f->shr->master_session_index = s->session_index; f->master_thread_index = s->thread_index; } @@ -1059,25 +1060,26 @@ segment_manager_format_sessions (segment_manager_t * sm, int verbose) u32 session_index, thread_index; session_t *session; - session_index = f->master_session_index; - thread_index = f->master_thread_index; + session_index = f->shr->master_session_index; + thread_index = f->master_thread_index; - session = session_get (session_index, thread_index); - str = format (0, "%U", format_session, session, verbose); + session = session_get (session_index, thread_index); + str = format (0, "%U", format_session, session, verbose); - if (verbose) - s = format (s, "%-40v%-20v%-15u%-10u", str, app_name, - app_wrk->api_client_index, app_wrk->connects_seg_manager); - else - s = format (s, "%-40v%-20v", str, app_name); + if (verbose) + s = format (s, "%-40v%-20v%-15u%-10u", str, app_name, + app_wrk->api_client_index, + app_wrk->connects_seg_manager); + else + s = format (s, "%-40v%-20v", str, app_name); - vlib_cli_output (vm, "%v", s); - vec_reset_length (s); - vec_free (str); + vlib_cli_output (vm, "%v", s); + vec_reset_length (s); + vec_free (str); - f = f->next; - } - vec_free (s); + f = f->next; + } + vec_free (s); } } /* *INDENT-ON* */ diff --git a/src/vnet/session/session.c b/src/vnet/session/session.c index b0a17bbb214..962df556de9 100644 --- a/src/vnet/session/session.c +++ b/src/vnet/session/session.c @@ -78,7 +78,7 @@ session_send_evt_to_thread (void *data, void *args, u32 thread_index, int session_send_io_evt_to_thread (svm_fifo_t * f, session_evt_type_t evt_type) { - return session_send_evt_to_thread (&f->master_session_index, 0, + return session_send_evt_to_thread (&f->shr->master_session_index, 0, f->master_thread_index, evt_type); } @@ -436,8 +436,8 @@ session_fifo_tuning (session_t * s, svm_fifo_t * f, { segment_manager_t *sm; sm = segment_manager_get (f->segment_manager); - ASSERT (f->size >= 4096); - ASSERT (f->size <= sm->max_fifo_size); + ASSERT (f->shr->size >= 4096); + ASSERT (f->shr->size <= sm->max_fifo_size); } } } @@ -611,9 +611,9 @@ session_notify_subscribers (u32 app_index, session_t * s, if (!app) return -1; - for (i = 0; i < f->n_subscribers; i++) + for (i = 0; i < f->shr->n_subscribers; i++) { - app_wrk = application_get_worker (app, f->subscribers[i]); + app_wrk = application_get_worker (app, f->shr->subscribers[i]); if (!app_wrk) continue; if (app_worker_lock_and_send_event (app_wrk, s, evt_type)) @@ -723,7 +723,7 @@ session_dequeue_notify (session_t * s) SESSION_IO_EVT_TX))) return -1; - if (PREDICT_FALSE (s->tx_fifo->n_subscribers)) + if (PREDICT_FALSE (s->tx_fifo->shr->n_subscribers)) return session_notify_subscribers (app_wrk->app_index, s, s->tx_fifo, SESSION_IO_EVT_TX); diff --git a/src/vnet/session/session_api.c b/src/vnet/session/session_api.c index d080ae44925..a8f92c3cc00 100644 --- a/src/vnet/session/session_api.c +++ b/src/vnet/session/session_api.c @@ -147,8 +147,8 @@ mq_send_session_accepted_cb (session_t * s) mp = (session_accepted_msg_t *) evt->data; clib_memset (mp, 0, sizeof (*mp)); mp->context = app->app_index; - mp->server_rx_fifo = pointer_to_uword (s->rx_fifo); - mp->server_tx_fifo = pointer_to_uword (s->tx_fifo); + mp->server_rx_fifo = pointer_to_uword (s->rx_fifo->shr); + mp->server_tx_fifo = pointer_to_uword (s->tx_fifo->shr); mp->segment_handle = session_segment_handle (s); mp->flags = s->flags; @@ -221,9 +221,9 @@ mq_notify_close_subscribers (u32 app_index, session_handle_t sh, if (!app) return; - for (i = 0; i < f->n_subscribers; i++) + for (i = 0; i < f->shr->n_subscribers; i++) { - if (!(app_wrk = application_get_worker (app, f->subscribers[i]))) + if (!(app_wrk = application_get_worker (app, f->shr->subscribers[i]))) continue; mq_send_session_close_evt (app_wrk, sh, SESSION_CTRL_EVT_DISCONNECTED); } @@ -305,8 +305,8 @@ mq_send_session_connected_cb (u32 app_wrk_index, u32 api_context, session_get_endpoint (s, &mp->lcl, 1 /* is_lcl */ ); - mp->server_rx_fifo = pointer_to_uword (s->rx_fifo); - mp->server_tx_fifo = pointer_to_uword (s->tx_fifo); + mp->server_rx_fifo = pointer_to_uword (s->rx_fifo->shr); + mp->server_tx_fifo = pointer_to_uword (s->tx_fifo->shr); mp->segment_handle = session_segment_handle (s); } else @@ -320,12 +320,12 @@ mq_send_session_connected_cb (u32 app_wrk_index, u32 api_context, mp->lcl.is_ip4 = cct->c_is_ip4; vpp_mq = session_main_get_vpp_event_queue (s->thread_index); mp->vpp_event_queue_address = pointer_to_uword (vpp_mq); - mp->server_rx_fifo = pointer_to_uword (s->rx_fifo); - mp->server_tx_fifo = pointer_to_uword (s->tx_fifo); + mp->server_rx_fifo = pointer_to_uword (s->rx_fifo->shr); + mp->server_tx_fifo = pointer_to_uword (s->tx_fifo->shr); mp->segment_handle = session_segment_handle (s); ss = ct_session_get_peer (s); - mp->ct_rx_fifo = pointer_to_uword (ss->tx_fifo); - mp->ct_tx_fifo = pointer_to_uword (ss->rx_fifo); + mp->ct_rx_fifo = pointer_to_uword (ss->tx_fifo->shr); + mp->ct_tx_fifo = pointer_to_uword (ss->rx_fifo->shr); mp->ct_segment_handle = session_segment_handle (ss); } @@ -386,8 +386,9 @@ mq_send_session_bound_cb (u32 app_wrk_index, u32 api_context, if (session_transport_service_type (ls) == TRANSPORT_SERVICE_CL) { - mp->rx_fifo = pointer_to_uword (ls->rx_fifo); - mp->tx_fifo = pointer_to_uword (ls->tx_fifo); + mp->rx_fifo = pointer_to_uword (ls->rx_fifo->shr); + mp->tx_fifo = pointer_to_uword (ls->tx_fifo->shr); + mp->segment_handle = session_segment_handle (ls); } done: @@ -443,6 +444,7 @@ mq_send_session_migrate_cb (session_t * s, session_handle_t new_sh) mp->vpp_thread_index = session_thread_from_handle (new_sh); vpp_evt_q = session_main_get_vpp_event_queue (mp->vpp_thread_index); mp->vpp_evt_q = pointer_to_uword (vpp_evt_q); + mp->segment_handle = session_segment_handle (s); svm_msg_q_add_and_unlock (app_mq, msg); } diff --git a/src/vnet/session/session_cli.c b/src/vnet/session/session_cli.c index a9197547501..0d541748d9e 100644 --- a/src/vnet/session/session_cli.c +++ b/src/vnet/session/session_cli.c @@ -27,14 +27,14 @@ format_session_fifos (u8 * s, va_list * args) return s; s = format (s, " Rx fifo: %U", format_svm_fifo, ss->rx_fifo, verbose); - if (verbose > 2 && ss->rx_fifo->has_event) + if (verbose > 2 && ss->rx_fifo->shr->has_event) { found = session_node_lookup_fifo_event (ss->rx_fifo, e); s = format (s, " session node event: %s\n", found ? "found" : "not found"); } s = format (s, " Tx fifo: %U", format_svm_fifo, ss->tx_fifo, verbose); - if (verbose > 2 && ss->tx_fifo->has_event) + if (verbose > 2 && ss->tx_fifo->shr->has_event) { found = session_node_lookup_fifo_event (ss->tx_fifo, e); s = format (s, " session node event: %s\n", diff --git a/src/vnet/session/session_debug.c b/src/vnet/session/session_debug.c index 33d1dc57196..c2718f3bfc2 100644 --- a/src/vnet/session/session_debug.c +++ b/src/vnet/session/session_debug.c @@ -187,7 +187,7 @@ session_node_cmp_event (session_event_t * e, svm_fifo_t * f) case SESSION_IO_EVT_BUILTIN_RX: case SESSION_IO_EVT_BUILTIN_TX: case SESSION_IO_EVT_TX_FLUSH: - if (e->session_index == f->master_session_index) + if (e->session_index == f->shr->master_session_index) return 1; break; case SESSION_CTRL_EVT_CLOSE: |