summaryrefslogtreecommitdiffstats
path: root/src/vat/api_format.c
diff options
context:
space:
mode:
authorFilip Tehlar <ftehlar@cisco.com>2017-05-17 17:21:10 +0200
committerFilip Tehlar <ftehlar@cisco.com>2017-05-30 07:18:54 +0200
commitd5a65db98d66c66b03b057ac568be05f2456f73c (patch)
tree8c8c5f14cf8402345e30f24c2491f144e96c8f9b /src/vat/api_format.c
parentcd76436097b9ddda76864af33c85aec9bab074a3 (diff)
LISP: L2 ARP handling
Change-Id: I1ec328cda73f7eaf7867cd8a2a17852ee0cd23f1 Signed-off-by: Filip Tehlar <ftehlar@cisco.com>
Diffstat (limited to 'src/vat/api_format.c')
-rw-r--r--src/vat/api_format.c233
1 files changed, 233 insertions, 0 deletions
diff --git a/src/vat/api_format.c b/src/vat/api_format.c
index f3e6f64c57e..3eff8ef0a20 100644
--- a/src/vat/api_format.c
+++ b/src/vat/api_format.c
@@ -3164,6 +3164,123 @@ end:
}
static void
+ vl_api_one_l2_arp_entries_get_reply_t_handler
+ (vl_api_one_l2_arp_entries_get_reply_t * mp)
+{
+ vat_main_t *vam = &vat_main;
+ u32 i, n;
+ int retval = clib_net_to_host_u32 (mp->retval);
+
+ if (retval)
+ goto end;
+
+ n = clib_net_to_host_u32 (mp->count);
+
+ for (i = 0; i < n; i++)
+ print (vam->ofp, "%U -> %U", format_ip4_address, &mp->entries[i].ip4,
+ format_ethernet_address, mp->entries[i].mac);
+
+end:
+ vam->retval = retval;
+ vam->result_ready = 1;
+}
+
+static void
+ vl_api_one_l2_arp_entries_get_reply_t_handler_json
+ (vl_api_one_l2_arp_entries_get_reply_t * mp)
+{
+ u8 *s = 0;
+ vat_main_t *vam = &vat_main;
+ vat_json_node_t *e = 0, root;
+ u32 i, n;
+ int retval = clib_net_to_host_u32 (mp->retval);
+ vl_api_one_l2_arp_entry_t *arp_entry;
+
+ if (retval)
+ goto end;
+
+ n = clib_net_to_host_u32 (mp->count);
+ vat_json_init_array (&root);
+
+ for (i = 0; i < n; i++)
+ {
+ e = vat_json_array_add (&root);
+ arp_entry = &mp->entries[i];
+
+ vat_json_init_object (e);
+ s = format (0, "%U", format_ethernet_address, arp_entry->mac);
+ vec_add1 (s, 0);
+
+ vat_json_object_add_string_copy (e, "mac", s);
+ vec_free (s);
+
+ s = format (0, "%U", format_ip4_address, &arp_entry->ip4);
+ vec_add1 (s, 0);
+ vat_json_object_add_string_copy (e, "ip4", s);
+ vec_free (s);
+ }
+
+ vat_json_print (vam->ofp, &root);
+ vat_json_free (&root);
+
+end:
+ vam->retval = retval;
+ vam->result_ready = 1;
+}
+
+static void
+ vl_api_one_l2_arp_bd_get_reply_t_handler
+ (vl_api_one_l2_arp_bd_get_reply_t * mp)
+{
+ vat_main_t *vam = &vat_main;
+ u32 i, n;
+ int retval = clib_net_to_host_u32 (mp->retval);
+
+ if (retval)
+ goto end;
+
+ n = clib_net_to_host_u32 (mp->count);
+
+ for (i = 0; i < n; i++)
+ {
+ print (vam->ofp, "%d", clib_net_to_host_u32 (mp->bridge_domains[i]));
+ }
+
+end:
+ vam->retval = retval;
+ vam->result_ready = 1;
+}
+
+static void
+ vl_api_one_l2_arp_bd_get_reply_t_handler_json
+ (vl_api_one_l2_arp_bd_get_reply_t * mp)
+{
+ vat_main_t *vam = &vat_main;
+ vat_json_node_t root;
+ u32 i, n;
+ int retval = clib_net_to_host_u32 (mp->retval);
+
+ if (retval)
+ goto end;
+
+ n = clib_net_to_host_u32 (mp->count);
+ vat_json_init_array (&root);
+
+ for (i = 0; i < n; i++)
+ {
+ vat_json_array_add_uint (&root,
+ clib_net_to_host_u32 (mp->bridge_domains[i]));
+ }
+
+ vat_json_print (vam->ofp, &root);
+ vat_json_free (&root);
+
+end:
+ vam->retval = retval;
+ vam->result_ready = 1;
+}
+
+static void
vl_api_one_adjacencies_get_reply_t_handler
(vl_api_one_adjacencies_get_reply_t * mp)
{
@@ -4158,6 +4275,10 @@ static void vl_api_flow_classify_details_t_handler_json
#define vl_api_vnet_ip6_nbr_counters_t_print vl_noop_handler
#define vl_api_one_adjacencies_get_reply_t_endian vl_noop_handler
#define vl_api_one_adjacencies_get_reply_t_print vl_noop_handler
+#define vl_api_one_l2_arp_bd_get_reply_t_print vl_noop_handler
+#define vl_api_one_l2_arp_entries_get_reply_t_endian vl_noop_handler
+#define vl_api_one_l2_arp_entries_get_reply_t_print vl_noop_handler
+#define vl_api_one_l2_arp_bd_get_reply_t_endian vl_noop_handler
/*
* Generate boilerplate reply handlers, which
@@ -4267,6 +4388,7 @@ _(one_add_del_map_request_itr_rlocs_reply) \
_(one_eid_table_add_del_map_reply) \
_(one_use_petr_reply) \
_(one_stats_enable_disable_reply) \
+_(one_add_del_l2_arp_entry_reply) \
_(one_stats_flush_reply) \
_(gpe_add_del_fwd_entry_reply) \
_(gpe_enable_disable_reply) \
@@ -4499,6 +4621,9 @@ _(ONE_STATS_FLUSH_REPLY, one_stats_flush_reply) \
_(ONE_STATS_ENABLE_DISABLE_REPLY, one_stats_enable_disable_reply) \
_(SHOW_ONE_STATS_ENABLE_DISABLE_REPLY, \
show_one_stats_enable_disable_reply) \
+_(ONE_ADD_DEL_L2_ARP_ENTRY_REPLY, one_add_del_l2_arp_entry_reply) \
+_(ONE_L2_ARP_BD_GET_REPLY, one_l2_arp_bd_get_reply) \
+_(ONE_L2_ARP_ENTRIES_GET_REPLY, one_l2_arp_entries_get_reply) \
_(GPE_SET_ENCAP_MODE_REPLY, gpe_set_encap_mode_reply) \
_(GPE_GET_ENCAP_MODE_REPLY, gpe_get_encap_mode_reply) \
_(GPE_ADD_DEL_IFACE_REPLY, gpe_add_del_iface_reply) \
@@ -14737,6 +14862,111 @@ api_show_one_rloc_probe_state (vat_main_t * vam)
#define api_show_lisp_rloc_probe_state api_show_one_rloc_probe_state
static int
+api_one_add_del_l2_arp_entry (vat_main_t * vam)
+{
+ vl_api_one_add_del_l2_arp_entry_t *mp;
+ unformat_input_t *input = vam->input;
+ u8 is_add = 1;
+ u8 mac_set = 0;
+ u8 bd_set = 0;
+ u8 ip_set = 0;
+ u8 mac[6] = { 0, };
+ u32 ip4 = 0, bd = ~0;
+ int ret;
+
+ /* Parse args required to build the message */
+ while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+ {
+ if (unformat (input, "del"))
+ is_add = 0;
+ else if (unformat (input, "mac %U", unformat_ethernet_address, mac))
+ mac_set = 1;
+ else if (unformat (input, "ip %U", unformat_ip4_address, &ip4))
+ ip_set = 1;
+ else if (unformat (input, "bd %d", &bd))
+ bd_set = 1;
+ else
+ {
+ errmsg ("parse error '%U'", format_unformat_error, input);
+ return -99;
+ }
+ }
+
+ if (!bd_set || !ip_set || (!mac_set && is_add))
+ {
+ errmsg ("Missing BD, IP or MAC!");
+ return -99;
+ }
+
+ M (ONE_ADD_DEL_L2_ARP_ENTRY, mp);
+ mp->is_add = is_add;
+ clib_memcpy (mp->mac, mac, 6);
+ mp->bd = clib_host_to_net_u32 (bd);
+ mp->ip4 = ip4;
+
+ /* send */
+ S (mp);
+
+ /* wait for reply */
+ W (ret);
+ return ret;
+}
+
+static int
+api_one_l2_arp_bd_get (vat_main_t * vam)
+{
+ vl_api_one_l2_arp_bd_get_t *mp;
+ int ret;
+
+ M (ONE_L2_ARP_BD_GET, mp);
+
+ /* send */
+ S (mp);
+
+ /* wait for reply */
+ W (ret);
+ return ret;
+}
+
+static int
+api_one_l2_arp_entries_get (vat_main_t * vam)
+{
+ vl_api_one_l2_arp_entries_get_t *mp;
+ unformat_input_t *input = vam->input;
+ u8 bd_set = 0;
+ u32 bd = ~0;
+ int ret;
+
+ /* Parse args required to build the message */
+ while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
+ {
+ if (unformat (input, "bd %d", &bd))
+ bd_set = 1;
+ else
+ {
+ errmsg ("parse error '%U'", format_unformat_error, input);
+ return -99;
+ }
+ }
+
+ if (!bd_set)
+ {
+ errmsg ("Expected bridge domain!");
+ return -99;
+ }
+
+ M (ONE_L2_ARP_ENTRIES_GET, mp);
+ mp->bd = clib_host_to_net_u32 (bd);
+
+ /* send */
+ S (mp);
+
+ /* wait for reply */
+ W (ret);
+ return ret;
+}
+
+static int
api_one_stats_enable_disable (vat_main_t * vam)
{
vl_api_one_stats_enable_disable_t *mp;
@@ -19052,6 +19282,9 @@ _(one_locator_set_dump, "[local | remote]") \
_(one_locator_dump, "ls_index <index> | ls_name <name>") \
_(one_eid_table_dump, "[eid <ipv4|ipv6>/<prefix> | <mac>] [vni] " \
"[local] | [remote]") \
+_(one_add_del_l2_arp_entry, "[del] mac <mac> bd <bd> ip4 <ip4>") \
+_(one_l2_arp_bd_get, "") \
+_(one_l2_arp_entries_get, "bd <bridge-domain>") \
_(one_stats_enable_disable, "enable|disalbe") \
_(show_one_stats_enable_disable, "") \
_(one_eid_table_vni_dump, "") \
id='n576' href='#n576'>576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840
/*
 * Copyright (c) 2016 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <svm/svm_fifo.h>
#include <vppinfra/cpu.h>

static inline u8
position_lt (svm_fifo_t * f, u32 a, u32 b)
{
  return (ooo_segment_distance_from_tail (f, a)
	  < ooo_segment_distance_from_tail (f, b));
}

static inline u8
position_leq (svm_fifo_t * f, u32 a, u32 b)
{
  return (ooo_segment_distance_from_tail (f, a)
	  <= ooo_segment_distance_from_tail (f, b));
}

static inline u8
position_gt (svm_fifo_t * f, u32 a, u32 b)
{
  return (ooo_segment_distance_from_tail (f, a)
	  > ooo_segment_distance_from_tail (f, b));
}

static inline u32
position_diff (svm_fifo_t * f, u32 posa, u32 posb)
{
  return ooo_segment_distance_from_tail (f, posa)
    - ooo_segment_distance_from_tail (f, posb);
}

static inline u32
ooo_segment_end_pos (svm_fifo_t * f, ooo_segment_t * s)
{
  return (s->start + s->length) % f->nitems;
}

u8 *
format_ooo_segment (u8 * s, va_list * args)
{
  svm_fifo_t *f = va_arg (*args, svm_fifo_t *);
  ooo_segment_t *seg = va_arg (*args, ooo_segment_t *);
  u32 normalized_start = (seg->start + f->nitems - f->tail) % f->nitems;
  s = format (s, "[%u, %u], len %u, next %d, prev %d", normalized_start,
	      (normalized_start + seg->length) % f->nitems, seg->length,
	      seg->next, seg->prev);
  return s;
}

u8 *
svm_fifo_dump_trace (u8 * s, svm_fifo_t * f)
{
#if SVM_FIFO_TRACE
  svm_fifo_trace_elem_t *seg = 0;
  int i = 0;

  if (f->trace)
    {
      vec_foreach (seg, f->trace)
      {
	s = format (s, "{%u, %u, %u}, ", seg->offset, seg->len, seg->action);
	i++;
	if (i % 5 == 0)
	  s = format (s, "\n");
      }
      s = format (s, "\n");
    }
  return s;
#else
  return 0;
#endif
}

u8 *
svm_fifo_replay (u8 * s, svm_fifo_t * f, u8 no_read, u8 verbose)
{
  int i, trace_len;
  u8 *data = 0;
  svm_fifo_trace_elem_t *trace;
  u32 offset;
  svm_fifo_t *dummy_fifo;

  if (!f)
    return s;

#if SVM_FIFO_TRACE
  trace = f->trace;
  trace_len = vec_len (trace);
#else
  trace = 0;
  trace_len = 0;
#endif

  dummy_fifo = svm_fifo_create (f->nitems);
  memset (f->data, 0xFF, f->nitems);

  vec_validate (data, f->nitems);
  for (i = 0; i < vec_len (data); i++)
    data[i] = i;

  for (i = 0; i < trace_len; i++)
    {
      offset = trace[i].offset;
      if (trace[i].action == 1)
	{
	  if (verbose)
	    s = format (s, "adding [%u, %u]:", trace[i].offset,
			(trace[i].offset +
			 trace[i].len) % dummy_fifo->nitems);
	  svm_fifo_enqueue_with_offset (dummy_fifo, trace[i].offset,
					trace[i].len, &data[offset]);
	}
      else if (trace[i].action == 2)
	{
	  if (verbose)
	    s = format (s, "adding [%u, %u]:", 0, trace[i].len);
	  svm_fifo_enqueue_nowait (dummy_fifo, trace[i].len, &data[offset]);
	}
      else if (!no_read)
	{
	  if (verbose)
	    s = format (s, "read: %u", trace[i].len);
	  svm_fifo_dequeue_drop (dummy_fifo, trace[i].len);
	}
      if (verbose)
	s = format (s, "%U", format_svm_fifo, dummy_fifo, 1);
    }

  s = format (s, "result: %U", format_svm_fifo, dummy_fifo, 1);

  return s;
}

u8 *
format_ooo_list (u8 * s, va_list * args)
{
  svm_fifo_t *f = va_arg (*args, svm_fifo_t *);
  u32 ooo_segment_index = f->ooos_list_head;
  ooo_segment_t *seg;

  while (ooo_segment_index != OOO_SEGMENT_INVALID_INDEX)
    {
      seg = pool_elt_at_index (f->ooo_segments, ooo_segment_index);
      s = format (s, "  %U\n", format_ooo_segment, f, seg);
      ooo_segment_index = seg->next;
    }

  return s;
}

u8 *
format_svm_fifo (u8 * s, va_list * args)
{
  svm_fifo_t *f = va_arg (*args, svm_fifo_t *);
  int verbose = va_arg (*args, int);

  s = format (s, "cursize %u nitems %u has_event %d\n",
	      f->cursize, f->nitems, f->has_event);
  s = format (s, " head %d tail %d\n", f->head, f->tail);

  if (verbose > 1)
    s = format
      (s, " server session %d thread %d client session %d thread %d\n",
       f->master_session_index, f->master_thread_index,
       f->client_session_index, f->client_thread_index);

  if (verbose)
    {
      s = format (s, " ooo pool %d active elts newest %u\n",
		  pool_elts (f->ooo_segments), f->ooos_newest);
      if (svm_fifo_has_ooo_data (f))
	s = format (s, " %U", format_ooo_list, f, verbose);
    }
  return s;
}

/** create an svm fifo, in the current heap. Fails vs blow up the process */
svm_fifo_t *
svm_fifo_create (u32 data_size_in_bytes)
{
  svm_fifo_t *f;
  u32 rounded_data_size;

  /* always round fifo data size to the next highest power-of-two */
  rounded_data_size = (1 << (max_log2 (data_size_in_bytes)));
  f = clib_mem_alloc_aligned_or_null (sizeof (*f) + rounded_data_size,
				      CLIB_CACHE_LINE_BYTES);
  if (f == 0)
    return 0;

  memset (f, 0, sizeof (*f));
  f->nitems = data_size_in_bytes;
  f->ooos_list_head = OOO_SEGMENT_INVALID_INDEX;
  f->refcnt = 1;
  return (f);
}

void
svm_fifo_free (svm_fifo_t * f)
{
  ASSERT (f->refcnt > 0);

  if (--f->refcnt == 0)
    {
      pool_free (f->ooo_segments);
      clib_mem_free (f);
    }
}

always_inline ooo_segment_t *
ooo_segment_new (svm_fifo_t * f, u32 start, u32 length)
{
  ooo_segment_t *s;

  pool_get (f->ooo_segments, s);

  s->start = start;
  s->length = length;

  s->prev = s->next = OOO_SEGMENT_INVALID_INDEX;

  return s;
}

always_inline void
ooo_segment_del (svm_fifo_t * f, u32 index)
{
  ooo_segment_t *cur, *prev = 0, *next = 0;
  cur = pool_elt_at_index (f->ooo_segments, index);

  if (cur->next != OOO_SEGMENT_INVALID_INDEX)
    {
      next = pool_elt_at_index (f->ooo_segments, cur->next);
      next->prev = cur->prev;
    }

  if (cur->prev != OOO_SEGMENT_INVALID_INDEX)
    {
      prev = pool_elt_at_index (f->ooo_segments, cur->prev);
      prev->next = cur->next;
    }
  else
    {
      f->ooos_list_head = cur->next;
    }

  pool_put (f->ooo_segments, cur);
}

/**
 * Add segment to fifo's out-of-order segment list. Takes care of merging
 * adjacent segments and removing overlapping ones.
 */
static void
ooo_segment_add (svm_fifo_t * f, u32 offset, u32 length)
{
  ooo_segment_t *s, *new_s, *prev, *next, *it;
  u32 new_index, s_end_pos, s_index;
  u32 normalized_position, normalized_end_position;

  ASSERT (offset + length <= ooo_segment_distance_from_tail (f, f->head));
  normalized_position = (f->tail + offset) % f->nitems;
  normalized_end_position = (f->tail + offset + length) % f->nitems;

  f->ooos_newest = OOO_SEGMENT_INVALID_INDEX;

  if (f->ooos_list_head == OOO_SEGMENT_INVALID_INDEX)
    {
      s = ooo_segment_new (f, normalized_position, length);
      f->ooos_list_head = s - f->ooo_segments;
      f->ooos_newest = f->ooos_list_head;
      return;
    }

  /* Find first segment that starts after new segment */
  s = pool_elt_at_index (f->ooo_segments, f->ooos_list_head);
  while (s->next != OOO_SEGMENT_INVALID_INDEX
	 && position_lt (f, s->start, normalized_position))
    s = pool_elt_at_index (f->ooo_segments, s->next);

  /* If we have a previous and we overlap it, use it as starting point */
  prev = ooo_segment_get_prev (f, s);
  if (prev
      && position_leq (f, normalized_position, ooo_segment_end_pos (f, prev)))
    {
      s = prev;
      s_end_pos = ooo_segment_end_pos (f, s);

      /* Since we have previous, normalized start position cannot be smaller
       * than prev->start. Check tail */
      ASSERT (position_lt (f, s->start, normalized_position));
      goto check_tail;
    }

  s_index = s - f->ooo_segments;
  s_end_pos = ooo_segment_end_pos (f, s);

  /* No overlap, add before current segment */
  if (position_lt (f, normalized_end_position, s->start))
    {
      new_s = ooo_segment_new (f, normalized_position, length);
      new_index = new_s - f->ooo_segments;

      /* Pool might've moved, get segment again */
      s = pool_elt_at_index (f->ooo_segments, s_index);
      if (s->prev != OOO_SEGMENT_INVALID_INDEX)
	{
	  new_s->prev = s->prev;
	  prev = pool_elt_at_index (f->ooo_segments, new_s->prev);
	  prev->next = new_index;
	}
      else
	{
	  /* New head */
	  f->ooos_list_head = new_index;
	}

      new_s->next = s_index;
      s->prev = new_index;
      f->ooos_newest = new_index;
      return;
    }
  /* No overlap, add after current segment */
  else if (position_gt (f, normalized_position, s_end_pos))
    {
      new_s = ooo_segment_new (f, normalized_position, length);
      new_index = new_s - f->ooo_segments;

      /* Pool might've moved, get segment again */
      s = pool_elt_at_index (f->ooo_segments, s_index);

      /* Needs to be last */
      ASSERT (s->next == OOO_SEGMENT_INVALID_INDEX);

      new_s->prev = s_index;
      s->next = new_index;
      f->ooos_newest = new_index;

      return;
    }

  /*
   * Merge needed
   */

  /* Merge at head */
  if (position_lt (f, normalized_position, s->start))
    {
      s->start = normalized_position;
      s->length = position_diff (f, s_end_pos, s->start);
      f->ooos_newest = s - f->ooo_segments;
    }

check_tail:

  /* Overlapping tail */
  if (position_gt (f, normalized_end_position, s_end_pos))
    {
      s->length = position_diff (f, normalized_end_position, s->start);

      /* Remove the completely overlapped segments in the tail */
      it = ooo_segment_next (f, s);
      while (it && position_leq (f, ooo_segment_end_pos (f, it),
				 normalized_end_position))
	{
	  next = ooo_segment_next (f, it);
	  ooo_segment_del (f, it - f->ooo_segments);
	  it = next;
	}

      /* If partial overlap with last, merge */
      if (it && position_leq (f, it->start, normalized_end_position))
	{
	  s->length = position_diff (f, ooo_segment_end_pos (f, it),
				     s->start);
	  ooo_segment_del (f, it - f->ooo_segments);
	}
      f->ooos_newest = s - f->ooo_segments;
    }
}

/**
 * Removes segments that can now be enqueued because the fifo's tail has
 * advanced. Returns the number of bytes added to tail.
 */
static int
ooo_segment_try_collect (svm_fifo_t * f, u32 n_bytes_enqueued)
{
  ooo_segment_t *s;
  u32 index, bytes = 0;
  i32 diff;

  s = pool_elt_at_index (f->ooo_segments, f->ooos_list_head);
  diff = ooo_segment_distance_to_tail (f, s->start);

  ASSERT (diff != n_bytes_enqueued);

  if (diff > n_bytes_enqueued)
    return 0;

  /* If last tail update overlaps one/multiple ooo segments, remove them */
  while (0 <= diff && diff < n_bytes_enqueued)
    {
      index = s - f->ooo_segments;

      /* Segment end is beyond the tail. Advance tail and remove segment */
      if (s->length > diff)
	{
	  bytes = s->length - diff;
	  f->tail += bytes;
	  f->tail %= f->nitems;
	  ooo_segment_del (f, index);
	  break;
	}

      /* If we have next go on */
      if (s->next != OOO_SEGMENT_INVALID_INDEX)
	{
	  s = pool_elt_at_index (f->ooo_segments, s->next);
	  diff = ooo_segment_distance_to_tail (f, s->start);
	  ooo_segment_del (f, index);
	}
      /* End of search */
      else
	{
	  ooo_segment_del (f, index);
	  break;
	}
    }

  ASSERT (bytes <= f->nitems);
  return bytes;
}

static int
svm_fifo_enqueue_internal (svm_fifo_t * f, u32 max_bytes,
			   const u8 * copy_from_here)
{
  u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
  u32 cursize, nitems;

  /* read cursize, which can only increase while we're working */
  cursize = svm_fifo_max_dequeue (f);
  f->ooos_newest = OOO_SEGMENT_INVALID_INDEX;

  if (PREDICT_FALSE (cursize == f->nitems))
    return -2;			/* fifo stuffed */

  nitems = f->nitems;

  /* Number of bytes we're going to copy */
  total_copy_bytes = (nitems - cursize) < max_bytes ?
    (nitems - cursize) : max_bytes;

  if (PREDICT_TRUE (copy_from_here != 0))
    {
      /* Number of bytes in first copy segment */
      first_copy_bytes = ((nitems - f->tail) < total_copy_bytes)
	? (nitems - f->tail) : total_copy_bytes;

      clib_memcpy (&f->data[f->tail], copy_from_here, first_copy_bytes);
      f->tail += first_copy_bytes;
      f->tail = (f->tail == nitems) ? 0 : f->tail;

      /* Number of bytes in second copy segment, if any */
      second_copy_bytes = total_copy_bytes - first_copy_bytes;
      if (second_copy_bytes)
	{
	  clib_memcpy (&f->data[f->tail], copy_from_here + first_copy_bytes,
		       second_copy_bytes);
	  f->tail += second_copy_bytes;
	  f->tail = (f->tail == nitems) ? 0 : f->tail;
	}
    }
  else
    {
      ASSERT (0);

      /* Account for a zero-copy enqueue done elsewhere */
      ASSERT (max_bytes <= (nitems - cursize));
      f->tail += max_bytes;
      f->tail = f->tail % nitems;
      total_copy_bytes = max_bytes;
    }

  svm_fifo_trace_add (f, f->head, total_copy_bytes, 2);

  /* Any out-of-order segments to collect? */
  if (PREDICT_FALSE (f->ooos_list_head != OOO_SEGMENT_INVALID_INDEX))
    total_copy_bytes += ooo_segment_try_collect (f, total_copy_bytes);

  /* Atomically increase the queue length */
  ASSERT (cursize + total_copy_bytes <= nitems);
  __sync_fetch_and_add (&f->cursize, total_copy_bytes);

  return (total_copy_bytes);
}

#define SVM_ENQUEUE_CLONE_TEMPLATE(arch, fn, tgt)                       \
  uword                                                                 \
  __attribute__ ((flatten))                                             \
  __attribute__ ((target (tgt)))                                        \
  CLIB_CPU_OPTIMIZED                                                    \
  fn ## _ ## arch ( svm_fifo_t * f, u32 max_bytes, u8 * copy_from_here) \
  { return fn (f, max_bytes, copy_from_here);}

static int
svm_fifo_enqueue_nowait_ma (svm_fifo_t * f, u32 max_bytes,
			    const u8 * copy_from_here)
{
  return svm_fifo_enqueue_internal (f, max_bytes, copy_from_here);
}

foreach_march_variant (SVM_ENQUEUE_CLONE_TEMPLATE,
		       svm_fifo_enqueue_nowait_ma);
CLIB_MULTIARCH_SELECT_FN (svm_fifo_enqueue_nowait_ma);

int
svm_fifo_enqueue_nowait (svm_fifo_t * f, u32 max_bytes,
			 const u8 * copy_from_here)
{
#if CLIB_DEBUG > 0
  return svm_fifo_enqueue_nowait_ma (f, max_bytes, copy_from_here);
#else
  static int (*fp) (svm_fifo_t *, u32, const u8 *);

  if (PREDICT_FALSE (fp == 0))
    fp = (void *) svm_fifo_enqueue_nowait_ma_multiarch_select ();

  return (*fp) (f, max_bytes, copy_from_here);
#endif
}

/**
 * Enqueue a future segment.
 *
 * Two choices: either copies the entire segment, or copies nothing
 * Returns 0 of the entire segment was copied
 * Returns -1 if none of the segment was copied due to lack of space
 */
static int
svm_fifo_enqueue_with_offset_internal (svm_fifo_t * f,
				       u32 offset,
				       u32 required_bytes,
				       u8 * copy_from_here)
{
  u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
  u32 cursize, nitems, normalized_offset;

  f->ooos_newest = OOO_SEGMENT_INVALID_INDEX;

  /* read cursize, which can only increase while we're working */
  cursize = svm_fifo_max_dequeue (f);
  nitems = f->nitems;

  ASSERT (required_bytes < nitems);

  normalized_offset = (f->tail + offset) % nitems;

  /* Will this request fit? */
  if ((required_bytes + offset) > (nitems - cursize))
    return -1;

  svm_fifo_trace_add (f, offset, required_bytes, 1);

  ooo_segment_add (f, offset, required_bytes);

  /* Number of bytes we're going to copy */
  total_copy_bytes = required_bytes;

  /* Number of bytes in first copy segment */
  first_copy_bytes = ((nitems - normalized_offset) < total_copy_bytes)
    ? (nitems - normalized_offset) : total_copy_bytes;

  clib_memcpy (&f->data[normalized_offset], copy_from_here, first_copy_bytes);

  /* Number of bytes in second copy segment, if any */
  second_copy_bytes = total_copy_bytes - first_copy_bytes;
  if (second_copy_bytes)
    {
      normalized_offset += first_copy_bytes;
      normalized_offset %= nitems;

      ASSERT (normalized_offset == 0);

      clib_memcpy (&f->data[normalized_offset],
		   copy_from_here + first_copy_bytes, second_copy_bytes);
    }

  return (0);
}


int
svm_fifo_enqueue_with_offset (svm_fifo_t * f,
			      u32 offset,
			      u32 required_bytes, u8 * copy_from_here)
{
  return svm_fifo_enqueue_with_offset_internal (f, offset, required_bytes,
						copy_from_here);
}


static int
svm_fifo_dequeue_internal (svm_fifo_t * f, u32 max_bytes, u8 * copy_here)
{
  u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
  u32 cursize, nitems;

  /* read cursize, which can only increase while we're working */
  cursize = svm_fifo_max_dequeue (f);
  if (PREDICT_FALSE (cursize == 0))
    return -2;			/* nothing in the fifo */

  nitems = f->nitems;

  /* Number of bytes we're going to copy */
  total_copy_bytes = (cursize < max_bytes) ? cursize : max_bytes;

  if (PREDICT_TRUE (copy_here != 0))
    {
      /* Number of bytes in first copy segment */
      first_copy_bytes = ((nitems - f->head) < total_copy_bytes)
	? (nitems - f->head) : total_copy_bytes;
      clib_memcpy (copy_here, &f->data[f->head], first_copy_bytes);
      f->head += first_copy_bytes;
      f->head = (f->head == nitems) ? 0 : f->head;

      /* Number of bytes in second copy segment, if any */
      second_copy_bytes = total_copy_bytes - first_copy_bytes;
      if (second_copy_bytes)
	{
	  clib_memcpy (copy_here + first_copy_bytes,
		       &f->data[f->head], second_copy_bytes);
	  f->head += second_copy_bytes;
	  f->head = (f->head == nitems) ? 0 : f->head;
	}
    }
  else
    {
      ASSERT (0);
      /* Account for a zero-copy dequeue done elsewhere */
      ASSERT (max_bytes <= cursize);
      f->head += max_bytes;
      f->head = f->head % nitems;
      cursize -= max_bytes;
      total_copy_bytes = max_bytes;
    }

  ASSERT (f->head <= nitems);
  ASSERT (cursize >= total_copy_bytes);
  __sync_fetch_and_sub (&f->cursize, total_copy_bytes);

  return (total_copy_bytes);
}

static int
svm_fifo_dequeue_nowait_ma (svm_fifo_t * f, u32 max_bytes, u8 * copy_here)
{
  return svm_fifo_dequeue_internal (f, max_bytes, copy_here);
}

#define SVM_FIFO_DEQUEUE_CLONE_TEMPLATE(arch, fn, tgt)          \
  uword                                                         \
  __attribute__ ((flatten))                                     \
  __attribute__ ((target (tgt)))                                \
  CLIB_CPU_OPTIMIZED                                            \
  fn ## _ ## arch ( svm_fifo_t * f, u32 max_bytes,              \
                    u8 * copy_here)                             \
  { return fn (f, max_bytes, copy_here);}

foreach_march_variant (SVM_FIFO_DEQUEUE_CLONE_TEMPLATE,
		       svm_fifo_dequeue_nowait_ma);
CLIB_MULTIARCH_SELECT_FN (svm_fifo_dequeue_nowait_ma);

int
svm_fifo_dequeue_nowait (svm_fifo_t * f, u32 max_bytes, u8 * copy_here)
{
#if CLIB_DEBUG > 0
  return svm_fifo_dequeue_nowait_ma (f, max_bytes, copy_here);
#else
  static int (*fp) (svm_fifo_t *, u32, u8 *);

  if (PREDICT_FALSE (fp == 0))
    fp = (void *) svm_fifo_dequeue_nowait_ma_multiarch_select ();

  return (*fp) (f, max_bytes, copy_here);
#endif
}

static int
svm_fifo_peek_ma (svm_fifo_t * f, u32 relative_offset, u32 max_bytes,
		  u8 * copy_here)
{
  u32 total_copy_bytes, first_copy_bytes, second_copy_bytes;
  u32 cursize, nitems, real_head;

  /* read cursize, which can only increase while we're working */
  cursize = svm_fifo_max_dequeue (f);
  if (PREDICT_FALSE (cursize < relative_offset))
    return -2;			/* nothing in the fifo */

  nitems = f->nitems;
  real_head = f->head + relative_offset;
  real_head = real_head >= nitems ? real_head - nitems : real_head;

  /* Number of bytes we're going to copy */
  total_copy_bytes = (cursize - relative_offset < max_bytes) ?
    cursize - relative_offset : max_bytes;

  if (PREDICT_TRUE (copy_here != 0))
    {
      /* Number of bytes in first copy segment */
      first_copy_bytes =
	((nitems - real_head) < total_copy_bytes) ?
	(nitems - real_head) : total_copy_bytes;
      clib_memcpy (copy_here, &f->data[real_head], first_copy_bytes);

      /* Number of bytes in second copy segment, if any */
      second_copy_bytes = total_copy_bytes - first_copy_bytes;
      if (second_copy_bytes)
	{
	  clib_memcpy (copy_here + first_copy_bytes, &f->data[0],
		       second_copy_bytes);
	}
    }
  return total_copy_bytes;
}

#define SVM_FIFO_PEEK_CLONE_TEMPLATE(arch, fn, tgt)                     \
  uword                                                                 \
  __attribute__ ((flatten))                                             \
  __attribute__ ((target (tgt)))                                        \
  CLIB_CPU_OPTIMIZED                                                    \
  fn ## _ ## arch ( svm_fifo_t * f, u32 relative_offset, u32 max_bytes, \
                    u8 * copy_here)                                     \
  { return fn (f, relative_offset, max_bytes, copy_here);}

foreach_march_variant (SVM_FIFO_PEEK_CLONE_TEMPLATE, svm_fifo_peek_ma);
CLIB_MULTIARCH_SELECT_FN (svm_fifo_peek_ma);

int
svm_fifo_peek (svm_fifo_t * f, u32 relative_offset, u32 max_bytes,
	       u8 * copy_here)
{
#if CLIB_DEBUG > 0
  return svm_fifo_peek_ma (f, relative_offset, max_bytes, copy_here);
#else
  static int (*fp) (svm_fifo_t *, u32, u32, u8 *);

  if (PREDICT_FALSE (fp == 0))
    fp = (void *) svm_fifo_peek_ma_multiarch_select ();

  return (*fp) (f, relative_offset, max_bytes, copy_here);
#endif
}

int
svm_fifo_dequeue_drop (svm_fifo_t * f, u32 max_bytes)
{
  u32 total_drop_bytes, first_drop_bytes, second_drop_bytes;
  u32 cursize, nitems;

  /* read cursize, which can only increase while we're working */
  cursize = svm_fifo_max_dequeue (f);
  if (PREDICT_FALSE (cursize == 0))
    return -2;			/* nothing in the fifo */

  nitems = f->nitems;

  /* Number of bytes we're going to drop */
  total_drop_bytes = (cursize < max_bytes) ? cursize : max_bytes;

  svm_fifo_trace_add (f, f->tail, total_drop_bytes, 3);

  /* Number of bytes in first copy segment */
  first_drop_bytes =
    ((nitems - f->head) < total_drop_bytes) ?
    (nitems - f->head) : total_drop_bytes;
  f->head += first_drop_bytes;
  f->head = (f->head == nitems) ? 0 : f->head;

  /* Number of bytes in second drop segment, if any */
  second_drop_bytes = total_drop_bytes - first_drop_bytes;
  if (second_drop_bytes)
    {
      f->head += second_drop_bytes;
      f->head = (f->head == nitems) ? 0 : f->head;
    }

  ASSERT (f->head <= nitems);
  ASSERT (cursize >= total_drop_bytes);
  __sync_fetch_and_sub (&f->cursize, total_drop_bytes);

  return total_drop_bytes;
}

u32
svm_fifo_number_ooo_segments (svm_fifo_t * f)
{
  return pool_elts (f->ooo_segments);
}

ooo_segment_t *
svm_fifo_first_ooo_segment (svm_fifo_t * f)
{
  return pool_elt_at_index (f->ooo_segments, f->ooos_list_head);
}

/**
 * Set fifo pointers to requested offset
 */
void
svm_fifo_init_pointers (svm_fifo_t * f, u32 pointer)
{
  f->head = f->tail = pointer % f->nitems;
}

/*
 * fd.io coding-style-patch-verification: ON
 *
 * Local Variables:
 * eval: (c-set-style "gnu")
 * End:
 */