/*
 * Copyright (c) 2015 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/*
 * cli.c: command line interface
 *
 * Copyright (c) 2008 Eliot Dresselhaus
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

#include <vlib/vlib.h>
#include <vlib/unix/unix.h>
#include <vppinfra/callback.h>
#include <vppinfra/cpu.h>
#include <vppinfra/elog.h>
#include <unistd.h>
#include <ctype.h>

/** \file src/vlib/cli.c Debug CLI Implementation
 */

int vl_api_set_elog_trace_api_messages (int enable);
int vl_api_get_elog_trace_api_messages (void);

static void *current_traced_heap;

/* Root of all show commands. */
/* *INDENT-OFF* */
VLIB_CLI_COMMAND (vlib_cli_show_command, static) = {
  .path = "show",
  .short_help = "Show commands",
};
/* *INDENT-ON* */

/* Root of all clear commands. */
/* *INDENT-OFF* */
VLIB_CLI_COMMAND (vlib_cli_clear_command, static) = {
  .path = "clear",
  .short_help = "Clear commands",
};
/* *INDENT-ON* */

/* Root of all set commands. */
/* *INDENT-OFF* */
VLIB_CLI_COMMAND (vlib_cli_set_command, static) = {
  .path = "set",
  .short_help = "Set commands",
};
/* *INDENT-ON* */

/* Root of all test commands. */
/* *INDENT-OFF* */
VLIB_CLI_COMMAND (vlib_cli_test_command, static) = {
  .path = "test",
  .short_help = "Test commands",
};
/* *INDENT-ON* */

/* Returns bitmap of commands which match key. */
static uword *
vlib_cli_sub_command_match (vlib_cli_command_t * c, unformat_input_t * input)
{
  int i, n;
  uword *match = 0;
  vlib_cli_parse_position_t *p;

  unformat_skip_white_space (input);

  for (i = 0;; i++)
    {
      uword k;

      k = unformat_get_input (input);
      switch (k)
	{
	case 'a' ... 'z':
	case 'A' ... 'Z':
	case '0' ... '9':
	case '-':
	case '_':
	  break;

	case ' ':
	case '\t':
	case '\r':
	case '\n':
	case UNFORMAT_END_OF_INPUT:
	  /* White space or end of input removes any non-white
	     matches that were before possible. */
	  if (i < vec_len (c->sub_command_positions)
	      && clib_bitmap_count_set_bits (match) > 1)
	    {
	      p = vec_elt_at_index (c->sub_command_positions, i);
	      for (n = 0; n < vec_len (p->bitmaps); n++)
		match = clib_bitmap_andnot (match, p->bitmaps[n]);
	    }
	  goto done;

	default:
	  unformat_put_input (input);
	  goto done;
	}

      if (i >= vec_len (c->sub_command_positions))
	{
	no_match:
	  clib_bitmap_free (match);
	  return 0;
	}

      p = vec_elt_at_index (c->sub_command_positions, i);
      if (vec_len (p->bitmaps) == 0)
	goto no_match;

      n = k - p->min_char;
      if (n < 0 || n >= vec_len (p->bitmaps))
	goto no_match;

      if (i == 0)
	match = clib_bitmap_dup (p->bitmaps[n]);
      else
	match = clib_bitmap_and (match, p->bitmaps[n]);

      if (clib_bitmap_is_zero (match))
	goto no_match;
    }

done:
  return match;
}

/* Looks for string based sub-input formatted { SUB-INPUT }. */
uword
unformat_vlib_cli_sub_input (unformat_input_t * i, va_list * args)
{
  unformat_input_t *sub_input = va_arg (*args, unformat_input_t *);
  u8 *s;
  uword c;

  while (1)
    {
      c = unformat_get_input (i);
      switch (c)
	{
	case ' ':
	case '\t':
	case '\n':
	case '\r':
	case '\f':
	  break;

	case '{':
	default:
	  /* Put back paren. */
	  if (c != UNFORMAT_END_OF_INPUT)
	    unformat_put_input (i);

	  if (c == '{' && unformat (i, "%v", &s))
	    {
	      unformat_init_vector (sub_input, s);
	      return 1;
	    }
	  return 0;
	}
    }
  return 0;
}

static vlib_cli_command_t *
get_sub_command (vlib_cli_main_t * cm, vlib_cli_command_t * parent, u32 si)
{
  vlib_cli_sub_command_t *s = vec_elt_at_index (parent->sub_commands, si);
  return vec_elt_at_index (cm->commands, s->index);
}

static uword
unformat_vlib_cli_sub_command (unformat_input_t * i, va_list * args)
{
  vlib_main_t __clib_unused *vm = va_arg (*args, vlib_main_t *);
  vlib_global_main_t *vgm = vlib_get_global_main ();
  vlib_cli_command_t *c = va_arg (*args, vlib_cli_command_t *);
  vlib_cli_command_t **result = va_arg (*args, vlib_cli_command_t **);
  vlib_cli_main_t *cm = &vgm->cli_main;
  uword *match_bitmap, is_unique, index;

  match_bitmap = vlib_cli_sub_command_match (c, i);
  is_unique = clib_bitmap_count_set_bits (match_bitmap) == 1;
  index = ~0;
  if (is_unique)
    {
      index = clib_bitmap_first_set (match_bitmap);
      *result = get_sub_command (cm, c, index);
    }
  clib_bitmap_free (match_bitmap);

  return is_unique;
}

static int
vlib_cli_cmp_strings (void *a1, void *a2)
{
  u8 *c1 = *(u8 **) a1;
  u8 *c2 = *(u8 **) a2;

  return vec_cmp (c1, c2);
}

u8 **
vlib_cli_get_possible_completions (u8 * str)
{
  vlib_cli_command_t *c;
  vlib_cli_sub_command_t *sc;
  vlib_global_main_t *vgm = vlib_get_global_main ();
  vlib_cli_main_t *vcm = &vgm->cli_main;
  uword *match_bitmap = 0;
  uword index, is_unique, help_next_level;
  u8 **result = 0;
  unformat_input_t input;
  unformat_init_vector (&input, vec_dup (str));
  c = vec_elt_at_index (vcm->commands, 0);

  /* remove trailing whitespace, except for one of them */
  while (vec_len (input.buffer) >= 2 &&
	 isspace (input.buffer[vec_len (input.buffer) - 1]) &&
	 isspace (input.buffer[vec_len (input.buffer) - 2]))
    {
      vec_del1 (input.buffer, vec_len (input.buffer) - 1);
    }

  /* if input is empty, directly return list of root commands */
  if (vec_len (input.buffer) == 0 ||
      (vec_len (input.buffer) == 1 && isspace (input.buffer[0])))
    {
      vec_foreach (sc, c->sub_commands)
      {
	vec_add1 (result, (u8 *) sc->name);
      }
      goto done;
    }

  /* add a trailing '?' so that vlib_cli_sub_command_match can find
   * all commands starting with the input string */
  vec_add1 (input.buffer, '?');

  while (1)
    {
      match_bitmap = vlib_cli_sub_command_match (c, &input);
      /* no match: return no result */
      if (match_bitmap == 0)
	{
	  goto done;
	}
      is_unique = clib_bitmap_count_set_bits (match_bitmap) == 1;
      /* unique match: try to step one subcommand level further */
      if (is_unique)
	{
	  /* stop if no more input */
	  if (input.index >= vec_len (input.buffer) - 1)
	    {
	      break;
	    }

	  index = clib_bitmap_first_set (match_bitmap);
	  c = get_sub_command (vcm, c, index);
	  clib_bitmap_free (match_bitmap);
	  continue;
	}
      /* multiple matches: stop here, return all matches */
      break;
    }

  /* remove trailing '?' */
  vec_del1 (input.buffer, vec_len (input.buffer) - 1);

  /* if we have a space at the end of input, and a unique match,
   * autocomplete the next level of subcommands */
  help_next_level = (vec_len (str) == 0) || isspace (str[vec_len (str) - 1]);
  /* *INDENT-OFF* */
  clib_bitmap_foreach (index, match_bitmap) {
    if (help_next_level && is_unique) {
	c = get_sub_command (vcm, c, index);
	vec_foreach (sc, c->sub_commands) {
	  vec_add1 (result, (u8*) sc->name);
	}
	goto done; /* break doesn't work in this macro-loop */
    }
    sc = &c->sub_commands[index];
    vec_add1(result, (u8*) sc->name);
  }
  /* *INDENT-ON* */

done:
  clib_bitmap_free (match_bitmap);
  unformat_free (&input);

  if (result)
    vec_sort_with_function (result, vlib_cli_cmp_strings);
  return result;
}

static u8 *
format_vlib_cli_command_help (u8 * s, va_list * args)
{
  vlib_cli_command_t *c = va_arg (*args, vlib_cli_command_t *);
  int is_long = va_arg (*args, int);
  if (is_long && c->long_help)
    s = format (s, "%s", c->long_help);
  else if (c->short_help)
    s = format (s, "%s", c->short_help);
  else
    s = format (s, "%v commands", c->path);
  return s;
}

static u8 *
format_vlib_cli_path (u8 * s, va_list * args)
{
  u8 *path = va_arg (*args, u8 *);

  s = format (s, "%v", path);

  return s;
}

static vlib_cli_command_t *
all_subs (vlib_cli_main_t * cm, vlib_cli_command_t * subs, u32 command_index)
{
  vlib_cli_command_t *c = vec_elt_at_index (cm->commands, command_index);
  vlib_cli_sub_command_t *sc;

  if (c->function)
    vec_add1 (subs, c[0]);

  vec_foreach (sc, c->sub_commands) subs = all_subs (cm, subs, sc->index);

  return subs;
}

static int
vlib_cli_cmp_rule (void *a1, void *a2)
{
  vlib_cli_sub_rule_t *r1 = a1;
  vlib_cli_sub_rule_t *r2 = a2;

  return vec_cmp (r1->name, r2->name);
}

static int
vlib_cli_cmp_command (void *a1, void *a2)
{
  vlib_cli_command_t *c1 = a1;
  vlib_cli_command_t *c2 = a2;

  return vec_cmp (c1->path, c2->path);
}

static clib_error_t *
vlib_cli_dispatch_sub_commands (vlib_main_t * vm,
				vlib_cli_main_t * cm,
				unformat_input_t * input,
				uword parent_command_index)
{
  vlib_global_main_t *vgm = vlib_get_global_main ();
  vlib_cli_command_t *parent, *c;
  clib_error_t *error = 0;
  unformat_input_t sub_input;
  u8 *string;
  uword is_main_dispatch = cm == &vgm->cli_main;

  parent = vec_elt_at_index (cm->commands, parent_command_index);
  if (is_main_dispatch && unformat (input, "help"))
    {
      uword help_at_end_of_line, i;

      help_at_end_of_line =
	unformat_check_input (input) == UNFORMAT_END_OF_INPUT;
      while (1)
	{
	  c = parent;
	  if (unformat_user
	      (input, unformat_vlib_cli_sub_command, vm, c, &parent))
	    ;

	  else if (!(unformat_check_input (input) == UNFORMAT_END_OF_INPUT))
	    goto unknown;

	  else
	    break;
	}

      /* help SUB-COMMAND => long format help.
         "help" at end of line: show all commands. */
      if (!help_at_end_of_line)
	vlib_cli_output (vm, "%U", format_vlib_cli_command_help, c,
			 /* is_long */ 1);

      else if (vec_len (c->sub_commands) == 0)
	vlib_cli_output (vm, "%v: no sub-commands", c->path);

      else
	{
	  vlib_cli_sub_rule_t *sr, *subs = 0;
	  vlib_cli_sub_command_t *sc;

	  vec_foreach (sc, c->sub_commands)
	  {
	    vec_add2 (subs, sr, 1);
	    sr->name = sc->name;
	    sr->command_index = sc->index;
	    sr->rule_index = ~0;
	  }

	  vec_sort_with_function (subs, vlib_cli_cmp_rule);

	  for (i = 0; i < vec_len (subs); i++)
	    {
	      vlib_cli_command_t *d;

	      d = vec_elt_at_index (cm->commands, subs[i].command_index);
	      vlib_cli_output
		(vm, "  %-30v %U", subs[i].name,
		 format_vlib_cli_command_help, d, /* is_long */ 0);
	    }

	  vec_free (subs);
	}
    }

  else if (is_main_dispatch
	   && (unformat (input, "choices") || unformat (input, "?")))
    {
      vlib_cli_command_t *sub, *subs;

      subs = all_subs (cm, 0, parent_command_index);
      vec_sort_with_function (subs, vlib_cli_cmp_command);
      vec_foreach (sub, subs)
	vlib_cli_output (vm, "  %-40U %U",
			 format_vlib_cli_path, sub->path,
			 format_vlib_cli_command_help, sub, /* is_long */ 0);
      vec_free (subs);
    }

  else if (unformat (input, "comment %v", &string))
    {
      vec_free (string);
    }

  else if (unformat (input, "vpplog %v", &string))
    {
      int i;
      /*
       * Delete leading whitespace, so "vpplog { this and that }"
       * and "vpplog this" line up nicely.
       */
      for (i = 0; i < vec_len (string); i++)
	if (string[i] != ' ')
	  break;
      if (i > 0)
	vec_delete (string, i, 0);

      vlib_log_notice (cm->log, "CLI: %v", string);
      vec_free (string);
    }

  else if (unformat (input, "uncomment %U",
		     unformat_vlib_cli_sub_input, &sub_input))
    {
      error =
	vlib_cli_dispatch_sub_commands (vm, cm, &sub_input,
					parent_command_index);
      unformat_free (&sub_input);
    }
  else if (unformat (input, "leak-check %U",
		     unformat_vlib_cli_sub_input, &sub_input))
    {
      u8 *leak_report;
      if (current_traced_heap)
	{
	  void *oldheap;
	  oldheap = clib_mem_set_heap (current_traced_heap);
	  clib_mem_trace (0);
	  clib_mem_set_heap (oldheap);
	  current_traced_heap = 0;
	}
      clib_mem_trace (1);
      error =
	vlib_cli_dispatch_sub_commands (vm, cm, &sub_input,
					parent_command_index);
      unformat_free (&sub_input);

      /* Otherwise, the clib_error_t shows up as a leak... */
      if (error)
	{
	  vlib_cli_output (vm, "%v", error->what);
	  clib_error_free (error);
	  error = 0;
	}

      (void) clib_mem_trace_enable_disable (0);
      leak_report = format (0, "%U", format_clib_mem_heap, 0,
			    1 /* verbose, i.e. print leaks */ );
      clib_mem_trace (0);
      vlib_cli_output (vm, "%v", leak_report);
      vec_free (leak_report);
    }

  else
    if (unformat_user (input, unformat_vlib_cli_sub_command, vm, parent, &c))
    {
      unformat_input_t *si;
      uword has_sub_commands =
	vec_len (c->sub_commands) + vec_len (c->sub_rules) > 0;

      si = input;
      if (unformat_user (input, unformat_vlib_cli_sub_input, &sub_input))
	si = &sub_input;

      if (has_sub_commands)
	error = vlib_cli_dispatch_sub_commands (vm, cm, si, c - cm->commands);

      if (has_sub_commands && !error)
	/* Found valid sub-command. */ ;

      else if (c->function)
	{
	  clib_error_t *c_error;

	  /* Skip white space for benefit of called function. */
	  unformat_skip_white_space (si);

	  if (unformat (si, "?"))
	    {
	      vlib_cli_output (vm, "  %-40U %U", format_vlib_cli_path, c->path, format_vlib_cli_command_help, c,	/* is_long */
			       0);
	    }
	  else
	    {
	      if (PREDICT_FALSE (vm->elog_trace_cli_commands))
		{
                  /* *INDENT-OFF* */
                  ELOG_TYPE_DECLARE (e) =
                    {
                      .format = "cli-cmd: %s",
                      .format_args = "T4",
                    };
                  /* *INDENT-ON* */
		  struct
		  {
		    u32 c;
		  } *ed;
		  ed = ELOG_DATA (vlib_get_elog_main (), e);
		  ed->c = elog_string (vlib_get_elog_main (), "%v", c->path);
		}

	      if (!c->is_mp_safe)
		vlib_worker_thread_barrier_sync (vm);
	      if (PREDICT_FALSE (vec_len (cm->perf_counter_cbs) != 0))
		clib_call_callbacks (cm->perf_counter_cbs, cm,
				     c - cm->commands, 0 /* before */ );

	      c->hit_counter++;
	      c_error = c->function (vm, si, c);

	      if (PREDICT_FALSE (vec_len (cm->perf_counter_cbs) != 0))
		clib_call_callbacks (cm->perf_counter_cbs, cm,
				     c - cm->commands, 1 /* after */ );
	      if (!c->is_mp_safe)
		vlib_worker_thread_barrier_release (vm);

	      if (PREDICT_FALSE (vm->elog_trace_cli_commands))
		{
                  /* *INDENT-OFF* */
                  ELOG_TYPE_DECLARE (e) =
                    {
                      .format = "cli-cmd: %s %s",
                      .format_args = "T4T4",
                    };
                  /* *INDENT-ON* */
		  struct
		  {
		    u32 c, err;
		  } *ed;
		  ed = ELOG_DATA (vlib_get_elog_main (), e);
		  ed->c = elog_string (vlib_get_elog_main (), "%v", c->path);
		  if (c_error)
		    {
		      vec_add1 (c_error->what, 0);
		      ed->err = elog_string (vlib_get_elog_main (),
					     (char *) c_error->what);
		      _vec_len (c_error->what) -= 1;
		    }
		  else
		    ed->err = elog_string (vlib_get_elog_main (), "OK");
		}

	      if (c_error)
		{
		  error =
		    clib_error_return (0, "%v: %v", c->path, c_error->what);
		  clib_error_free (c_error);
		  /* Free sub input. */
		  if (si != input)
		    unformat_free (si);

		  return error;
		}
	    }

	  /* Free any previous error. */
	  clib_error_free (error);
	}

      else if (!error)
	error = clib_error_return (0, "%v: no sub-commands", c->path);

      /* Free sub input. */
      if (si != input)
	unformat_free (si);
    }

  else
    goto unknown;

  return error;

unknown:
  if (parent->path)
    return clib_error_return (0, "%v: unknown input `%U'", parent->path,
			      format_unformat_error, input);
  else
    return clib_error_return (0, "unknown input `%U'", format_unformat_error,
			      input);
}


void vlib_unix_error_report (vlib_main_t *, clib_error_t *)
  __attribute__ ((weak));

void
vlib_unix_error_report (vlib_main_t * vm, clib_error_t * error)
{
}

/* Process CLI input. */
int
vlib_cli_input (vlib_main_t * vm,
		unformat_input_t * input,
		vlib_cli_output_function_t * function, uword function_arg)
{
  vlib_global_main_t *vgm = vlib_get_global_main ();
  vlib_process_t *cp = vlib_get_current_process (vm);
  clib_error_t *error;
  vlib_cli_output_function_t *save_function;
  uword save_function_arg;
  int rv = 0;

  save_function = cp->output_function;
  save_function_arg = cp->output_function_arg;

  cp->output_function = function;
  cp->output_function_arg = function_arg;

  do
    {
      error = vlib_cli_dispatch_sub_commands (vm, &vgm->cli_main, input,
					      /* parent */ 0);
    }
  while (!error && !unformat (input, "%U", unformat_eof));

  if (error)
    {
      vlib_cli_output (vm, "%v", error->what);
      vlib_unix_error_report (vm, error);
      /* clib_error_return is unfortunately often called with a '0'
         return code */
      rv = error->code != 0 ? error->code : -1;
      clib_error_free (error);
    }

  cp->output_function = save_function;
  cp->output_function_arg = save_function_arg;
  return rv;
}

/* Output to current CLI connection. */
void
vlib_cli_output (vlib_main_t * vm, char *fmt, ...)
{
  vlib_process_t *cp = vlib_get_current_process (vm);
  va_list va;
  u8 *s;

  va_start (va, fmt);
  s = va_format (0, fmt, &va);
  va_end (va);

  /* some format functions might return 0
   * e.g. show int addr */
  if (NULL == s)
    return;

  /* Terminate with \n if not present. */
  if (vec_len (s) > 0 && s[vec_len (s) - 1] != '\n')
    vec_add1 (s, '\n');

  if ((!cp) || (!cp->output_function))
    fformat (stdout, "%v", s);
  else
    cp->output_function (cp->output_function_arg, s, vec_len (s));

  vec_free (s);
}

void *vl_msg_push_heap (void) __attribute__ ((weak));
void *
vl_msg_push_heap (void)
{
  return 0;
}

void vl_msg_pop_heap (void *oldheap) __attribute__ ((weak));
void
vl_msg_pop_heap (void *oldheap)
{
}

void *vlib_stats_push_heap (void *) __attribute__ ((weak));
void *
vlib_stats_push_heap (void *notused)
{
  return 0;
}

static clib_error_t *
show_memory_usage (vlib_main_t * vm,
		   unformat_input_t * input, vlib_cli_command_t * cmd)
{
  clib_mem_main_t *mm = &clib_mem_main;
  int verbose __attribute__ ((unused)) = 0;
  int api_segment = 0, stats_segment = 0, main_heap = 0, numa_heaps = 0;
  int map = 0;
  clib_error_t *error;
  u32 index = 0;
  int i;
  uword clib_mem_trace_enable_disable (uword enable);
  uword was_enabled;


  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
    {
      if (unformat (input, "verbose"))
	verbose = 1;
      else if (unformat (input, "api-segment"))
	api_segment = 1;
      else if (unformat (input, "stats-segment"))
	stats_segment = 1;
      else if (unformat (input, "main-heap"))
	main_heap = 1;
      else if (unformat (input, "numa-heaps"))
	numa_heaps = 1;
      else if (unformat (input, "map"))
	map = 1;
      else
	{
	  error = clib_error_return (0, "unknown input `%U'",
				     format_unformat_error, input);
	  return error;
	}
    }

  if ((api_segment + stats_segment + main_heap + numa_heaps + map) == 0)
    return clib_error_return
      (0, "Need one of api-segment, stats-segment, main-heap, numa-heaps "
       "or map");

  if (api_segment)
    {
      void *oldheap = vl_msg_push_heap ();
      was_enabled = clib_mem_trace_enable_disable (0);
      u8 *s_in_svm = format (0, "%U\n", format_clib_mem_heap, 0, 1);
      vl_msg_pop_heap (oldheap);
      u8 *s = vec_dup (s_in_svm);

      oldheap = vl_msg_push_heap ();
      vec_free (s_in_svm);
      clib_mem_trace_enable_disable (was_enabled);
      vl_msg_pop_heap (oldheap);
      vlib_cli_output (vm, "API segment");
      vlib_cli_output (vm, "%v", s);
      vec_free (s);
    }
  if (stats_segment)
    {
      void *oldheap = vlib_stats_push_heap (0);
      was_enabled = clib_mem_trace_enable_disable (0);
      u8 *s_in_svm = format (0, "%U\n", format_clib_mem_heap, 0, 1);
      if (oldheap)
	clib_mem_set_heap (oldheap);
      u8 *s = vec_dup (s_in_svm);

      oldheap = vlib_stats_push_heap (0);
      vec_free (s_in_svm);
      if (oldheap)
	{
	  clib_mem_trace_enable_disable (was_enabled);
	  clib_mem_set_heap (oldheap);
	}
      vlib_cli_output (vm, "Stats segment");
      vlib_cli_output (vm, "%v", s);
      vec_free (s);
    }


  {
    if (main_heap)
      {
	/*
	 * Note: the foreach_vlib_main causes allocator traffic,
	 * so shut off tracing before we go there...
	 */
	was_enabled = clib_mem_trace_enable_disable (0);

	foreach_vlib_main ()
	  {
	    vlib_cli_output (vm, "%sThread %d %s\n", index ? "\n" : "", index,
			     vlib_worker_threads[index].name);
	    vlib_cli_output (vm, "  %U\n", format_clib_mem_heap,
			     mm->per_cpu_mheaps[index], verbose);
	    index++;
	  }

	/* Restore the trace flag */
	clib_mem_trace_enable_disable (was_enabled);
      }
    if (numa_heaps)
      {
	for (i = 0; i < ARRAY_LEN (mm->per_numa_mheaps); i++)
	  {
	    if (mm->per_numa_mheaps[i] == 0)
	      continue;
	    if (mm->per_numa_mheaps[i] == mm->per_cpu_mheaps[i])
	      {
		vlib_cli_output (vm, "Numa %d uses the main heap...", i);
		continue;
	      }
	    was_enabled = clib_mem_trace_enable_disable (0);

	    vlib_cli_output (vm, "Numa %d:", i);
	    vlib_cli<style>.highlight .hll { background-color: #ffffcc }
.highlight .c { color: #888888 } /* Comment */
.highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */
.highlight .k { color: #008800; font-weight: bold } /* Keyword */
.highlight .ch { color: #888888 } /* Comment.Hashbang */
.highlight .cm { color: #888888 } /* Comment.Multiline */
.highlight .cp { color: #cc0000; font-weight: bold } /* Comment.Preproc */
.highlight .cpf { color: #888888 } /* Comment.PreprocFile */
.highlight .c1 { color: #888888 } /* Comment.Single */
.highlight .cs { color: #cc0000; font-weight: bold; background-color: #fff0f0 } /* Comment.Special */
.highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */
.highlight .ge { font-style: italic } /* Generic.Emph */
.highlight .gr { color: #aa0000 } /* Generic.Error */
.highlight .gh { color: #333333 } /* Generic.Heading */
.highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */
.highlight .go { color: #888888 } /* Generic.Output */
.highlight .gp { color: #555555 } /* Generic.Prompt */
.highlight .gs { font-weight: bold } /* Generic.Strong */
.highlight .gu { color: #666666 } /* Generic.Subheading */
.highlight .gt { color: #aa0000 } /* Generic.Traceback */
.highlight .kc { color: #008800; font-weight: bold } /* Keyword.Constant */
.highlight .kd { color: #008800; font-weight: bold } /* Keyword.Declaration */
.highlight .kn { color: #008800; font-weight: bold } /* Keyword.Namespace */
.highlight .kp { color: #008800 } /* Keyword.Pseudo */
.highlight .kr { color: #008800; font-weight: bold } /* Keyword.Reserved */
.highlight .kt { color: #888888; font-weight: bold } /* Keyword.Type */
.highlight .m { color: #0000DD; font-weight: bold } /* Literal.Number */
.highlight .s { color: #dd2200; background-color: #fff0f0 } /* Literal.String */
.highlight .na { color: #336699 } /* Name.Attribute */
.highlight .nb { color: #003388 } /* Name.Builtin */
.highlight .nc { color: #bb0066; font-weight: bold } /* Name.Class */
.highlight .no { color: #003366; font-weight: bold } /* Name.Constant */
.highlight .nd { color: #555555 } /* Name.Decorator */
.highlight .ne { color: #bb0066; font-weight: bold } /* Name.Exception */
.highlight .nf { color: #0066bb; font-weight: bold } /* Name.Function */
.highlight .nl { color: #336699; font-style: italic } /* Name.Label */
.highlight .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */
.highlight .py { color: #336699; font-weight: bold } /* Name.Property */
.highlight .nt { color: #bb0066; font-weight: bold } /* Name.Tag */
.highlight .nv { color: #336699 } /* Name.Variable */
.highlight .ow { color: #008800 } /* Operator.Word */
.highlight .w { color: #bbbbbb } /* Text.Whitespace */
.highlight .mb { color: #0000DD; font-weight: bold } /* Literal.Number.Bin */
.highlight .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */
.highlight .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */
.highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */
.highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */
.highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */
.highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */
.highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */
.highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */
.highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */
.highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */
.highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */
.highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */
.highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */
.highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */
.highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */
.highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */
.highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */
.highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */
.highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */
.highlight .vc { color: #336699 } /* Name.Variable.Class */
.highlight .vg { color: #dd7700 } /* Name.Variable.Global */
.highlight .vi { color: #3333bb } /* Name.Variable.Instance */
.highlight .vm { color: #336699 } /* Name.Variable.Magic */
.highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */</style><div class="highlight"><pre><span></span><span class="cm">/*</span>
<span class="cm"> * Copyright (c) 2015 Cisco and/or its affiliates.</span>
<span class="cm"> * Licensed under the Apache License, Version 2.0 (the &quot;License&quot;);</span>
<span class="cm"> * you may not use this file except in compliance with the License.</span>
<span class="cm"> * You may obtain a copy of the License at:</span>
<span class="cm"> *</span>
<span class="cm"> *     http://www.apache.org/licenses/LICENSE-2.0</span>
<span class="cm"> *</span>
<span class="cm"> * Unless required by applicable law or agreed to in writing, software</span>
<span class="cm"> * distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
<span class="cm"> * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
<span class="cm"> * See the License for the specific language governing permissions and</span>
<span class="cm"> * limitations under the License.</span>
<span class="cm"> */</span>

<span class="cm">/** @cond DOCUMENTATION_IS_IN_BIHASH_DOC_H */</span>

<span class="kt">void</span> <span class="nf">BV</span> <span class="p">(</span><span class="n">clib_bihash_init</span><span class="p">)</span>
  <span class="p">(</span><span class="n">BVT</span> <span class="p">(</span><span class="n">clib_bihash</span><span class="p">)</span> <span class="o">*</span> <span class="n">h</span><span class="p">,</span> <span class="kt">char</span> <span class="o">*</span><span class="n">name</span><span class="p">,</span> <span class="n">u32</span> <span class="n">nbuckets</span><span class="p">,</span> <span class="n">uword</span> <span class="n">memory_size</span><span class="p">)</span>
<span class="p">{</span>
  <span class="kt">void</span> <span class="o">*</span><span class="n">oldheap</span><span class="p">;</span>

  <span class="n">nbuckets</span> <span class="o">=</span> <span class="mi">1</span> <span class="o">&lt;&lt;</span> <span class="p">(</span><span class="n">max_log2</span> <span class="p">(</span><span class="n">nbuckets</span><span class="p">));</span>

  <span class="n">h</span><span class="o">-&gt;</span><span class="n">name</span> <span class="o">=</span> <span class="p">(</span><span class="n">u8</span> <span class="o">*</span><span class="p">)</span> <span class="n">name</span><span class="p">;</span>
  <span class="n">h</span><span class="o">-&gt;</span><span class="n">nbuckets</span> <span class="o">=</span> <span class="n">nbuckets</span><span class="p">;</span>
  <span class="n">h</span><span class="o">-&gt;</span><span class="n">log2_nbuckets</span> <span class="o">=</span> <span class="n">max_log2</span> <span class="p">(</span><span class="n">nbuckets</span><span class="p">);</span>

  <span class="n">h</span><span class="o">-&gt;</span><span class="n">mheap</span> <span class="o">=</span> <span class="n">mheap_alloc</span> <span class="p">(</span><span class="mi">0</span> <span class="cm">/* use VM */</span> <span class="p">,</span> <span class="n">memory_size</span><span class="p">);</span>

  <span class="n">oldheap</span> <span class="o">=</span> <span class="n">clib_mem_set_heap</span> <span class="p">(</span><span class="n">h</span><span class="o">-&gt;</span><span class="n">mheap</span><span class="p">);</span>
  <span class="n">vec_validate_aligned</span> <span class="p">(</span><span class="n">h</span><span class="o">-&gt;</span><span class="n">buckets</span><span class="p">,</span> <span class="n">nbuckets</span> <span class="o">-</span> <span class="mi">1</span><span class="p">,</span> <span class="n">CLIB_CACHE_LINE_BYTES</span><span class="p">);</span>
  <span class="n">h</span><span class="o">-&gt;</span><span class="n">writer_lock</span> <span class="o">=</span> <span class="n">clib_mem_alloc_aligned</span> <span class="p">(</span><span class="n">CLIB_CACHE_LINE_BYTES</span><span class="p">,</span>
					   <span class="n">CLIB_CACHE_LINE_BYTES</span><span class="p">);</span>

  <span class="n">clib_mem_set_heap</span> <span class="p">(</span><span class="n">oldheap</span><span class="p">);</span>
<span class="p">}</span>

<span class="kt">void</span> <span class="nf">BV</span> <span class="p">(</span><span class="n">clib_bihash_free</span><span class="p">)</span> <span class="p">(</span><span class="n">BVT</span> <span class="p">(</span><span class="n">clib_bihash</span><span class="p">)</span> <span class="o">*</span> <span class="n">h</span><span class="p">)</span>
<span class="p">{</span>
  <span class="n">mheap_free</span> <span class="p">(</span><span class="n">h</span><span class="o">-&gt;</span><span class="n">mheap</span><span class="p">);</span>
  <span class="n">memset</span> <span class="p">(</span><span class="n">h</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="k">sizeof</span> <span class="p">(</span><span class="o">*</span><span class="n">h</span><span class="p">));</span>
<span class="p">}</span>

<span class="k">static</span>
<span class="nf">BVT</span> <span class="p">(</span><span class="n">clib_bihash_value</span><span class="p">)</span> <span class="o">*</span>
<span class="n">BV</span> <span class="p">(</span><span class="n">value_alloc</span><span class="p">)</span> <span class="p">(</span><span class="n">BVT</span> <span class="p">(</span><span class="n">clib_bihash</span><span class="p">)</span> <span class="o">*</span> <span class="n">h</span><span class="p">,</span> <span class="n">u32</span> <span class="n">log2_pages</span><span class="p">)</span>
<span class="p">{</span>
  <span class="n">BVT</span> <span class="p">(</span><span class="n">clib_bihash_value</span><span class="p">)</span> <span class="o">*</span> <span class="n">rv</span> <span class="o">=</span> <span class="mi">0</span><span class="p">;</span>
  <span class="kt">void</span> <span class="o">*</span><span class="n">oldheap</span><span class="p">;</span>

  <span class="n">ASSERT</span> <span class="p">(</span><span class="n">h</span><span class="o">-&gt;</span><span class="n">writer_lock</span><span class="p">[</span><span class="mi">0</span><span class="p">]);</span>
  <span class="k">if</span> <span class="p">(</span><span class="n">log2_pages</span> <span class="o">&gt;=</span> <span class="n">vec_len</span> <span class="p">(</span><span class="n">h</span><span class="o">-&gt;</span><span class="n">freelists</span><span class="p">)</span> <span class="o">||</span> <span class="n">h</span><span class="o">-&gt;</span><span class="n">freelists</span><span class="p">[</span><span class="n">log2_pages</span><span class="p">]</span> <span class="o">==</span> <span class="mi">0</span><span class="p">)</span>
    <span class="p">{</span>
      <span class="n">oldheap</span> <span class="o">=</span> <span class="n">clib_mem_set_heap</span> <span class="p">(</span><span class="n">h</span><span class="o">-&gt;</span><span class="n">mheap</span><span class="p">);</span>

      <span class="n">vec_validate</span> <span class="p">(</span><span class="n">h</span><span class="o">-&gt;</span><span class="n">freelists</span><span class="p">,</span> <span class="n">log2_pages</span><span class="p">);</span>
      <span class="n">vec_validate_aligned</span> <span class="p">(</span><span class="n">rv</span><span class="p">,</span> <span class="p">(</span><span class="mi">1</span> <span class="o">&lt;&lt;</span> <span class="n">log2_pages</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">,</span> <span class="n">CLIB_CACHE_LINE_BYTES</span><span class="p">);</span>
      <span class="n">clib_mem_set_heap</span> <span class="p">(</span><span class="n">oldheap</span><span class="p">);</span>
      <span class="k">goto</span> <span class="n">initialize</span><span class="p">;</span>
    <span class="p">}</span>
  <span class="n">rv</span> <span class="o">=</span> <span class="n">h</span><span class="o">-&gt;</span><span class="n">freelists</span><span class="p">[</span><span class="n">log2_pages</span><span class="p">];</span>
  <span class="n">h</span><span class="o">-&gt;</span><span class="n">freelists</span><span class="p">[</span><span class="n">log2_pages</span><span class="p">]</span> <span class="o">=</span> <span class="n">rv</span><span class="o">-&gt;</span><span class="n">next_free</span><span class="p">;</span>

<span class="nl">initialize</span><span class="p">:</span>
  <span class="n">ASSERT</span> <span class="p">(</span><span class="n">rv</span><span class="p">);</span>
  <span class="n">ASSERT</span> <span class="p">(</span><span class="n">vec_len</span> <span class="p">(</span><span class="n">rv</span><span class="p">)</span> <span class="o">==</span> <span class="p">(</span><span class="mi">1</span> <span class="o">&lt;&lt;</span> <span class="n">log2_pages</span><span class="p">));</span>
  <span class="cm">/*</span>
<span class="cm">   * Latest gcc complains that the length arg is zero</span>
<span class="cm">   * if we replace (1&lt;&lt;log2_pages) with vec_len(rv).</span>
<span class="cm">   * No clue.</span>
<span class="cm">   */</span>
  <span class="n">memset</span> <span class="p">(</span><span class="n">rv</span><span class="p">,</span> <span class="mh">0xff</span><span class="p">,</span> <span class="k">sizeof</span> <span class="p">(</span><span class="o">*</span><span class="n">rv</span><span class="p">)</span> <span class="o">*</span> <span class="p">(</span><span class="mi">1</span> <span class="o">&lt;&lt;</span> <span class="n">log2_pages</span><span class="p">));</span>
  <span class="k">return</span> <span class="n">rv</span><span class="p">;</span>
<span class="p">}</span>

<span class="k">static</span> <span class="kt">void</span>
<span class="nf">BV</span> <span class="p">(</span><span class="n">value_free</span><span class="p">)</span> <span class="p">(</span><span class="n">BVT</span> <span class="p">(</span><span class="n">clib_bihash</span><span class="p">)</span> <span class="o">*</span> <span class="n">h</span><span class="p">,</span> <span class="n">BVT</span> <span class="p">(</span><span class="n">clib_bihash_value</span><span class="p">)</span> <span class="o">*</span> <span class="n">v</span><span class="p">)</span>
<span class="p">{</span>
  <span class="n">u32</span> <span class="n">log2_pages</span><span class="p">;</span>

  <span class="n">ASSERT</span> <span class="p">(</span><span class="n">h</span><span class="o">-&gt;</span><span class="n">writer_lock</span><span class="p">[</span><span class="mi">0</span><span class="p">]);</span>

  <span class="n">log2_pages</span> <span class="o">=</span> <span class="n">min_log2</span> <span class="p">(</span><span class="n">vec_len</span> <span class="p">(</span><span class="n">v</span><span class="p">));</span>

  <span class="n">ASSERT</span> <span class="p">(</span><span class="n">vec_len</span> <span class="p">(</span><span class="n">h</span><span class="o">-&gt;</span><span class="n">freelists</span><span class="p">)</span> <span class="o">&gt;</span> <span class="n">log2_pages</span><span class="p">);</span>

  <span class="n">v</span><span class="o">-&gt;</span><span class="n">next_free</span> <span class="o">=</span> <span class="n">h</span><span class="o">-&gt;</span><span class="n">freelists</span><span class="p">[</span><span class="n">log2_pages</span><span class="p">];</span>
  <span class="n">h</span><span class="o">-&gt;</span><span class="n">freelists</span><span class="p">[</span><span class="n">log2_pages</span><span class="p">]</span> <span class="o">=</span> <span class="n">v</span><span class="p">;</span>
<span class="p">}</span>

<span class="k">static</span> <span class="kr">inline</span> <span class="kt">void</span>
<span class="nf">BV</span> <span class="p">(</span><span class="n">make_working_copy</span><span class="p">)</span> <span class="p">(</span><span class="n">BVT</span> <span class="p">(</span><span class="n">clib_bihash</span><span class="p">)</span> <span class="o">*</span> <span class="n">h</span><span class="p">,</span> <span class="n">clib_bihash_bucket_t</span> <span class="o">*</span> <span class="n">b</span><span class="p">)</span>
<span class="p">{</span>
  <span class="n">BVT</span> <span class="p">(</span><span class="n">clib_bihash_value</span><span class="p">)</span> <span class="o">*</span> <span class="n">v</span><span class="p">;</span>
  <span class="n">clib_bihash_bucket_t</span> <span class="n">working_bucket</span> <span class="n">__attribute__</span> <span class="p">((</span><span class="n">aligned</span> <span class="p">(</span><span class="mi">8</span><span class="p">)));</span>
  <span class="kt">void</span> <span class="o">*</span><span class="n">oldheap</span><span class="p">;</span>
  <span class="n">BVT</span> <span class="p">(</span><span class="n">clib_bihash_value</span><span class="p">)</span> <span class="o">*</span> <span class="n">working_copy</span><span class="p">;</span>
  <span class="n">u32</span> <span class="n">cpu_number</span> <span class="o">=</span> <span class="n">os_get_cpu_number</span> <span class="p">();</span>

  <span class="k">if</span> <span class="p">(</span><span class="n">cpu_number</span> <span class="o">&gt;=</span> <span class="n">vec_len</span> <span class="p">(</span><span class="n">h</span><span class="o">-&gt;</span><span class="n">working_copies</span><span class="p">))</span>
    <span class="p">{</span>
      <span class="n">oldheap</span> <span class="o">=</span> <span class="n">clib_mem_set_heap</span> <span class="p">(</span><span class="n">h</span><span class="o">-&gt;</span><span class="n">mheap</span><span class="p">);</span>
      <span class="n">vec_validate</span> <span class="p">(</span><span class="n">h</span><span class="o">-&gt;</span><span class="n">working_copies</span><span class="p">,</span> <span class="n">cpu_number</span><span class="p">);</span>
      <span class="n">clib_mem_set_heap</span> <span class="p">(</span><span class="n">oldheap</span><span class="p">);</span>
    <span class="p">}</span>

  <span class="cm">/*</span>
<span class="cm">   * working_copies are per-cpu so that near-simultaneous</span>
<span class="cm">   * updates from multiple threads will not result in sporadic, spurious</span>
<span class="cm">   * lookup failures.</span>
<span class="cm">   */</span>
  <span class="n">working_copy</span> <span class="o">=</span> <span class="n">h</span><span class="o">-&gt;</span><span class="n">working_copies</span><span class="p">[</span><span class="n">cpu_number</span><span class="p">];</span>

  <span class="n">h</span><span class="o">-&gt;</span><span class="n">saved_bucket</span><span class="p">.</span><span class="n">as_u64</span> <span class="o">=</span> <span class="n">b</span><span class="o">-&gt;</span><span class="n">as_u64</span><span class="p">;</span>
  <span class="n">oldheap</span> <span class="o">=</span> <span class="n">clib_mem_set_heap</span> <span class="p">(</span><span class="n">h</span><span class="o">-&gt;</span><span class="n">mheap</span><span class="p">);</span>

  <span class="k">if</span> <span class="p">((</span><span class="mi">1</span> <span class="o">&lt;&lt;</span> <span class="n">b</span><span class="o">-&gt;</span><span class="n">log2_pages</span><span class="p">)</span> <span class="o">&gt;</span> <span class="n">vec_len</span> <span class="p">(</span><span class="n">working_copy</span><span class="p">))</span>
    <span class="p">{</span>
      <span class="n">vec_validate_aligned</span> <span class="p">(</span><span class="n">working_copy</span><span class="p">,</span> <span class="p">(</span><span class="mi">1</span> <span class="o">&lt;&lt;</span> <span class="n">b</span><span class="o">-&gt;</span><span class="n">log2_pages</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">,</span>
			    <span class="k">sizeof</span> <span class="p">(</span><span class="n">u64</span><span class="p">));</span>
      <span class="n">h</span><span class="o">-&gt;</span><span class="n">working_copies</span><span class="p">[</span><span class="n">cpu_number</span><span class="p">]</span> <span class="o">=</span> <span class="n">working_copy</span><span class="p">;</span>
    <span class="p">}</span>

  <span class="n">_vec_len</span> <span class="p">(</span><span class="n">working_copy</span><span class="p">)</span> <span class="o">=</span> <span class="mi">1</span> <span class="o">&lt;&lt;</span> <span class="n">b</span><span class="o">-&gt;</span><span class="n">log2_pages</span><span class="p">;</span>
  <span class="n">clib_mem_set_heap</span> <span class="p">(</span><span class="n">oldheap</span><span class="p">);</span>

  <span class="n">v</span> <span class="o">=</span> <span class="n">BV</span> <span class="p">(</span><span class="n">clib_bihash_get_value</span><span class="p">)</span> <span class="p">(</span><span class="n">h</span><span class="p">,</span> <span class="n">b</span><span class="o">-&gt;</span><span class="n">offset</span><span class="p">);</span>

  <span class="n">clib_memcpy</span> <span class="p">(</span><span class="n">working_copy</span><span class="p">,</span> <span class="n">v</span><span class="p">,</span> <span class="k">sizeof</span> <span class="p">(</span><span class="o">*</span><span class="n">v</span><span class="p">)</span> <span class="o">*</span> <span class="p">(</span><span class="mi">1</span> <span class="o">&lt;&lt;</span> <span class="n">b</span><span class="o">-&gt;</span><span class="n">log2_pages</span><span class="p">));</span>
  <span class="n">working_bucket</span><span class="p">.</span><span class="n">as_u64</span> <span class="o">=</span> <span class="n">b</span><span class="o">-&gt;</span><span class="n">as_u64</span><span class="p">;</span>
  <span class="n">working_bucket</span><span class="p">.</span><span class="n">offset</span> <span class="o">=</span> <span class="n">BV</span> <span class="p">(</span><span class="n">clib_bihash_get_offset</span><span class="p">)</span> <span class="p">(</span><span class="n">h</span><span class="p">,</span> <span class="n">working_copy</span><span class="p">);</span>
  <span class="n">CLIB_MEMORY_BARRIER</span> <span class="p">();</span>
  <span class="n">b</span><span class="o">-&gt;</span><span class="n">as_u64</span> <span class="o">=</span> <span class="n">working_bucket</span><span class="p">.</span><span class="n">as_u64</span><span class="p">;</span>
  <span class="n">h</span><span class="o">-&gt;</span><span class="n">working_copies</span><span class="p">[</span><span class="n">cpu_number</span><span class="p">]</span> <span class="o">=</span> <span class="n">working_copy</span><span class="p">;</span>
<span class="p">}</span>

<span class="k">static</span>
<span class="nf">BVT</span> <span class="p">(</span><span class="n">clib_bihash_value</span><span class="p">)</span> <span class="o">*</span>
<span class="n">BV</span> <span class="p">(</span><span class="n">split_and_rehash</span><span class="p">)</span>
  <span class="p">(</span><span class="n">BVT</span> <span class="p">(</span><span class="n">clib_bihash</span><span class="p">)</span> <span class="o">*</span> <span class="n">h</span><span class="p">,</span>
   <span class="n">BVT</span> <span class="p">(</span><span class="n">clib_bihash_value</span><span class="p">)</span> <span class="o">*</span> <span class="n">old_values</span><span class="p">,</span> <span class="n">u32</span> <span cl