summaryrefslogtreecommitdiffstats
path: root/doxygen/Makefile
AgeCommit message (Expand)AuthorFilesLines
2020-09-21build: remove opensuse build infraDave Wallace1-3/+0
2019-11-15build: fix docs/doxygen targetsDave Wallace1-2/+5
2019-11-05docs: fix 'make doxygen' under python3Paul Vinciguerra1-3/+3
2019-07-03doxygen: improve .md file discoveryDave Barach1-0/+2
2018-08-03API: Remove legacy vlibsocket code.Ole Troan1-1/+0
2017-12-15apps: refactor uri and update build infraFlorin Coras1-1/+0
2017-09-28General documentation updatesChris Luke1-2/+2
2017-06-09Sample plugin: Add sample plugin documentationRay Kinsella1-2/+1
2017-05-10doxygen: Fix some pathsChris Luke1-1/+3
2017-03-29Bugfixing and documentation for SRv6Pablo Camarillo1-3/+3
2017-02-22Add ref to test framework docs in doxygen output.Dave Wallace1-5/+1
2017-02-02Added support for openSUSEMarco Varlese1-1/+4
2017-01-11Remove vcgn pluginDamjan Marion1-2/+1
2017-01-10Revert "vppctl: bash completion for vppctl commands"Damjan Marion1-8/+17
2017-01-09vppctl: bash completion for vppctl commandsPadraig Connolly1-17/+8
2017-01-01Move java,lua api and remaining plugins to src/Damjan Marion1-4/+3
2016-12-28Repair Doxygen build infrastructureChris Luke1-11/+27
2016-11-28Add support for using documentation siphons in multiple waysChris Luke1-11/+25
2016-09-23Enable doc building on MacOSChris Luke1-1/+25
2016-09-21Refactor pre-Doxy siphon scripts; VPP-396Chris Luke1-12/+36
2016-09-20Add structure to some of the documentation; VPP-223Chris Luke1-1/+5
2016-09-09Check for zero-sized Graphvix config file on Ubuntu; VPP-396Chris Luke1-1/+2
2016-09-09On Ubuntu check for graphviz system configChris Luke1-0/+3
2016-09-07VPP-346 Improve Doxygen include path mechanismChris Luke1-22/+54
2016-09-06VPP-346 More VPP doc fixesChris Luke1-7/+30
2016-09-02VPP-221 Improve doxygen dependency checkChris Luke1-1/+5
2016-09-01VPP-346 A swathe of doc fixesChris Luke1-0/+4
2016-08-31VPP-221 Loosen Doxygen CLI command struct parserChris Luke1-0/+1
2016-08-31VPP-221 CLI auto-documentation infrastructureChris Luke1-0/+122
47 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 
/*
 * Copyright (c) 2020 Intel and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <vlib/vlib.h>
#include <vnet/plugin/plugin.h>
#include <vpp/app/version.h>

#include "crypto_sw_scheduler.h"

int
crypto_sw_scheduler_set_worker_crypto (u32 worker_idx, u8 enabled)
{
  crypto_sw_scheduler_main_t *cm = &crypto_sw_scheduler_main;
  vlib_thread_main_t *tm = vlib_get_thread_main ();
  crypto_sw_scheduler_per_thread_data_t *ptd = 0;
  u32 count = 0, i = vlib_num_workers () > 0;

  if (worker_idx >= vlib_num_workers ())
    {
      return VNET_API_ERROR_INVALID_VALUE;
    }

  for (; i < tm->n_vlib_mains; i++)
    {
      ptd = cm->per_thread_data + i;
      count += ptd->self_crypto_enabled;
    }

  if (enabled || count > 1)
    {
      cm->per_thread_data[vlib_get_worker_thread_index
			  (worker_idx)].self_crypto_enabled = enabled;
    }
  else				/* cannot disable all crypto workers */
    {
      return VNET_API_ERROR_INVALID_VALUE_2;
    }
  return 0;
}

static void
crypto_sw_scheduler_key_handler (vlib_main_t * vm, vnet_crypto_key_op_t kop,
				 vnet_crypto_key_index_t idx)
{
  crypto_sw_scheduler_main_t *cm = &crypto_sw_scheduler_main;
  vnet_crypto_key_t *key = vnet_crypto_get_key (idx);

  vec_validate (cm->keys, idx);

  if (key->type == VNET_CRYPTO_KEY_TYPE_LINK)
    {
      if (kop == VNET_CRYPTO_KEY_OP_DEL)
	{
	  cm->keys[idx].index_crypto = UINT32_MAX;
	  cm->keys[idx].index_integ = UINT32_MAX;
	}
      else
	{
	  cm->keys[idx] = *key;
	}
    }
}

static int
crypto_sw_scheduler_frame_enqueue (vlib_main_t * vm,
				   vnet_crypto_async_frame_t * frame)
{
  crypto_sw_scheduler_main_t *cm = &crypto_sw_scheduler_main;
  crypto_sw_scheduler_per_thread_data_t *ptd
    = vec_elt_at_index (cm->per_thread_data, vm->thread_index);
  crypto_sw_scheduler_queue_t *q = ptd->queues[frame->op];
  u64 head = q->head;

  if (q->jobs[head & CRYPTO_SW_SCHEDULER_QUEUE_MASK])
    {
      u32 n_elts = frame->n_elts, i;
      for (i = 0; i < n_elts; i++)
	frame->elts[i].status = VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR;
      return -1;
    }
  q->jobs[head & CRYPTO_SW_SCHEDULER_QUEUE_MASK] = frame;
  head += 1;
  CLIB_MEMORY_STORE_BARRIER ();
  q->head = head;
  return 0;
}

static_always_inline vnet_crypto_async_frame_t *
crypto_sw_scheduler_get_pending_frame (crypto_sw_scheduler_queue_t * q)
{
  vnet_crypto_async_frame_t *f;
  u32 i;
  u32 tail = q->tail;
  u32 head = q->head;

  for (i = tail; i < head; i++)
    {
      f = q->jobs[i & CRYPTO_SW_SCHEDULER_QUEUE_MASK];
      if (!f)
	continue;
      if (clib_atomic_bool_cmp_and_swap
	  (&f->state, VNET_CRYPTO_FRAME_STATE_PENDING,
	   VNET_CRYPTO_FRAME_STATE_WORK_IN_PROGRESS))
	{
	  return f;
	}
    }
  return NULL;
}

static_always_inline vnet_crypto_async_frame_t *
crypto_sw_scheduler_get_completed_frame (crypto_sw_scheduler_queue_t * q)
{
  vnet_crypto_async_frame_t *f = 0;
  if (q->jobs[q->tail & CRYPTO_SW_SCHEDULER_QUEUE_MASK]
      && q->jobs[q->tail & CRYPTO_SW_SCHEDULER_QUEUE_MASK]->state
      >= VNET_CRYPTO_FRAME_STATE_SUCCESS)
    {
      u32 tail = q->tail;
      CLIB_MEMORY_STORE_BARRIER ();
      q->tail++;
      f = q->jobs[tail & CRYPTO_SW_SCHEDULER_QUEUE_MASK];
      q->jobs[tail & CRYPTO_SW_SCHEDULER_QUEUE_MASK] = 0;
    }
  return f;
}

static_always_inline void
cryptodev_sw_scheduler_sgl (vlib_main_t * vm,
			    crypto_sw_scheduler_per_thread_data_t * ptd,
			    vlib_buffer_t * b, vnet_crypto_op_t * op,
			    i32 offset, i32 len)
{
  vnet_crypto_op_chunk_t *ch;
  vlib_buffer_t *nb = b;
  u32 n_chunks = 0;
  u32 chunk_index = vec_len (ptd->chunks);

  op->flags |= VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS;

  while (len)
    {
      if (nb->current_data + nb->current_length > offset)
	{
	  vec_add2 (ptd->chunks, ch, 1);
	  ch->src = ch->dst = nb->data + offset;
	  ch->len
	    = clib_min (nb->current_data + nb->current_length - offset, len);
	  len -= ch->len;
	  offset = 0;
	  n_chunks++;
	  if (!len)
	    break;
	}
      if (offset)
	offset -= nb->current_data + nb->current_length;
      if (nb->flags & VLIB_BUFFER_NEXT_PRESENT)
	nb = vlib_get_buffer (vm, nb->next_buffer);
      else
	break;
    }

  ASSERT (offset == 0 && len == 0);
  op->chunk_index = chunk_index;
  op->n_chunks = n_chunks;
}

static_always_inline void
crypto_sw_scheduler_convert_aead (vlib_main_t * vm,
				  crypto_sw_scheduler_per_thread_data_t * ptd,
				  vnet_crypto_async_frame_elt_t * fe,
				  u32 index, u32 bi,
				  vnet_crypto_op_id_t op_id, u16 aad_len,
				  u8 tag_len)
{
  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
  vnet_crypto_op_t *op = 0;

  if (fe->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS)
    {
      vec_add2 (ptd->chained_crypto_ops, op, 1);
      cryptodev_sw_scheduler_sgl (vm, ptd, b, op, fe->crypto_start_offset,
				  fe->crypto_total_length);
    }
  else
    {
      vec_add2 (ptd->crypto_ops, op, 1);
      op->src = op->dst = b->data + fe->crypto_start_offset;
      op->len = fe->crypto_total_length;
    }

  op->op = op_id;
  op->tag = fe->tag;
  op->flags = fe->flags;
  op->key_index = fe->key_index;
  op->iv = fe->iv;
  op->aad = fe->aad;
  op->aad_len = aad_len;
  op->tag_len = tag_len;
  op->user_data = index;
}

static_always_inline void
crypto_sw_scheduler_convert_link_crypto (vlib_main_t * vm,
					 crypto_sw_scheduler_per_thread_data_t
					 * ptd, vnet_crypto_key_t * key,
					 vnet_crypto_async_frame_elt_t * fe,
					 u32 index, u32 bi,
					 vnet_crypto_op_id_t crypto_op_id,
					 vnet_crypto_op_id_t integ_op_id,
					 u32 digest_len, u8 is_enc)
{
  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
  vnet_crypto_op_t *crypto_op = 0, *integ_op = 0;

  if (fe->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS)
    {
      vec_add2 (ptd->chained_crypto_ops, crypto_op, 1);
      vec_add2 (ptd->chained_integ_ops, integ_op, 1);
      cryptodev_sw_scheduler_sgl (vm, ptd, b, crypto_op,
				  fe->crypto_start_offset,
				  fe->crypto_total_length);
      cryptodev_sw_scheduler_sgl (vm, ptd, b, integ_op,
				  fe->integ_start_offset,
				  fe->crypto_total_length +
				  fe->integ_length_adj);
    }
  else
    {
      vec_add2 (ptd->crypto_ops, crypto_op, 1);
      vec_add2 (ptd->integ_ops, integ_op, 1);
      crypto_op->src = crypto_op->dst = b->data + fe->crypto_start_offset;
      crypto_op->len = fe->crypto_total_length;
      integ_op->src = integ_op->dst = b->data + fe->integ_start_offset;
      integ_op->len = fe->crypto_total_length + fe->integ_length_adj;
    }

  crypto_op->op = crypto_op_id;
  crypto_op->iv = fe->iv;
  crypto_op->key_index = key->index_crypto;
  crypto_op->user_data = 0;
  integ_op->op = integ_op_id;
  integ_op->digest = fe->digest;
  integ_op->digest_len = digest_len;
  integ_op->key_index = key->index_integ;
  crypto_op->flags = integ_op->flags = 0;
  if (is_enc)
    crypto_op->flags |= VNET_CRYPTO_OP_FLAG_INIT_IV;
  else
    integ_op->flags |= VNET_CRYPTO_OP_FLAG_HMAC_CHECK;
  crypto_op->user_data = integ_op->user_data = index;
}

static_always_inline void
process_ops (vlib_main_t * vm, vnet_crypto_async_frame_t * f,
	     vnet_crypto_op_t * ops, u8 * state)
{
  u32 n_fail, n_ops = vec_len (ops);
  vnet_crypto_op_t *op = ops;

  if (n_ops == 0)
    return;

  n_fail = n_ops - vnet_crypto_process_ops (vm, op, n_ops);

  while (n_fail)
    {
      ASSERT (op - ops < n_ops);

      if (op->status != VNET_CRYPTO_OP_STATUS_COMPLETED)
	{
	  f->elts[op->user_data].status = op->status;
	  *state = VNET_CRYPTO_FRAME_STATE_ELT_ERROR;
	  n_fail--;
	}
      op++;
    }
}

static_always_inline void
process_chained_ops (vlib_main_t * vm, vnet_crypto_async_frame_t * f,
		     vnet_crypto_op_t * ops, vnet_crypto_op_chunk_t * chunks,
		     u8 * state)
{
  u32 n_fail, n_ops = vec_len (ops);
  vnet_crypto_op_t *op = ops;

  if (n_ops == 0)
    return;

  n_fail = n_ops - vnet_crypto_process_chained_ops (vm, op, chunks, n_ops);

  while (n_fail)
    {
      ASSERT (op - ops < n_ops);

      if (op->status != VNET_CRYPTO_OP_STATUS_COMPLETED)
	{
	  f->elts[op->user_data].status = op->status;
	  *state = VNET_CRYPTO_FRAME_STATE_ELT_ERROR;
	  n_fail--;
	}
      op++;
    }
}

static_always_inline vnet_crypto_async_frame_t *
crypto_sw_scheduler_dequeue_aead (vlib_main_t * vm,
				  vnet_crypto_async_op_id_t async_op_id,
				  vnet_crypto_op_id_t sync_op_id, u8 tag_len,
				  u8 aad_len, u32 * nb_elts_processed,
				  u32 * enqueue_thread_idx)
{
  crypto_sw_scheduler_main_t *cm = &crypto_sw_scheduler_main;
  crypto_sw_scheduler_per_thread_data_t *ptd = 0;
  crypto_sw_scheduler_queue_t *q = 0;
  vnet_crypto_async_frame_t *f = 0;
  vnet_crypto_async_frame_elt_t *fe;
  u32 *bi;
  u32 n_elts;
  int i = 0;
  u8 state = VNET_CRYPTO_FRAME_STATE_SUCCESS;

  if (cm->per_thread_data[vm->thread_index].self_crypto_enabled)
    {
      /* *INDENT-OFF* */
      vec_foreach_index (i, cm->per_thread_data)
      {
        ptd = cm->per_thread_data + i;
        q = ptd->queues[async_op_id];
        f = crypto_sw_scheduler_get_pending_frame (q);
        if (f)
          break;
      }
      /* *INDENT-ON* */
    }

  ptd = cm->per_thread_data + vm->thread_index;

  if (f)
    {
      *nb_elts_processed = n_elts = f->n_elts;
      fe = f->elts;
      bi = f->buffer_indices;

      vec_reset_length (ptd->crypto_ops);
      vec_reset_length (ptd->chained_crypto_ops);
      vec_reset_length (ptd->chunks);

      while (n_elts--)
	{
	  if (n_elts > 1)
	    CLIB_PREFETCH (fe + 1, CLIB_CACHE_LINE_BYTES, LOAD);

	  crypto_sw_scheduler_convert_aead (vm, ptd, fe, fe - f->elts, bi[0],
					    sync_op_id, aad_len, tag_len);
	  bi++;
	  fe++;
	}

      process_ops (vm, f, ptd->crypto_ops, &state);
      process_chained_ops (vm, f, ptd->chained_crypto_ops, ptd->chunks,
			   &state);
      f->state = state;
      *enqueue_thread_idx = f->enqueue_thread_index;
    }

  return crypto_sw_scheduler_get_completed_frame (ptd->queues[async_op_id]);
}

static_always_inline vnet_crypto_async_frame_t *
crypto_sw_scheduler_dequeue_link (vlib_main_t * vm,
				  vnet_crypto_async_op_id_t async_op_id,
				  vnet_crypto_op_id_t sync_crypto_op_id,
				  vnet_crypto_op_id_t sync_integ_op_id,
				  u16 digest_len, u8 is_enc,
				  u32 * nb_elts_processed,
				  u32 * enqueue_thread_idx)
{
  crypto_sw_scheduler_main_t *cm = &crypto_sw_scheduler_main;
  crypto_sw_scheduler_per_thread_data_t *ptd = 0;
  crypto_sw_scheduler_queue_t *q = 0;
  vnet_crypto_async_frame_t *f = 0;
  vnet_crypto_async_frame_elt_t *fe;
  u32 *bi;
  u32 n_elts;
  int i = 0;
  u8 state = VNET_CRYPTO_FRAME_STATE_SUCCESS;

  if (cm->per_thread_data[vm->thread_index].self_crypto_enabled)
    {
      /* *INDENT-OFF* */
      vec_foreach_index (i, cm->per_thread_data)
      {
        ptd = cm->per_thread_data + i;
        q = ptd->queues[async_op_id];
        f = crypto_sw_scheduler_get_pending_frame (q);
        if (f)
          break;
      }
      /* *INDENT-ON* */
    }

  ptd = cm->per_thread_data + vm->thread_index;

  if (f)
    {
      vec_reset_length (ptd->crypto_ops);
      vec_reset_length (ptd->integ_ops);
      vec_reset_length (ptd->chained_crypto_ops);
      vec_reset_length (ptd->chained_integ_ops);
      vec_reset_length (ptd->chunks);

      *nb_elts_processed = n_elts = f->n_elts;
      fe = f->elts;
      bi = f->buffer_indices;

      while (n_elts--)
	{
	  if (n_elts > 1)
	    CLIB_PREFETCH (fe + 1, CLIB_CACHE_LINE_BYTES, LOAD);

	  crypto_sw_scheduler_convert_link_crypto (vm, ptd,
						   cm->keys + fe->key_index,
						   fe, fe - f->elts, bi[0],
						   sync_crypto_op_id,
						   sync_integ_op_id,
						   digest_len, is_enc);
	  bi++;
	  fe++;
	}

      if (is_enc)
	{
	  process_ops (vm, f, ptd->crypto_ops, &state);
	  process_chained_ops (vm, f, ptd->chained_crypto_ops, ptd->chunks,
			       &state);
	  process_ops (vm, f, ptd->integ_ops, &state);
	  process_chained_ops (vm, f, ptd->chained_integ_ops, ptd->chunks,
			       &state);
	}
      else
	{
	  process_ops (vm, f, ptd->integ_ops, &state);
	  process_chained_ops (vm, f, ptd->chained_integ_ops, ptd->chunks,
			       &state);
	  process_ops (vm, f, ptd->crypto_ops, &state);
	  process_chained_ops (vm, f, ptd->chained_crypto_ops, ptd->chunks,
			       &state);
	}

      f->state = state;
      *enqueue_thread_idx = f->enqueue_thread_index;
    }

  return crypto_sw_scheduler_get_completed_frame (ptd->queues[async_op_id]);
}

static clib_error_t *
sw_scheduler_set_worker_crypto (vlib_main_t * vm, unformat_input_t * input,
				vlib_cli_command_t * cmd)
{
  unformat_input_t _line_input, *line_input = &_line_input;
  u32 worker_index;
  u8 crypto_enable;
  int rv;

  /* Get a line of input. */
  if (!unformat_user (input, unformat_line_input, line_input))
    return 0;

  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
    {
      if (unformat (line_input, "worker %u", &worker_index))
	{
	  if (unformat (line_input, "crypto"))
	    {
	      if (unformat (line_input, "on"))
		crypto_enable = 1;
	      else if (unformat (line_input, "off"))
		crypto_enable = 0;
	      else
		return (clib_error_return (0, "unknown input '%U'",
					   format_unformat_error,
					   line_input));
	    }
	  else
	    return (clib_error_return (0, "unknown input '%U'",
				       format_unformat_error, line_input));
	}
      else
	return (clib_error_return (0, "unknown input '%U'",
				   format_unformat_error, line_input));
    }

  rv = crypto_sw_scheduler_set_worker_crypto (worker_index, crypto_enable);
  if (rv == VNET_API_ERROR_INVALID_VALUE)
    {
      return (clib_error_return (0, "invalid worker idx: %d", worker_index));
    }
  else if (rv == VNET_API_ERROR_INVALID_VALUE_2)
    {
      return (clib_error_return (0, "cannot disable all crypto workers"));
    }
  return 0;
}

/*?
 * This command sets if worker will do crypto processing.
 *
 * @cliexpar
 * Example of how to set worker crypto processing off:
 * @cliexstart{set sw_scheduler worker 0 crypto off}
 * @cliexend
 ?*/
/* *INDENT-OFF* */
VLIB_CLI_COMMAND (cmd_set_sw_scheduler_worker_crypto, static) = {
  .path = "set sw_scheduler",
  .short_help = "set sw_scheduler worker <idx> crypto <on|off>",
  .function = sw_scheduler_set_worker_crypto,
  .is_mp_safe = 1,
};
/* *INDENT-ON* */

static clib_error_t *
sw_scheduler_show_workers (vlib_main_t * vm, unformat_input_t * input,
			   vlib_cli_command_t * cmd)
{
  crypto_sw_scheduler_main_t *cm = &crypto_sw_scheduler_main;
  u32 i;

  vlib_cli_output (vm, "%-7s%-20s%-8s", "ID", "Name", "Crypto");
  for (i = 1; i < vlib_thread_main.n_vlib_mains; i++)
    {
      vlib_cli_output (vm, "%-7d%-20s%-8s", vlib_get_worker_index (i),
		       (vlib_worker_threads + i)->name,
		       cm->
		       per_thread_data[i].self_crypto_enabled ? "on" : "off");
    }

  return 0;
}

/*?
 * This command displays sw_scheduler workers.
 *
 * @cliexpar
 * Example of how to show workers:
 * @cliexstart{show sw_scheduler workers}
 * @cliexend
 ?*/
/* *INDENT-OFF* */
VLIB_CLI_COMMAND (cmd_show_sw_scheduler_workers, static) = {
  .path = "show sw_scheduler workers",
  .short_help = "show sw_scheduler workers",
  .function = sw_scheduler_show_workers,
  .is_mp_safe = 1,
};
/* *INDENT-ON* */

clib_error_t *
sw_scheduler_cli_init (vlib_main_t * vm)
{
  return 0;
}

VLIB_INIT_FUNCTION (sw_scheduler_cli_init);

/* *INDENT-OFF* */
#define _(n, s, k, t, a)                                                      \
  static vnet_crypto_async_frame_t                                            \
      *crypto_sw_scheduler_frame_dequeue_##n##_TAG_##t##_AAD_##a##_enc (      \
          vlib_main_t *vm, u32 *nb_elts_processed, u32 * thread_idx)          \
  {                                                                           \
    return crypto_sw_scheduler_dequeue_aead (                                 \
        vm, VNET_CRYPTO_OP_##n##_TAG##t##_AAD##a##_ENC,                       \
        VNET_CRYPTO_OP_##n##_ENC, t, a, nb_elts_processed, thread_idx);       \
  }                                                                           \
  static vnet_crypto_async_frame_t                                            \
      *crypto_sw_scheduler_frame_dequeue_##n##_TAG_##t##_AAD_##a##_dec (      \
          vlib_main_t *vm, u32 *nb_elts_processed, u32 * thread_idx)          \
  {                                                                           \
    return crypto_sw_scheduler_dequeue_aead (                                 \
        vm, VNET_CRYPTO_OP_##n##_TAG##t##_AAD##a##_DEC,                       \
        VNET_CRYPTO_OP_##n##_DEC, t, a, nb_elts_processed, thread_idx);       \
  }
foreach_crypto_aead_async_alg
#undef _

#define _(c, h, s, k, d)                                                      \
  static vnet_crypto_async_frame_t                                            \
      *crypto_sw_scheduler_frame_dequeue_##c##_##h##_TAG##d##_enc (           \
          vlib_main_t *vm, u32 *nb_elts_processed, u32 * thread_idx)          \
  {                                                                           \
    return crypto_sw_scheduler_dequeue_link (                                 \
        vm, VNET_CRYPTO_OP_##c##_##h##_TAG##d##_ENC,                          \
        VNET_CRYPTO_OP_##c##_ENC, VNET_CRYPTO_OP_##h##_HMAC, d, 1,            \
        nb_elts_processed, thread_idx);                                       \
  }                                                                           \
  static vnet_crypto_async_frame_t                                            \
      *crypto_sw_scheduler_frame_dequeue_##c##_##h##_TAG##d##_dec (           \
          vlib_main_t *vm, u32 *nb_elts_processed, u32 * thread_idx)          \
  {                                                                           \
    return crypto_sw_scheduler_dequeue_link (                                 \
        vm, VNET_CRYPTO_OP_##c##_##h##_TAG##d##_DEC,                          \
        VNET_CRYPTO_OP_##c##_DEC, VNET_CRYPTO_OP_##h##_HMAC, d, 0,            \
        nb_elts_processed, thread_idx);                                       \
  }
    foreach_crypto_link_async_alg
#undef _
        /* *INDENT-ON* */

crypto_sw_scheduler_main_t crypto_sw_scheduler_main;
clib_error_t *
crypto_sw_scheduler_init (vlib_main_t * vm)
{
  crypto_sw_scheduler_main_t *cm = &crypto_sw_scheduler_main;
  vlib_thread_main_t *tm = vlib_get_thread_main ();
  clib_error_t *error = 0;
  crypto_sw_scheduler_per_thread_data_t *ptd;

  u32 queue_size = CRYPTO_SW_SCHEDULER_QUEUE_SIZE * sizeof (void *)
    + sizeof (crypto_sw_scheduler_queue_t);

  vec_validate_aligned (cm->per_thread_data, tm->n_vlib_mains - 1,
			CLIB_CACHE_LINE_BYTES);

  vec_foreach (ptd, cm->per_thread_data)
  {
    ptd->self_crypto_enabled = 1;
    u32 i;
    for (i = 0; i < VNET_CRYPTO_ASYNC_OP_N_IDS; i++)
      {
	crypto_sw_scheduler_queue_t *q
	  = clib_mem_alloc_aligned (queue_size, CLIB_CACHE_LINE_BYTES);
	ASSERT (q != 0);
	ptd->queues[i] = q;
	clib_memset_u8 (q, 0, queue_size);
      }
  }

  cm->crypto_engine_index =
    vnet_crypto_register_engine (vm, "sw_scheduler", 100,
				 "SW Scheduler Async Engine");

  vnet_crypto_register_key_handler (vm, cm->crypto_engine_index,
				    crypto_sw_scheduler_key_handler);

  crypto_sw_scheduler_api_init (vm);

  /* *INDENT-OFF* */
#define _(n, s, k, t, a)                                                      \
  vnet_crypto_register_async_handler (                                        \
      vm, cm->crypto_engine_index,                                            \
      VNET_CRYPTO_OP_##n##_TAG##t##_AAD##a##_ENC,                             \
      crypto_sw_scheduler_frame_enqueue,                                      \
      crypto_sw_scheduler_frame_dequeue_##n##_TAG_##t##_AAD_##a##_enc);       \
  vnet_crypto_register_async_handler (                                        \
      vm, cm->crypto_engine_index,                                            \
      VNET_CRYPTO_OP_##n##_TAG##t##_AAD##a##_DEC,                             \
      crypto_sw_scheduler_frame_enqueue,                                      \
      crypto_sw_scheduler_frame_dequeue_##n##_TAG_##t##_AAD_##a##_dec);
  foreach_crypto_aead_async_alg
#undef _

#define _(c, h, s, k, d)                                                      \
  vnet_crypto_register_async_handler (                                        \
      vm, cm->crypto_engine_index, VNET_CRYPTO_OP_##c##_##h##_TAG##d##_ENC,   \
      crypto_sw_scheduler_frame_enqueue,                                      \
      crypto_sw_scheduler_frame_dequeue_##c##_##h##_TAG##d##_enc);            \
  vnet_crypto_register_async_handler (                                        \
      vm, cm->crypto_engine_index, VNET_CRYPTO_OP_##c##_##h##_TAG##d##_DEC,   \
      crypto_sw_scheduler_frame_enqueue,                                      \
      crypto_sw_scheduler_frame_dequeue_##c##_##h##_TAG##d##_dec);
      foreach_crypto_link_async_alg
#undef _
      /* *INDENT-ON* */

  if (error)
    vec_free (cm->per_thread_data);

  return error;
}

/* *INDENT-OFF* */
VLIB_INIT_FUNCTION (crypto_sw_scheduler_init) = {
  .runs_after = VLIB_INITS ("vnet_crypto_init"),
};

VLIB_PLUGIN_REGISTER () = {
  .version = VPP_BUILD_VER,
  .description = "SW Scheduler Crypto Async Engine plugin",
};
/* *INDENT-ON* */

/*
 * fd.io coding-style-patch-verification: ON
 *
 * Local Variables:
 * eval: (c-set-style "gnu")
 * End:
 */