aboutsummaryrefslogtreecommitdiffstats
path: root/hicn-light/src/config/controlSetWldr.c
blob: 9da40403698807f40e7da09b7a853f8ab17d698d (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
/*
 * Copyright (c) 2017-2019 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <src/config.h>

#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <strings.h>

#include <parc/assert/parc_Assert.h>

#include <parc/algol/parc_Memory.h>

#include <src/config/controlSetDebug.h>
#include <src/core/dispatcher.h>
#include <src/core/forwarder.h>

#include <src/utils/commands.h>
#include <src/utils/utils.h>

static CommandReturn _controlSetWldr_Execute(CommandParser *parser,
                                             CommandOps *ops, PARCList *args);
static CommandReturn _controlSetWldr_HelpExecute(CommandParser *parser,
                                                 CommandOps *ops,
                                                 PARCList *args);

static const char *_commandSetWldr = "set wldr";
static const char *_commandSetWldrHelp = "help set wldr";

// ====================================================

CommandOps *controlSetWldr_Create(ControlState *state) {
  return commandOps_Create(state, _commandSetWldr, NULL,
                           _controlSetWldr_Execute, commandOps_Destroy);
}

CommandOps *controlSetWldr_HelpCreate(ControlState *state) {
  return commandOps_Create(state, _commandSetWldrHelp, NULL,
                           _controlSetWldr_HelpExecute, commandOps_Destroy);
}

// ====================================================

static CommandReturn _controlSetWldr_HelpExecute(CommandParser *parser,
                                                 CommandOps *ops,
                                                 PARCList *args) {
  printf("set wldr <on|off> <connection_id>\n");
  printf("\n");
  return CommandReturn_Success;
}

static CommandReturn _controlSetWldr_Execute(CommandParser *parser,
                                             CommandOps *ops, PARCList *args) {
  ControlState *state = ops->closure;

  if (parcList_Size(args) != 4) {
    _controlSetWldr_HelpExecute(parser, ops, args);
    return CommandReturn_Failure;
  }

  if (((strcmp(parcList_GetAtIndex(args, 0), "set") != 0) ||
       (strcmp(parcList_GetAtIndex(args, 1), "wldr") != 0))) {
    _controlSetWldr_HelpExecute(parser, ops, args);
    return CommandReturn_Failure;
  }

  bool active;
  if (strcmp(parcList_GetAtIndex(args, 2), "on") == 0) {
    active = true;
  } else if (strcmp(parcList_GetAtIndex(args, 2), "off") == 0) {
    active = false;
  } else {
    _controlSetWldr_HelpExecute(parser, ops, args);
    return CommandReturn_Failure;
  }

  // check if valid connid
  const char *symbolicOrConnid = parcList_GetAtIndex(args, 3);

  if (!utils_ValidateSymbolicName(symbolicOrConnid) &&
      !utils_IsNumber(symbolicOrConnid)) {
    printf(
        "ERROR: Invalid symbolic or connid:\nsymbolic name must begin with an "
        "alpha followed by alphanum;\nconnid must be an integer\n");
    return CommandReturn_Failure;
  }

  // allocate command payload
  set_wldr_command *setWldrCommand =
      parcMemory_AllocateAndClear(sizeof(set_wldr_command));
  strcpy(setWldrCommand->symbolicOrConnid, symbolicOrConnid);
  if (active) {
    setWldrCommand->activate = ACTIVATE_ON;
  } else {
    setWldrCommand->activate = ACTIVATE_OFF;
  }

  // send message and receive response
  struct iovec *response = utils_SendRequest(state, SET_WLDR, setWldrCommand,
                                             sizeof(set_wldr_command));

  if (!response) {  // get NULL pointer
    return CommandReturn_Failure;
  }

  parcMemory_Deallocate(&response);  // free iovec pointer
  return CommandReturn_Success;
}
'#n818'>818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195
/*
 *------------------------------------------------------------------
 * Copyright (c) 2016 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *------------------------------------------------------------------
 */

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>

#include <vlib/vlib.h>
#include <vlib/unix/unix.h>
#include <vnet/vnet.h>
#include <vnet/ethernet/ethernet.h>
#include <vnet/gso/gro_func.h>
#include <vnet/gso/hdr_offset_parser.h>
#include <vnet/ip/ip4_packet.h>
#include <vnet/ip/ip6_packet.h>
#include <vnet/tcp/tcp_packet.h>
#include <vnet/udp/udp_packet.h>
#include <vnet/devices/virtio/virtio.h>

#define foreach_virtio_tx_func_error	       \
_(NO_FREE_SLOTS, "no free tx slots")           \
_(TRUNC_PACKET, "packet > buffer size -- truncated in tx ring") \
_(PENDING_MSGS, "pending msgs in tx ring") \
_(INDIRECT_DESC_ALLOC_FAILED, "indirect descriptor allocation failed - packet drop") \
_(OUT_OF_ORDER, "out-of-order buffers in used ring") \
_(GSO_PACKET_DROP, "gso disabled on itf  -- gso packet drop") \
_(CSUM_OFFLOAD_PACKET_DROP, "checksum offload disabled on itf -- csum offload packet drop")

typedef enum
{
#define _(f,s) VIRTIO_TX_ERROR_##f,
  foreach_virtio_tx_func_error
#undef _
    VIRTIO_TX_N_ERROR,
} virtio_tx_func_error_t;

static char *virtio_tx_func_error_strings[] = {
#define _(n,s) s,
  foreach_virtio_tx_func_error
#undef _
};

static u8 *
format_virtio_device (u8 * s, va_list * args)
{
  u32 dev_instance = va_arg (*args, u32);
  int verbose = va_arg (*args, int);
  u32 indent = format_get_indent (s);

  s = format (s, "VIRTIO interface");
  if (verbose)
    {
      s = format (s, "\n%U instance %u", format_white_space, indent + 2,
		  dev_instance);
    }
  return s;
}

typedef struct
{
  u32 buffer_index;
  u32 sw_if_index;
  generic_header_offset_t gho;
  vlib_buffer_t buffer;
} virtio_tx_trace_t;

static u8 *
format_virtio_tx_trace (u8 * s, va_list * va)
{
  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
  CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
  virtio_tx_trace_t *t = va_arg (*va, virtio_tx_trace_t *);
  u32 indent = format_get_indent (s);

  s = format (s, "%Ubuffer 0x%x: %U\n",
	      format_white_space, indent,
	      t->buffer_index, format_vnet_buffer, &t->buffer);
  s =
    format (s, "%U%U\n", format_white_space, indent,
	    format_generic_header_offset, &t->gho);
  s =
    format (s, "%U%U", format_white_space, indent,
	    format_ethernet_header_with_length, t->buffer.pre_data,
	    sizeof (t->buffer.pre_data));
  return s;
}

static void
virtio_tx_trace (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_t *b0,
		 u32 bi, int is_tun)
{
  virtio_tx_trace_t *t;
  t = vlib_add_trace (vm, node, b0, sizeof (t[0]));
  t->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX];
  t->buffer_index = bi;
  if (is_tun)
    {
      int is_ip4 = 0, is_ip6 = 0;

      switch (((u8 *) vlib_buffer_get_current (b0))[0] & 0xf0)
	{
	case 0x40:
	  is_ip4 = 1;
	  break;
	case 0x60:
	  is_ip6 = 1;
	  break;
	default:
	  break;
	}
      vnet_generic_header_offset_parser (b0, &t->gho, 0, is_ip4, is_ip6);
    }
  else
    vnet_generic_header_offset_parser (b0, &t->gho, 1,
				       b0->flags &
				       VNET_BUFFER_F_IS_IP4,
				       b0->flags & VNET_BUFFER_F_IS_IP6);

  clib_memcpy_fast (&t->buffer, b0, sizeof (*b0) - sizeof (b0->pre_data));
  clib_memcpy_fast (t->buffer.pre_data, vlib_buffer_get_current (b0),
		    sizeof (t->buffer.pre_data));
}

static void
virtio_interface_drop_inline (vlib_main_t *vm, virtio_if_t *vif,
			      uword node_index, u32 *buffers, u16 n,
			      virtio_tx_func_error_t error)
{
  vlib_error_count (vm, node_index, error, n);
  vlib_increment_simple_counter (vnet_main.interface_main.sw_if_counters +
				   VNET_INTERFACE_COUNTER_DROP,
				 vm->thread_index, vif->sw_if_index, n);
  vlib_buffer_free (vm, buffers, n);
}

static void
virtio_memset_ring_u32 (u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
{
  ASSERT (n_buffers <= ring_size);

  if (PREDICT_TRUE (start + n_buffers <= ring_size))
    {
      clib_memset_u32 (ring + start, ~0, n_buffers);
    }
  else
    {
      clib_memset_u32 (ring + start, ~0, ring_size - start);
      clib_memset_u32 (ring, ~0, n_buffers - (ring_size - start));
    }
}

static void
virtio_free_used_device_desc_split (vlib_main_t *vm, virtio_vring_t *vring,
				    uword node_index)
{
  u16 used = vring->desc_in_use;
  u16 sz = vring->size;
  u16 mask = sz - 1;
  u16 last = vring->last_used_idx;
  u16 n_left = vring->used->idx - last;
  u16 out_of_order_count = 0;

  if (n_left == 0)
    return;

  while (n_left)
    {
      vring_used_elem_t *e = &vring->used->ring[last & mask];
      u16 slot, n_buffers;
      slot = n_buffers = e->id;

      while (e->id == (n_buffers & mask))
	{
	  n_left--;
	  last++;
	  n_buffers++;
	  vring_desc_t *d = &vring->desc[e->id];
	  u16 next;
	  while (d->flags & VRING_DESC_F_NEXT)
	    {
	      n_buffers++;
	      next = d->next;
	      d = &vring->desc[next];
	    }
	  if (n_left == 0)
	    break;
	  e = &vring->used->ring[last & mask];
	}
      vlib_buffer_free_from_ring (vm, vring->buffers, slot,
				  sz, (n_buffers - slot));
      virtio_memset_ring_u32 (vring->buffers, slot, sz, (n_buffers - slot));
      used -= (n_buffers - slot);

      if (n_left > 0)
	{
	  vlib_buffer_free (vm, &vring->buffers[e->id], 1);
	  vring->buffers[e->id] = ~0;
	  used--;
	  last++;
	  n_left--;
	  out_of_order_count++;
	  vring->flags |= VRING_TX_OUT_OF_ORDER;
	}
    }

  /*
   * Some vhost-backends give buffers back in out-of-order fashion in used ring.
   * It impacts the overall virtio-performance.
   */
  if (out_of_order_count)
    vlib_error_count (vm, node_index, VIRTIO_TX_ERROR_OUT_OF_ORDER,
		      out_of_order_count);

  vring->desc_in_use = used;
  vring->last_used_idx = last;
}

static void
virtio_free_used_device_desc_packed (vlib_main_t *vm, virtio_vring_t *vring,
				     uword node_index)
{
  vring_packed_desc_t *d;
  u16 sz = vring->size;
  u16 last = vring->last_used_idx;
  u16 n_buffers = 0, start;
  u16 flags;

  if (vring->desc_in_use == 0)
    return;

  d = &vring->packed_desc[last];
  flags = d->flags;
  start = d->id;

  while ((flags & VRING_DESC_F_AVAIL) == (vring->used_wrap_counter << 7) &&
	 (flags & VRING_DESC_F_USED) == (vring->used_wrap_counter << 15))
    {
      last++;
      n_buffers++;

      if (last >= sz)
	{
	  last = 0;
	  vring->used_wrap_counter ^= 1;
	}
      d = &vring->packed_desc[last];
      flags = d->flags;
    }

  if (n_buffers)
    {
      vlib_buffer_free_from_ring (vm, vring->buffers, start, sz, n_buffers);
      virtio_memset_ring_u32 (vring->buffers, start, sz, n_buffers);
      vring->desc_in_use -= n_buffers;
      vring->last_used_idx = last;
    }
}

static void
virtio_free_used_device_desc (vlib_main_t *vm, virtio_vring_t *vring,
			      uword node_index, int packed)
{
  if (packed)
    virtio_free_used_device_desc_packed (vm, vring, node_index);
  else
    virtio_free_used_device_desc_split (vm, vring, node_index);

}

static void
set_checksum_offsets (vlib_buffer_t *b, virtio_net_hdr_v1_t *hdr,
		      const int is_l2)
{
  vnet_buffer_oflags_t oflags = vnet_buffer (b)->oflags;

  if (b->flags & VNET_BUFFER_F_IS_IP4)
    {
      ip4_header_t *ip4;
      generic_header_offset_t gho = { 0 };
      vnet_generic_header_offset_parser (b, &gho, is_l2, 1 /* ip4 */ ,
					 0 /* ip6 */ );
      hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
      hdr->csum_start = gho.l4_hdr_offset;	// 0x22;
      if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
	{
	  hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
	}
      else if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
	{
	  hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum);
	}

      /*
       * virtio devices do not support IP4 checksum offload. So driver takes care
       * of it while doing tx.
       */
      ip4 =
	(ip4_header_t *) (vlib_buffer_get_current (b) + gho.l3_hdr_offset);
      if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
	ip4->checksum = ip4_header_checksum (ip4);
    }
  else if (b->flags & VNET_BUFFER_F_IS_IP6)
    {
      generic_header_offset_t gho = { 0 };
      vnet_generic_header_offset_parser (b, &gho, is_l2, 0 /* ip4 */ ,
					 1 /* ip6 */ );
      hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
      hdr->csum_start = gho.l4_hdr_offset;	// 0x36;
      if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
	{
	  hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
	}
      else if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
	{
	  hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum);
	}
    }
}

static void
set_gso_offsets (vlib_buffer_t *b, virtio_net_hdr_v1_t *hdr, const int is_l2)
{
  vnet_buffer_oflags_t oflags = vnet_buffer (b)->oflags;

  if (b->flags & VNET_BUFFER_F_IS_IP4)
    {
      ip4_header_t *ip4;
      generic_header_offset_t gho = { 0 };
      vnet_generic_header_offset_parser (b, &gho, is_l2, 1 /* ip4 */ ,
					 0 /* ip6 */ );
      hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
      hdr->gso_size = vnet_buffer2 (b)->gso_size;
      hdr->hdr_len = gho.hdr_sz;
      hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
      hdr->csum_start = gho.l4_hdr_offset;	// 0x22;
      hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
      ip4 =
	(ip4_header_t *) (vlib_buffer_get_current (b) + gho.l3_hdr_offset);
      /*
       * virtio devices do not support IP4 checksum offload. So driver takes care
       * of it while doing tx.
       */
      if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
	ip4->checksum = ip4_header_checksum (ip4);
    }
  else if (b->flags & VNET_BUFFER_F_IS_IP6)
    {
      generic_header_offset_t gho = { 0 };
      vnet_generic_header_offset_parser (b, &gho, is_l2, 0 /* ip4 */ ,
					 1 /* ip6 */ );
      hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
      hdr->gso_size = vnet_buffer2 (b)->gso_size;
      hdr->hdr_len = gho.hdr_sz;
      hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
      hdr->csum_start = gho.l4_hdr_offset;	// 0x36;
      hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
    }
}

static u16
add_buffer_to_slot (vlib_main_t *vm, vlib_node_runtime_t *node,
		    virtio_if_t *vif, virtio_vring_t *vring, u32 bi,
		    u16 free_desc_count, u16 avail, u16 next, u16 mask,
		    int hdr_sz, int do_gso, int csum_offload, int is_pci,
		    int is_tun, int is_indirect, int is_any_layout)
{
  u16 n_added = 0;
  vring_desc_t *d;
  int is_l2 = !is_tun;
  d = &vring->desc[next];
  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
  virtio_net_hdr_v1_t *hdr = vlib_buffer_get_current (b) - hdr_sz;
  u32 drop_inline = ~0;

  clib_memset_u8 (hdr, 0, hdr_sz);

  if (b->flags & VNET_BUFFER_F_GSO)
    {
      if (do_gso)
	set_gso_offsets (b, hdr, is_l2);
      else
	{
	  drop_inline = VIRTIO_TX_ERROR_GSO_PACKET_DROP;
	  goto done;
	}
    }
  else if (b->flags & VNET_BUFFER_F_OFFLOAD)
    {
      if (csum_offload)
	set_checksum_offsets (b, hdr, is_l2);
      else
	{
	  drop_inline = VIRTIO_TX_ERROR_CSUM_OFFLOAD_PACKET_DROP;
	  goto done;
	}
    }

  if (PREDICT_FALSE (b->flags & VLIB_BUFFER_IS_TRACED))
    {
      virtio_tx_trace (vm, node, b, bi, is_tun);
    }

  if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
    {
      d->addr = ((is_pci) ? vlib_buffer_get_current_pa (vm, b) :
		 pointer_to_uword (vlib_buffer_get_current (b))) - hdr_sz;
      d->len = b->current_length + hdr_sz;
      d->flags = 0;
    }
  else if (is_indirect)
    {
      /*
       * We are using single vlib_buffer_t for indirect descriptor(s)
       * chain. Single descriptor is 16 bytes and vlib_buffer_t
       * has 2048 bytes space. So maximum long chain can have 128
       * (=2048/16) indirect descriptors.
       * It can easily support 65535 bytes of Jumbo frames with
       * each data buffer size of 512 bytes minimum.
       */
      u32 indirect_buffer = 0;
      if (PREDICT_FALSE (vlib_buffer_alloc (vm, &indirect_buffer, 1) == 0))
	{
	  drop_inline = VIRTIO_TX_ERROR_INDIRECT_DESC_ALLOC_FAILED;
	  goto done;
	}

      vlib_buffer_t *indirect_desc = vlib_get_buffer (vm, indirect_buffer);
      indirect_desc->current_data = 0;
      indirect_desc->flags |= VLIB_BUFFER_NEXT_PRESENT;
      indirect_desc->next_buffer = bi;
      bi = indirect_buffer;

      vring_desc_t *id =
	(vring_desc_t *) vlib_buffer_get_current (indirect_desc);
      u32 count = 1;
      if (is_pci)
	{
	  d->addr = vlib_physmem_get_pa (vm, id);
	  id->addr = vlib_buffer_get_current_pa (vm, b) - hdr_sz;

	  /*
	   * If VIRTIO_F_ANY_LAYOUT is not negotiated, then virtio_net_hdr
	   * should be presented in separate descriptor and data will start
	   * from next descriptor.
	   */
	  if (is_any_layout)
	    id->len = b->current_length + hdr_sz;
	  else
	    {
	      id->len = hdr_sz;
	      id->flags = VRING_DESC_F_NEXT;
	      id->next = count;
	      count++;
	      id++;
	      id->addr = vlib_buffer_get_current_pa (vm, b);
	      id->len = b->current_length;
	    }
	  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
	    {
	      id->flags = VRING_DESC_F_NEXT;
	      id->next = count;
	      count++;
	      id++;
	      b = vlib_get_buffer (vm, b->next_buffer);
	      id->addr = vlib_buffer_get_current_pa (vm, b);
	      id->len = b->current_length;
	    }
	}
      else			/* VIRTIO_IF_TYPE_[TAP | TUN] */
	{
	  d->addr = pointer_to_uword (id);
	  /* first buffer in chain */
	  id->addr = pointer_to_uword (vlib_buffer_get_current (b)) - hdr_sz;
	  id->len = b->current_length + hdr_sz;

	  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
	    {
	      id->flags = VRING_DESC_F_NEXT;
	      id->next = count;
	      count++;
	      id++;
	      b = vlib_get_buffer (vm, b->next_buffer);
	      id->addr = pointer_to_uword (vlib_buffer_get_current (b));
	      id->len = b->current_length;
	    }
	}
      id->flags = 0;
      id->next = 0;
      d->len = count * sizeof (vring_desc_t);
      d->flags = VRING_DESC_F_INDIRECT;
    }
  else if (is_pci)
    {
      u16 count = next;
      vlib_buffer_t *b_temp = b;
      u16 n_buffers_in_chain = 1;

      /*
       * Check the length of the chain for the required number of
       * descriptors. Return from here, retry to get more descriptors,
       * if chain length is greater than available descriptors.
       */
      while (b_temp->flags & VLIB_BUFFER_NEXT_PRESENT)
	{
	  n_buffers_in_chain++;
	  b_temp = vlib_get_buffer (vm, b_temp->next_buffer);
	}

      if (n_buffers_in_chain > free_desc_count)
	return n_buffers_in_chain;

      d->addr = vlib_buffer_get_current_pa (vm, b) - hdr_sz;
      d->len = b->current_length + hdr_sz;

      while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
	{
	  d->flags = VRING_DESC_F_NEXT;
	  vring->buffers[count] = bi;
	  b->flags &=
	    ~(VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID);
	  bi = b->next_buffer;
	  b->next_buffer = 0;
	  n_added++;
	  count = (count + 1) & mask;
	  d->next = count;
	  d = &vring->desc[count];
	  b = vlib_get_buffer (vm, bi);
	  d->addr = vlib_buffer_get_current_pa (vm, b);
	  d->len = b->current_length;
	}
      d->flags = 0;
      vring->buffers[count] = bi;
      vring->avail->ring[avail & mask] = next;
      n_added++;
      return n_added;
    }
  else
    {
      ASSERT (0);
    }
  vring->buffers[next] = bi;
  vring->avail->ring[avail & mask] = next;
  n_added++;

done:
  if (drop_inline != ~0)
    virtio_interface_drop_inline (vm, vif, node->node_index, &bi, 1,
				  drop_inline);

  return n_added;
}

static u16
add_buffer_to_slot_packed (vlib_main_t *vm, vlib_node_runtime_t *node,
			   virtio_if_t *vif, virtio_vring_t *vring, u32 bi,
			   u16 next, int hdr_sz, int do_gso, int csum_offload,
			   int is_pci, int is_tun, int is_indirect,
			   int is_any_layout)
{
  u16 n_added = 0, flags = 0;
  int is_l2 = !is_tun;
  vring_packed_desc_t *d = &vring->packed_desc[next];
  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
  virtio_net_hdr_v1_t *hdr = vlib_buffer_get_current (b) - hdr_sz;
  u32 drop_inline = ~0;

  clib_memset (hdr, 0, hdr_sz);

  if (b->flags & VNET_BUFFER_F_GSO)
    {
      if (do_gso)
	set_gso_offsets (b, hdr, is_l2);
      else
	{
	  drop_inline = VIRTIO_TX_ERROR_GSO_PACKET_DROP;
	  goto done;
	}
    }
  else if (b->flags & VNET_BUFFER_F_OFFLOAD)
    {
      if (csum_offload)
	set_checksum_offsets (b, hdr, is_l2);
      else
	{
	  drop_inline = VIRTIO_TX_ERROR_CSUM_OFFLOAD_PACKET_DROP;
	  goto done;
	}
    }
  if (PREDICT_FALSE (b->flags & VLIB_BUFFER_IS_TRACED))
    {
      virtio_tx_trace (vm, node, b, bi, is_tun);
    }

  if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
    {
      d->addr =
	((is_pci) ? vlib_buffer_get_current_pa (vm,
						b) :
	 pointer_to_uword (vlib_buffer_get_current (b))) - hdr_sz;
      d->len = b->current_length + hdr_sz;
    }
  else if (is_indirect)
    {
      /*
       * We are using single vlib_buffer_t for indirect descriptor(s)
       * chain. Single descriptor is 16 bytes and vlib_buffer_t
       * has 2048 bytes space. So maximum long chain can have 128
       * (=2048/16) indirect descriptors.
       * It can easily support 65535 bytes of Jumbo frames with
       * each data buffer size of 512 bytes minimum.
       */
      u32 indirect_buffer = 0;
      if (PREDICT_FALSE (vlib_buffer_alloc (vm, &indirect_buffer, 1) == 0))
	{
	  drop_inline = VIRTIO_TX_ERROR_INDIRECT_DESC_ALLOC_FAILED;
	  goto done;
	}

      vlib_buffer_t *indirect_desc = vlib_get_buffer (vm, indirect_buffer);
      indirect_desc->current_data = 0;
      indirect_desc->flags |= VLIB_BUFFER_NEXT_PRESENT;
      indirect_desc->next_buffer = bi;
      bi = indirect_buffer;

      vring_packed_desc_t *id =
	(vring_packed_desc_t *) vlib_buffer_get_current (indirect_desc);
      u32 count = 1;
      if (is_pci)
	{
	  d->addr = vlib_physmem_get_pa (vm, id);
	  id->addr = vlib_buffer_get_current_pa (vm, b) - hdr_sz;

	  /*
	   * If VIRTIO_F_ANY_LAYOUT is not negotiated, then virtio_net_hdr
	   * should be presented in separate descriptor and data will start
	   * from next descriptor.
	   */
	  if (is_any_layout)
	    id->len = b->current_length + hdr_sz;
	  else
	    {
	      id->len = hdr_sz;
	      id->flags = 0;
	      id->id = 0;
	      count++;
	      id++;
	      id->addr = vlib_buffer_get_current_pa (vm, b);
	      id->len = b->current_length;
	    }
	  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
	    {
	      id->flags = 0;
	      id->id = 0;
	      count++;
	      id++;
	      b = vlib_get_buffer (vm, b->next_buffer);
	      id->addr = vlib_buffer_get_current_pa (vm, b);
	      id->len = b->current_length;
	    }
	}
      id->flags = 0;
      id->id = 0;
      d->len = count * sizeof (vring_packed_desc_t);
      flags = VRING_DESC_F_INDIRECT;
    }
  else
    {
      ASSERT (0);
    }
  if (vring->avail_wrap_counter)
    {
      flags |= VRING_DESC_F_AVAIL;
      flags &= ~VRING_DESC_F_USED;
    }
  else
    {
      flags &= ~VRING_DESC_F_AVAIL;
      flags |= VRING_DESC_F_USED;
    }

  d->id = next;
  d->flags = flags;
  vring->buffers[next] = bi;
  n_added++;

done:
  if (drop_inline != ~0)
    virtio_interface_drop_inline (vm, vif, node->node_index, &bi, 1,
				  drop_inline);

  return n_added;
}

static uword
virtio_interface_tx_packed_gso_inline (vlib_main_t *vm,
				       vlib_node_runtime_t *node,
				       virtio_if_t *vif, virtio_if_type_t type,
				       virtio_vring_t *vring, u32 *buffers,
				       u16 n_left, const int do_gso,
				       const int csum_offload)
{
  int is_pci = (type == VIRTIO_IF_TYPE_PCI);
  int is_tun = (type == VIRTIO_IF_TYPE_TUN);
  int is_indirect =
    ((vif->features & VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC)) != 0);
  int is_any_layout =
    ((vif->features & VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT)) != 0);
  const int hdr_sz = vif->virtio_net_hdr_sz;
  u16 sz = vring->size;
  u16 used, next, n_buffers = 0, n_buffers_left = 0;
  u16 n_vectors = n_left;


  used = vring->desc_in_use;
  next = vring->desc_next;

  if (vif->packet_buffering)
    {
      n_buffers = n_buffers_left = virtio_vring_n_buffers (vring->buffering);

      while (n_buffers_left && used < sz)
	{
	  u16 n_added = 0;

	  u32 bi = virtio_vring_buffering_read_from_front (vring->buffering);
	  if (bi == ~0)
	    break;
	  n_added = add_buffer_to_slot_packed (
	    vm, node, vif, vring, bi, next, hdr_sz, do_gso, csum_offload,
	    is_pci, is_tun, is_indirect, is_any_layout);
	  n_buffers_left--;
	  if (PREDICT_FALSE (n_added == 0))
	    continue;

	  used++;
	  next++;
	  if (next >= sz)
	    {
	      next = 0;
	      vring->avail_wrap_counter ^= 1;
	    }
	}
    }

  while (n_left && used < sz)
    {
      u16 n_added = 0;

      n_added = add_buffer_to_slot_packed (
	vm, node, vif, vring, buffers[0], next, hdr_sz, do_gso, csum_offload,
	is_pci, is_tun, is_indirect, is_any_layout);
      buffers++;
      n_left--;
      if (PREDICT_FALSE (n_added == 0))
	continue;

      used++;
      next++;
      if (next >= sz)
	{
	  next = 0;
	  vring->avail_wrap_counter ^= 1;
	}
    }

  if (n_left != n_vectors || n_buffers != n_buffers_left)
    {
      CLIB_MEMORY_STORE_BARRIER ();
      vring->desc_next = next;
      vring->desc_in_use = used;
      CLIB_MEMORY_BARRIER ();
      if (vring->device_event->flags != VRING_EVENT_F_DISABLE)
	virtio_kick (vm, vring, vif);
    }

  return n_left;
}

static void
virtio_find_free_desc (virtio_vring_t *vring, u16 size, u16 mask, u16 req,
		       u16 next, u32 *first_free_desc_index,
		       u16 *free_desc_count)
{
  u16 start = 0;
  /* next is used as hint: from where to start looking */
  for (u16 i = 0; i < size; i++, next++)
    {
      if (vring->buffers[next & mask] == ~0)
	{
	  if (*first_free_desc_index == ~0)
	    {
	      *first_free_desc_index = (next & mask);
	      start = i;
	      (*free_desc_count)++;
	      req--;
	      if (req == 0)
		break;
	    }
	  else
	    {
	      if (start + *free_desc_count == i)
		{
		  (*free_desc_count)++;
		  req--;
		  if (req == 0)
		    break;
		}
	      else
		break;
	    }
	}
    }
}

static u16
virtio_interface_tx_split_gso_inline (vlib_main_t *vm,
				      vlib_node_runtime_t *node,
				      virtio_if_t *vif, virtio_if_type_t type,
				      virtio_vring_t *vring, u32 *buffers,
				      u16 n_left, int do_gso, int csum_offload)
{
  u16 used, next, avail, n_buffers = 0, n_buffers_left = 0;
  int is_pci = (type == VIRTIO_IF_TYPE_PCI);
  int is_tun = (type == VIRTIO_IF_TYPE_TUN);
  int is_indirect =
    ((vif->features & VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC)) != 0);
  int is_any_layout =
    ((vif->features & VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT)) != 0);
  u16 sz = vring->size;
  int hdr_sz = vif->virtio_net_hdr_sz;
  u16 mask = sz - 1;
  u16 n_vectors = n_left;

  used = vring->desc_in_use;
  next = vring->desc_next;
  avail = vring->avail->idx;

  u16 free_desc_count = 0;

  if (PREDICT_FALSE (vring->flags & VRING_TX_OUT_OF_ORDER))
    {
      u32 first_free_desc_index = ~0;

      virtio_find_free_desc (vring, sz, mask, n_left, next,
			     &first_free_desc_index, &free_desc_count);

      if (free_desc_count)
	next = first_free_desc_index;
    }
  else
    free_desc_count = sz - used;

  if (vif->packet_buffering)
    {
      n_buffers = n_buffers_left = virtio_vring_n_buffers (vring->buffering);

      while (n_buffers_left && free_desc_count)
	{
	  u16 n_added = 0;

	  u32 bi = virtio_vring_buffering_read_from_front (vring->buffering);
	  if (bi == ~0)
	    break;

	  n_added = add_buffer_to_slot (vm, node, vif, vring, bi,
					free_desc_count, avail, next, mask,
					hdr_sz, do_gso, csum_offload, is_pci,
					is_tun, is_indirect, is_any_layout);
	  if (PREDICT_FALSE (n_added == 0))
	    {
	      n_buffers_left--;
	      continue;
	    }
	  else if (PREDICT_FALSE (n_added > free_desc_count))
	    break;

	  avail++;
	  next = (next + n_added) & mask;
	  used += n_added;
	  n_buffers_left--;
	  free_desc_count -= n_added;
	}
    }

  while (n_left && free_desc_count)
    {
      u16 n_added = 0;

      n_added =
	add_buffer_to_slot (vm, node, vif, vring, buffers[0], free_desc_count,
			    avail, next, mask, hdr_sz, do_gso, csum_offload,
			    is_pci, is_tun, is_indirect, is_any_layout);

      if (PREDICT_FALSE (n_added == 0))
	{
	  buffers++;
	  n_left--;
	  continue;
	}
      else if (PREDICT_FALSE (n_added > free_desc_count))
	break;

      avail++;
      next = (next + n_added) & mask;
      used += n_added;
      buffers++;
      n_left--;
      free_desc_count -= n_added;
    }

  if (n_left != n_vectors || n_buffers != n_buffers_left)
    {
      clib_atomic_store_seq_cst (&vring->avail->idx, avail);
      vring->desc_next = next;
      vring->desc_in_use = used;
      if ((clib_atomic_load_seq_cst (&vring->used->flags) &
	   VRING_USED_F_NO_NOTIFY) == 0)
	virtio_kick (vm, vring, vif);
    }

  return n_left;
}

static u16
virtio_interface_tx_gso_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
				virtio_if_t *vif, virtio_if_type_t type,
				virtio_vring_t *vring, u32 *buffers,
				u16 n_left, int packed, int do_gso,
				int csum_offload)
{
  if (packed)
    return virtio_interface_tx_packed_gso_inline (vm, node, vif, type, vring,
						  buffers, n_left,
						  do_gso, csum_offload);
  else
    return virtio_interface_tx_split_gso_inline (vm, node, vif, type, vring,
						 buffers, n_left,
						 do_gso, csum_offload);
}

static u16
virtio_interface_tx_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
			    virtio_if_t *vif, virtio_vring_t *vring,
			    virtio_if_type_t type, u32 *buffers, u16 n_left,
			    int packed)
{
  vnet_main_t *vnm = vnet_get_main ();
  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vif->hw_if_index);

  if (hw->caps & VNET_HW_INTERFACE_CAP_SUPPORTS_TCP_GSO)
    return virtio_interface_tx_gso_inline (vm, node, vif, type, vring,
					   buffers, n_left, packed,
					   1 /* do_gso */ ,
					   1 /* checksum offload */ );
  else if (hw->caps & VNET_HW_INTERFACE_CAP_SUPPORTS_L4_TX_CKSUM)
    return virtio_interface_tx_gso_inline (vm, node, vif, type, vring,
					   buffers, n_left, packed,
					   0 /* no do_gso */ ,
					   1 /* checksum offload */ );
  else
    return virtio_interface_tx_gso_inline (vm, node, vif, type, vring,
					   buffers, n_left, packed,
					   0 /* no do_gso */ ,
					   0 /* no checksum offload */ );
}

VNET_DEVICE_CLASS_TX_FN (virtio_device_class) (vlib_main_t * vm,
					       vlib_node_runtime_t * node,
					       vlib_frame_t * frame)
{
  virtio_main_t *nm = &virtio_main;
  vnet_interface_output_runtime_t *rund = (void *) node->runtime_data;
  virtio_if_t *vif = pool_elt_at_index (nm->interfaces, rund->dev_instance);
  u16 qid = vm->thread_index % vif->num_txqs;
  virtio_vring_t *vring = vec_elt_at_index (vif->txq_vrings, qid);
  u16 n_left = frame->n_vectors;
  u32 *buffers = vlib_frame_vector_args (frame);
  u32 to[GRO_TO_VECTOR_SIZE (n_left)];
  int packed = vif->is_packed;
  u16 n_vectors = frame->n_vectors;

  clib_spinlock_lock_if_init (&vring->lockp);

  if (vif->packet_coalesce)
    {
      n_vectors = n_left =
	vnet_gro_inline (vm, vring->flow_table, buffers, n_left, to);
      buffers = to;
    }

  u16 retry_count = 2;

retry:
  /* free consumed buffers */
  virtio_free_used_device_desc (vm, vring, node->node_index, packed);

  if (vif->type == VIRTIO_IF_TYPE_TAP)
    n_left = virtio_interface_tx_inline (vm, node, vif, vring,
					 VIRTIO_IF_TYPE_TAP,
					 &buffers[n_vectors - n_left],
					 n_left, packed);
  else if (vif->type == VIRTIO_IF_TYPE_PCI)
    n_left = virtio_interface_tx_inline (vm, node, vif, vring,
					 VIRTIO_IF_TYPE_PCI,
					 &buffers[n_vectors - n_left],
					 n_left, packed);
  else if (vif->type == VIRTIO_IF_TYPE_TUN)
    n_left = virtio_interface_tx_inline (vm, node, vif, vring,
					 VIRTIO_IF_TYPE_TUN,
					 &buffers[n_vectors - n_left],
					 n_left, packed);
  else
    ASSERT (0);

  if (n_left && retry_count--)
    goto retry;

  if (vif->packet_buffering && n_left)
    {
      u16 n_buffered = virtio_vring_buffering_store_packets (vring->buffering,
							     &buffers
							     [n_vectors
							      - n_left],
							     n_left);
      n_left -= n_buffered;
    }
  if (n_left)
    virtio_interface_drop_inline (vm, vif, node->node_index,
				  &buffers[n_vectors - n_left], n_left,
				  VIRTIO_TX_ERROR_NO_FREE_SLOTS);

  clib_spinlock_unlock_if_init (&vring->lockp);

  return frame->n_vectors - n_left;
}

static void
virtio_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index,
				u32 node_index)
{
  virtio_main_t *apm = &virtio_main;
  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
  virtio_if_t *vif = pool_elt_at_index (apm->interfaces, hw->dev_instance);

  /* Shut off redirection */
  if (node_index == ~0)
    {
      vif->per_interface_next_index = node_index;
      return;
    }

  vif->per_interface_next_index =
    vlib_node_add_next (vlib_get_main (), virtio_input_node.index,
			node_index);
}

static void
virtio_clear_hw_interface_counters (u32 instance)
{
  /* Nothing for now */
}

static void
virtio_set_rx_interrupt (virtio_if_t *vif, virtio_vring_t *vring)
{
  if (vif->is_packed)
    vring->driver_event->flags &= ~VRING_EVENT_F_DISABLE;
  else
    vring->avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
}

static void
virtio_set_rx_polling (virtio_if_t *vif, virtio_vring_t *vring)
{
  if (vif->is_packed)
    vring->driver_event->flags |= VRING_EVENT_F_DISABLE;
  else
    vring->avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
}

static clib_error_t *
virtio_interface_rx_mode_change (vnet_main_t * vnm, u32 hw_if_index, u32 qid,
				 vnet_hw_if_rx_mode mode)
{
  vlib_main_t *vm = vnm->vlib_main;
  virtio_main_t *mm = &virtio_main;
  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
  virtio_if_t *vif = pool_elt_at_index (mm->interfaces, hw->dev_instance);
  virtio_vring_t *rx_vring = vec_elt_at_index (vif->rxq_vrings, qid);

  if (vif->type == VIRTIO_IF_TYPE_PCI && !(vif->support_int_mode))
    {
      virtio_set_rx_polling (vif, rx_vring);
      return clib_error_return (0, "interrupt mode is not supported");
    }

  if (mode == VNET_HW_IF_RX_MODE_POLLING)
    {
      if (vif->packet_coalesce || vif->packet_buffering)
	{
	  if (mm->interrupt_queues_count > 0)
	    mm->interrupt_queues_count--;
	  if (mm->interrupt_queues_count == 0)
	    vlib_process_signal_event (vm,
				       virtio_send_interrupt_node.index,
				       VIRTIO_EVENT_STOP_TIMER, 0);
	}
      virtio_set_rx_polling (vif, rx_vring);
    }
  else
    {
      if (vif->packet_coalesce || vif->packet_buffering)
	{
	  mm->interrupt_queues_count++;
	  if (mm->interrupt_queues_count == 1)
	    vlib_process_signal_event (vm,
				       virtio_send_interrupt_node.index,
				       VIRTIO_EVENT_START_TIMER, 0);
	}
      virtio_set_rx_interrupt (vif, rx_vring);
    }

  rx_vring->mode = mode;

  return 0;
}

static clib_error_t *
virtio_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
{
  virtio_main_t *mm = &virtio_main;
  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
  virtio_if_t *vif = pool_elt_at_index (mm->interfaces, hw->dev_instance);

  if (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP)
    {
      vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP;
      vnet_hw_interface_set_flags (vnm, vif->hw_if_index,
				   VNET_HW_INTERFACE_FLAG_LINK_UP);
    }
  else
    {
      vif->flags &= ~VIRTIO_IF_FLAG_ADMIN_UP;
      vnet_hw_interface_set_flags (vnm, vif->hw_if_index, 0);
    }
  return 0;
}

static clib_error_t *
virtio_subif_add_del_function (vnet_main_t * vnm,
			       u32 hw_if_index,
			       struct vnet_sw_interface_t *st, int is_add)
{
  /* Nothing for now */
  return 0;
}

/* *INDENT-OFF* */
VNET_DEVICE_CLASS (virtio_device_class) = {
  .name = "virtio",
  .format_device_name = format_virtio_device_name,
  .format_device = format_virtio_device,
  .format_tx_trace = format_virtio_tx_trace,
  .tx_function_n_errors = VIRTIO_TX_N_ERROR,
  .tx_function_error_strings = virtio_tx_func_error_strings,
  .rx_redirect_to_node = virtio_set_interface_next_node,
  .clear_counters = virtio_clear_hw_interface_counters,
  .admin_up_down_function = virtio_interface_admin_up_down,
  .subif_add_del_function = virtio_subif_add_del_function,
  .rx_mode_change_function = virtio_interface_rx_mode_change,
};

/* *INDENT-ON* */

/*
 * fd.io coding-style-patch-verification: ON
 *
 * Local Variables:
 * eval: (c-set-style "gnu")
 * End:
 */