/*
 * Copyright (c) 2015 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/*
  Copyright (c) 2001, 2002, 2003 Eliot Dresselhaus

  Permission is hereby granted, free of charge, to any person obtaining
  a copy of this software and associated documentation files (the
  "Software"), to deal in the Software without restriction, including
  without limitation the rights to use, copy, modify, merge, publish,
  distribute, sublicense, and/or sell copies of the Software, and to
  permit persons to whom the Software is furnished to do so, subject to
  the following conditions:

  The above copyright notice and this permission notice shall be
  included in all copies or substantial portions of the Software.

  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

#ifndef included_random_h
#define included_random_h

#include <vppinfra/clib.h>
#include <vppinfra/vec.h>	/* for vec_resize */
#include <vppinfra/format.h>	/* for unformat_input_t */

/** \file
    Linear Congruential Random Number Generator

    This specific random number generator is described in
    "Numerical Recipes in C", 2nd edition, page 284. If you need
    random numbers with really excellent statistics, take a look
    at Chapter 7...

    By definition, a linear congruential random number generator
    is of the form: rand[i+1] = a*rand[i] + c (mod m) for specific
    values of (a,c,m).

    In this case, choose m = 2**32 and use the low-order 32-bits of
    the 64-bit product a*N[i]. Knuth suggests the use of a=1664525,
    H.W. Lewis has tested C=1013904223 extensively. This routine is
    reputedly as good as any 32-bit LCRN, and costs only a single
    multiply-add.

    Several variants: 32/64-bit, machine word width,
    f64 on the closed interval [0,1].
*/

/** \brief 32-bit random number generator */
always_inline u32
random_u32 (u32 * seed)
{
  *seed = (1664525 * *seed) + 1013904223;
  return *seed;
}

/* External test routine. */
int test_random_main (unformat_input_t * input);

/** \brief Maximum value returned by random_u32() */
always_inline u32
random_u32_max (void)
{
  return 0xffffffff;
}

#ifdef CLIB_UNIX

#include <unistd.h>		/* for getpid */

/** \brief Default random seed (unix/linux user-mode) */
always_inline uword
random_default_seed (void)
{
  return getpid ();
}

#endif

#ifdef CLIB_LINUX_KERNEL

#include <linux/sched.h>	/* for jiffies */

/** \brief Default random seed (Linux kernel) */
always_inline uword
random_default_seed (void)
{
  return jiffies;
}

#endif

#ifdef CLIB_STANDALONE
e
/*
 * Copyright (c) 2018 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef __included_vmnet_vmnet_h__
#define __included_vmnet_vmnet_h__

#define foreach_vmxnet3_tx_func_error	       \
  _(ERROR_PACKETS, "error packets") \
  _(NO_FREE_SLOTS, "no free tx slots")

typedef enum
{
#define _(f,s) VMXNET3_TX_ERROR_##f,
  foreach_vmxnet3_tx_func_error
#undef _
    VMXNET3_TX_N_ERROR,
} vmxnet3_tx_func_error_t;

#define foreach_vmxnet3_rxmode_flags \
  _(0, UCAST, "unicast") \
  _(1, MCAST, "multicast")		   \
  _(2, BCAST, "broadcast") \
  _(3, ALL_MULTI, "all multicast") \
  _(4, PROMISC, "promiscuous")

enum
{
#define _(a, b, c) VMXNET3_RXMODE_##b = (1 << a),
  foreach_vmxnet3_rxmode_flags
#undef _
};

/* BAR 0 */
#define VMXNET3_REG_IMR     0x0000	/* Interrupt Mask Register */
#define VMXNET3_REG_TXPROD  0x0600	/* Tx Producer Index */
#define VMXNET3_REG_RXPROD  0x0800	/* Rx Producer Index for ring 1 */
#define VMXNET3_REG_RXPROD2 0x0A00	/* Rx Producer Index for ring 2 */


/* BAR 1 */
#define VMXNET3_REG_VRRS 0x0000	/* VMXNET3 Revision Report Selection */
#define VMXNET3_REG_UVRS 0x0008	/* UPT Version Report Selection */
#define VMXNET3_REG_DSAL 0x0010	/* Driver Shared Address Low */
#define VMXNET3_REG_DSAH 0x0018	/* Driver Shared Address High */
#define VMXNET3_REG_CMD  0x0020	/* Command */
#define VMXNET3_REG_MACL 0x0028	/* MAC Address Low */
#define VMXNET3_REG_MACH 0x0030	/* MAC Address High */
#define VMXNET3_REG_ICR  0x0038	/* Interrupt Cause Register */
#define VMXNET3_REG_ECR  0x0040	/* Event Cause Register */

#define VMXNET3_VLAN_LEN 4
#define VMXNET3_FCS_LEN  4
#define VMXNET3_MTU (1514 + VMXNET3_VLAN_LEN + VMXNET3_FCS_LEN)

#define VMXNET3_RXF_BTYPE (1 << 14)	/* rx body buffer type */
#define VMXNET3_RXF_GEN   (1 << 31)	/* rx generation */
#define VMXNET3_RXCF_IP6  (1 << 20)	/* rx ip6 packet */
#define VMXNET3_RXCF_IP4  (1 << 21)	/* rx ip4 packet */
#define VMXNET3_RXCF_GEN  (1 << 31)	/* rx completion generation */
#define VMXNET3_RXC_INDEX (0xFFF)	/* rx completion index mask */

#define VMXNET3_TXF_GEN  (1 << 14)	/* tx generation */
#define VMXNET3_TXF_EOP  (1 << 12)	/* tx end of packet */
#define VMXNET3_TXF_CQ   (1 << 13)	/* tx completion request */
#define VMXNET3_TXCF_GEN (1 << 31)	/* tx completion generation */
#define VMXNET3_TXC_INDEX (0xFFF)	/* tx completion index mask */

#define VMXNET3_RX_RING_SIZE 2
#define VMXNET3_INPUT_REFILL_THRESHOLD 32
#define VMXNET3_NUM_TX_DESC 1024
#define VMXNET3_NUM_TX_COMP VMXNET3_NUM_TX_DESC
#define VMXNET3_NUM_RX_DESC 1024
#define VMXNET3_NUM_RX_COMP VMXNET3_NUM_RX_DESC

#define VMXNET3_VERSION_MAGIC 0x69505845
#define VMXNET3_SHARED_MAGIC  0xbabefee1
#define VMXNET3_VERSION_SELECT     1
#define VMXNET3_UPT_VERSION_SELECT 1
#define VMXNET3_MAX_INTRS          25
#define VMXNET3_IC_DISABLE_ALL     0x1

#define VMXNET3_GOS_BITS_32     (1 << 0)
#define VMXNET3_GOS_BITS_64     (2 << 0)
#define VMXNET3_GOS_TYPE_LINUX  (1 << 2)
#define VMXNET3_RXCL_LEN_MASK   (0x3FFF)	// 14 bits
#define VMXNET3_RXCL_ERROR      (1 << 14)
#define VMXNET3_RXCI_EOP        (1 << 14)
#define VMXNET3_RXCI_SOP        (1 << 15)

#define foreach_vmxnet3_device_flags \
  _(0, INITIALIZED, "initialized") \
  _(1, ERROR, "error")		   \
  _(2, ADMIN_UP, "admin-up") \
  _(3, IOVA, "iova") \
  _(4, LINK_UP, "link-up") \
  _(5, SHARED_TXQ_LOCK, "shared-txq-lock") \
  _(6, ELOG, "elog")

enum
{
#define _(a, b, c) VMXNET3_DEVICE_F_##b = (1 << a),
  foreach_vmxnet3_device_flags
#undef _
};

#define foreach_vmxnet3_set_cmds \
  _(0, ACTIVATE_DEV, "activate device") \
  _(1, QUIESCE_DEV, "quiesce device") \
  _(2, RESET_DEV, "reset device") \
  _(3, UPDATE_RX_MODE, "update rx mode") \
  _(4, UPDATE_MAC_FILTERS, "update mac filters") \
  _(5, UPDATE_VLAN_FILTERS, "update vlan filters") \
  _(6, UPDATE_RSSIDT, "update rss idt") \
  _(7, UPDATE_IML, "update iml") \
  _(8, UPDATE_PMCFG, "update pm cfg") \
  _(9, UPDATE_FEATURE, "update feature") \
  _(10, STOP_EMULATION, "stop emulation") \
  _(11, LOAD_PLUGIN, "load plugin") \
  _(12, ACTIVATE_VF, "activate vf") \
  _(13, RESERVED3, "reserved 3") \
  _(14, RESERVED4, "reservced 4") \
  _(15, REGISTER_MEMREGS, "register mem regs")

enum
{
#define _(a, b, c) VMXNET3_CMD_##b = (a + 0xCAFE0000),
  foreach_vmxnet3_set_cmds
#undef _
};

#define foreach_vmxnet3_get_cmds \
  _(0, GET_QUEUE_STATUS, "get queue status") \
  _(1, GET_STATS, "get stats") \
  _(2, GET_LINK, "get link") \
  _(3, GET_PERM_MAC_LO, "get perm mac lo") \
  _(4, GET_PERM_MAC_HI, "get perm mac hi") \
  _(5, GET_DID_LO, "get did lo") \
  _(6, GET_DID_HI, "get did hi") \
  _(7, GET_DEV_EXTRA_INFO, "get dev extra info") \
  _(8, GET_CONF_INTR, "get conf intr") \
  _(9, GET_ADAPTIVE_RING_INFO, "get adaptive ring info") \
  _(10, GET_TXDATA_DESC_SIZE, "gte txdata desc size") \
  _(11, RESERVED5, "reserved5")

enum
{
#define _(a, b, c) VMXNET3_CMD_##b = (a + 0xF00D0000),
  foreach_vmxnet3_get_cmds
#undef _
};

typedef CLIB_PACKED (struct
		     {
		     u32 version; u32 guest_info; u32 version_support;
		     u32 upt_version_support; u64 upt_features;
		     u64 driver_data_address; u64 queue_desc_address;
		     u32 driver_data_len; u32 queue_desc_len;
		     u32 mtu;
		     u16 max_num_rx_sg; u8 num_tx_queues; u8 num_rx_queues;
		     u32 pad[4];
		     }) vmxnet3_misc_config;

typedef CLIB_PACKED (struct
		     {
		     u8 mask_mode;
		     u8 num_intrs;
		     u8 event_intr_index;
		     u8 moderation_level[VMXNET3_MAX_INTRS]; u32 control;
		     u32 pad[2];
		     }) vmxnet3_interrupt_config;

typedef CLIB_PACKED (struct
		     {
		     u32 mode;
		     u16 multicast_len;
		     u16 pad; u64 multicast_address; u8 vlan_filter[512];
		     }) vmxnet3_rx_filter_config;

typedef CLIB_PACKED (struct
		     {
		     u32 version; u32 length;
		     u64 address;
		     }) vmxnet3_variable_config;

typedef CLIB_PACKED (struct
		     {
		     u32 magic;
		     u32 pad;
		     vmxnet3_misc_config misc;
		     vmxnet3_interrupt_config interrupt;
		     vmxnet3_rx_filter_config rx_filter;
		     vmxnet3_variable_config rss;
		     vmxnet3_variable_config pattern;
		     vmxnet3_variable_config plugin; u32 ecr;
		     u32 pad1[5];
		     }) vmxnet3_shared;

typedef CLIB_PACKED (struct
		     {
		     u8 stopped;
		     u8 pad[3];
		     u32 error;
		     }) vmxnet3_queue_status;

typedef CLIB_PACKED (struct
		     {
		     u32 num_deferred; u32 threshold;
		     u64 pad;
		     }) vmxnet3_tx_queue_control;

typedef CLIB_PACKED (struct
		     {
		     u64 desc_address;
		     u64 data_address;
		     u64 comp_address; u64 driver_data_address; u64 pad;
		     u32 num_desc;
		     u32 num_data;
		     u32 num_comp; u32 driver_data_len; u8 intr_index;
		     u8 pad1[7];
		     }) vmxnet3_tx_queue_config;

typedef CLIB_PACKED (struct
		     {
		     u64 tso_pkts;
		     u64 tso_bytes;
		     u64 ucast_pkts; u64 ucast_bytes; u64 mcast_pkts;
		     u64 mcast_bytes;
		     u64 bcast_pkts; u64 bcast_bytes; u64 error_pkts;
		     u64 discard_pkts;
		     }) vmxnet3_tx_stats;

typedef CLIB_PACKED (struct
		     {
		     vmxnet3_tx_queue_control ctrl;
		     vmxnet3_tx_queue_config cfg;
		     vmxnet3_queue_status status; vmxnet3_tx_stats stats;
		     u8 pad[88];
		     }) vmxnet3_tx_queue;

typedef CLIB_PACKED (struct
		     {
		     u8 update_prod; u8 pad[7];
		     u64 pad1;
		     }) vmxnet3_rx_queue_control;

typedef CLIB_PACKED (struct
		     {
		     u64 desc_address[2];
		     u64 comp_address; u64 driver_data_address; u64 pad;
		     u32 num_desc[2];
		     u32 num_comp; u32 driver_data_len; u8 intr_index;
		     u8 pad1[7];
		     }) vmxnet3_rx_queue_config;

typedef CLIB_PACKED (struct
		     {
		     u64 lro_pkts;
		     u64 lro_bytes;
		     u64 ucast_pkts; u64 ucast_bytes; u64 mcast_pkts;
		     u64 mcast_bytes;
		     u64 bcast_pkts; u64 bcast_bytes; u64 nobuf_pkts;
		     u64 error_pkts;
		     }) vmxnet3_rx_stats;

typedef CLIB_PACKED (struct
		     {
		     vmxnet3_rx_queue_control ctrl;
		     vmxnet3_rx_queue_config cfg;
		     vmxnet3_queue_status status; vmxnet3_rx_stats stats;
		     u8 pad[88];
		     }) vmxnet3_rx_queue;

typedef CLIB_PACKED (struct
		     {
		     vmxnet3_tx_queue tx; vmxnet3_rx_queue rx;
		     }) vmxnet3_queues;

/*
 * flags:
 *   buffer length   -- bits 0-13
 *   buffer type     -- bit  14
 *   descriptor type -- bit  15
 *   reserved        -- bits 16-30
 *   generation      -- bit  31
 */
typedef CLIB_PACKED (struct
		     {
		     u64 address;
		     u32 flags;
		     u32 pad;
		     }) vmxnet3_rx_desc;

/*
 * index:
 *   RX desc index           -- bits 0-11
 *   ext1                    -- bits 12-13
 *   end of packet           -- bit  14
 *   start of packet         -- bit  15
 *   ring ID                 -- bits 16-25
 *   RSS hash type           -- bits 26-29
 *   checksum not calculated -- bit  30
 *   ext2                    -- bit  31
 *
 * rss: RSS hash value
 *
 * len:
 *   data length             -- bits 0-13
 *   error                   -- bit  14
 *   tag is stripped         -- bit  15
 *   tag stripped            -- bits 16-31
 *
 * flags:
 *   checksum                -- bits 0 - 15
 *   tcp/udp checksum correct-- bit  16
 *   udp packet              -- bit  17
 *   tcp packet              -- bit  18
 *   ip checksum correct     -- bit  19
 *   ipv6                    -- bit  20
 *   ipv4                    -- bit  21
 *   ip fragment             -- bit  22
 *   frame crc correct       -- bit  23
 *   completion type         -- bits 24-30
 *   generation              -- bit  31
 */
typedef CLIB_PACKED (struct
		     {
		     u32 index; u32 rss;
		     u32 len;
		     u32 flags;
		     }) vmxnet3_rx_comp;

/*
 * index:
 *   TX desc index           -- bits 0-11
 *   ext1                    -- bits 12-31
 *
 * flags:
 *   reserved                -- bits 0-23
 *   completion type         -- bits 24-30
 *   generation              -- bit  31
 */
typedef CLIB_PACKED (struct
		     {
		     u32 index;
		     u32 pad[2];
		     u32 flags;
		     }) vmxnet3_tx_comp;

/*
 * flags[0]:
 *   length                  -- bits 0-13
 *   generation              -- bit  14
 *   reserved                -- bit  15
 *   descriptor type         -- bit  16
 *   ext1                    -- bit  17
 *   MSS, checksum offset    -- bits 18-31
 * flags[1]:
 *   header length           -- bits 0-9
 *   offload mode            -- bits 10-11
 *   end of packet           -- bit  12
 *   completion request      -- bit  13
 *   ext2                    -- bit  14
 *   vlan tag insertion      -- bit  15
 *   tag to insert           -- bits 16-31
 */
typedef CLIB_PACKED (struct
		     {
		     u64 address;
		     u32 flags[2];
		     }) vmxnet3_tx_desc;

typedef struct
{
  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
  u32 *bufs;
  u32 gen;
  u16 fill;
  u16 rid;
  u16 produce;
  u16 consume;
} vmxnet3_rx_ring;

typedef struct
{
  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
  u64 next;
  u32 gen;
} vmxnet3_rx_comp_ring;

typedef struct
{
  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
  u16 size;
  u8 int_mode;
  vmxnet3_rx_ring rx_ring[VMXNET3_RX_RING_SIZE];
  vmxnet3_rx_desc *rx_desc[VMXNET3_RX_RING_SIZE];
  vmxnet3_rx_comp *rx_comp;
  vmxnet3_rx_comp_ring rx_comp_ring;
} vmxnet3_rxq_t;

typedef struct
{
  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
  u32 *bufs;
  u32 gen;
  u16 produce;
  u16 consume;
} vmxnet3_tx_ring;

typedef struct
{
  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
  u64 next;
  u32 gen;
} vmxnet3_tx_comp_ring;

typedef struct
{
  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
  u16 size;
  clib_spinlock_t lock;

  vmxnet3_tx_desc *tx_desc;
  vmxnet3_tx_comp *tx_comp;
  vmxnet3_tx_ring tx_ring;
  vmxnet3_tx_comp_ring tx_comp_ring;
} vmxnet3_txq_t;

typedef CLIB_PACKED (struct
		     {
		     vmxnet3_queues queues; vmxnet3_shared shared;
		     }) vmxnet3_dma;

typedef struct
{
  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
  u32 flags;
  u32 per_interface_next_index;

  u32 dev_instance;
  u32 sw_if_index;
  u32 hw_if_index;
  vlib_pci_dev_handle_t pci_dev_handle;
  vlib_pci_addr_t pci_addr;
  void *bar[2];

  /* queues */
  vmxnet3_rxq_t *rxqs;
  vmxnet3_txq_t *txqs;

  u16 num_tx_queues;
  u16 num_rx_queues;
  u16 num_intrs;

  u8 version;
  u8 mac_addr[6];

  /* error */
  clib_error_t *error;

  vmxnet3_dma *dma;

} vmxnet3_device_t;

typedef struct
{
  vmxnet3_device_t *devices;
  vlib_physmem_region_index_t physmem_region;
  u32 physmem_region_alloc;
  u16 msg_id_base;
} vmxnet3_main_t;

extern vmxnet3_main_t vmxnet3_main;

typedef struct
{
  vlib_pci_addr_t addr;
  u32 enable_elog;
  u16 rxq_size;
  u16 txq_size;
  /* return */
  i32 rv;
  u32 sw_if_index;
  clib_error_t *error;
} vmxnet3_create_if_args_t;

typedef struct
{
  u32 next_index;
  u32 hw_if_index;
  vlib_buffer_t buffer;
} vmxnet3_input_trace_t;

void vmxnet3_create_if (vlib_main_t * vm, vmxnet3_create_if_args_t * args);
void vmxnet3_delete_if (vlib_main_t * vm, vmxnet3_device_t * ad);

extern clib_error_t *vmxnet3_plugin_api_hookup (vlib_main_t * vm);
extern vlib_node_registration_t vmxnet3_input_node;
extern vnet_device_class_t vmxnet3_device_class;

/* format.c */
format_function_t format_vmxnet3_device;
format_function_t format_vmxnet3_device_name;
format_function_t format_vmxnet3_input_trace;

static_always_inline void
vmxnet3_reg_write (vmxnet3_device_t * vd, u8 bar, u32 addr, u32 val)
{
  *(volatile u32 *) ((u8 *) vd->bar[bar] + addr) = val;
}

static_always_inline u32
vmxnet3_reg_read (vmxnet3_device_t * vd, u8 bar, u32 addr)
{
  return *(volatile u32 *) (vd->bar[bar] + addr);
}

static_always_inline uword
vmxnet3_dma_addr (vlib_main_t * vm, vmxnet3_device_t * vd, void *p)
{
  vmxnet3_main_t *vmxm = &vmxnet3_main;

  return (vd->flags & VMXNET3_DEVICE_F_IOVA) ? pointer_to_uword (p) :
    vlib_physmem_virtual_to_physical (vm, vmxm->physmem_region, p);
}

static_always_inline void
vmxnet3_rx_ring_advance_produce (vmxnet3_rxq_t * rxq, vmxnet3_rx_ring * ring)
{
  ring->produce++;
  if (PREDICT_FALSE (ring->produce == rxq->size))
    {
      ring->produce = 0;
      ring->gen ^= VMXNET3_RXF_GEN;
    }
}

static_always_inline clib_error_t *
vmxnet3_rxq_refill_ring0 (vlib_main_t * vm, vmxnet3_device_t * vd,
			  vmxnet3_rxq_t * rxq)
{
  vmxnet3_rx_desc *rxd;
  u16 n_refill, n_alloc;
  vmxnet3_rx_ring *ring;

  ring = &rxq->rx_ring[0];
  n_refill = rxq->size - ring->fill;

  if (PREDICT_TRUE (n_refill <= VMXNET3_INPUT_REFILL_THRESHOLD))
    return 0;

  n_alloc =
    vlib_buffer_alloc_to_ring (vm, ring->bufs, ring->produce, rxq->size,
			       n_refill);
  if (PREDICT_FALSE (n_alloc != n_refill))
    {
      if (n_alloc)
	vlib_buffer_free_from_ring (vm, ring->bufs, ring->produce, rxq->size,
				    n_alloc);
      return clib_error_return (0, "buffer alloc failed");
    }

  while (n_alloc)
    {
      rxd = &rxq->rx_desc[0][ring->produce];
      rxd->address =
	vlib_get_buffer_data_physical_address (vm, ring->bufs[ring->produce]);
      rxd->flags = ring->gen | VLIB_BUFFER_DATA_SIZE;

      vmxnet3_rx_ring_advance_produce (rxq, ring);
      ring->fill++;
      n_alloc--;
    }

  vmxnet3_reg_write (vd, 0, VMXNET3_REG_RXPROD, ring->produce);

  return 0;
}

static_always_inline clib_error_t *
vmxnet3_rxq_refill_ring1 (vlib_main_t * vm, vmxnet3_device_t * vd,
			  vmxnet3_rxq_t * rxq)
{
  vmxnet3_rx_desc *rxd;
  u16 n_refill, n_alloc;
  vmxnet3_rx_ring *ring;

  ring = &rxq->rx_ring[1];
  n_refill = rxq->size - ring->fill;

  if (PREDICT_TRUE (n_refill <= VMXNET3_INPUT_REFILL_THRESHOLD))
    return 0;

  n_alloc =
    vlib_buffer_alloc_to_ring (vm, ring->bufs, ring->produce, rxq->size,
			       n_refill);
  if (PREDICT_FALSE (n_alloc != n_refill))
    {
      if (n_alloc)
	vlib_buffer_free_from_ring (vm, ring->bufs, ring->produce, rxq->size,
				    n_alloc);
      return clib_error_return (0, "buffer alloc failed");
    }

  while (n_alloc)
    {
      rxd = &rxq->rx_desc[1][ring->produce];
      rxd->address =
	vlib_get_buffer_data_physical_address (vm, ring->bufs[ring->produce]);
      rxd->flags = ring->gen | VLIB_BUFFER_DATA_SIZE | VMXNET3_RXF_BTYPE;

      vmxnet3_rx_ring_advance_produce (rxq, ring);
      ring->fill++;
      n_alloc--;
    }

  vmxnet3_reg_write (vd, 0, VMXNET3_REG_RXPROD2, ring->produce);

  return 0;
}

#endif /* __included_vmnet_vmnet_h__ */
/*
 * fd.io coding-style-patch-verification: ON
 *
 * Local Variables:
 * eval: (c-set-style "gnu")
 * End:
 */