/* * Copyright (c) 2015 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #define _GNU_SOURCE #include #include #include #include #include #include DECLARE_CJ_GLOBAL_LOG; #define FRAME_QUEUE_NELTS 32 u32 vl (void *p) { return vec_len (p); } vlib_worker_thread_t *vlib_worker_threads; vlib_thread_main_t vlib_thread_main; /* * Barrier tracing can be enabled on a normal build to collect information * on barrier use, including timings and call stacks. Deliberately not * keyed off CLIB_DEBUG, because that can add significant overhead which * imapacts observed timings. */ #ifdef BARRIER_TRACING /* * Output of barrier tracing can be to syslog or elog as suits */ #ifdef BARRIER_TRACING_ELOG static u32 elog_id_for_msg_name (const char *msg_name) { uword *p, r; static uword *h; u8 *name_copy; if (!h) h = hash_create_string (0, sizeof (uword)); p = hash_get_mem (h, msg_name); if (p) return p[0]; r = elog_string (&vlib_global_main.elog_main, "%s", msg_name); name_copy = format (0, "%s%c", msg_name, 0); hash_set_mem (h, name_copy, r); return r; } /* * elog Barrier trace functions, which are nulled out if BARRIER_TRACING isn't * defined */ static inline void barrier_trace_sync (f64 t_entry, f64 t_open, f64 t_closed) { /* *INDENT-OFF* */ ELOG_TYPE_DECLARE (e) = { .format = "barrier <%d#%s(O:%dus:%dus)(%dus)", .format_args = "i4T4i4i4i4", }; /* *INDENT-ON* */ struct { u32 count, caller, t_entry, t_open, t_closed; } *ed = 0; ed = ELOG_DATA (&vlib_global_main.elog_main, e); ed->count = (int) vlib_worker_threads[0].barrier_sync_count; ed->caller = elog_id_for_msg_name (vlib_worker_threads[0].barrier_caller); ed->t_entry = (int) (1000000.0 * t_entry); ed->t_open = (int) (1000000.0 * t_open); ed->t_closed = (int) (1000000.0 * t_closed); } static inline void barrier_trace_sync_rec (f64 t_entry) { /* *INDENT-OFF* */ ELOG_TYPE_DECLARE (e) = { .format = "barrier <%d(%dus)%s", .format_args = "i4i4T4", }; /* *INDENT-ON* */ struct { u32 depth, t_entry, caller; } *ed = 0; ed = ELOG_DATA (&vlib_global_main.elog_main, e); ed->depth = (int) vlib_worker_threads[0].recursion_level - 1; ed->t_entry = (int) (1000000.0 * t_entry); ed->caller = elog_id_for_msg_name (vlib_worker_threads[0].barrier_caller); } static inline void barrier_trace_release_rec (f64 t_entry) { /* *INDENT-OFF* */ ELOG_TYPE_DECLARE (e) = { .format = "barrier (%dus)%d>", .format_args = "i4i4", }; /* *INDENT-ON* */ struct { u32 t_entry, depth; } *ed = 0; ed = ELOG_DATA (&vlib_global_main.elog_main, e); ed->t_entry = (int) (1000000.0 * t_entry); ed->depth = (int) vlib_worker_threads[0].recursion_level; } static inline void barrier_trace_release (f64 t_entry, f64 t_closed_total, f64 t_update_main) { /* *INDENT-OFF* */ ELOG_TYPE_DECLARE (e) = { .format = "barrier (%dus){%d}(C:%dus)#%d>", .format_args = "i4i4i4i4", }; /* *INDENT-ON* */ struct { u32 t_entry, t_update_main, t_closed_total, count; } *ed = 0; ed = ELOG_DATA (&vlib_global_main.elog_main, e); ed->t_entry = (int) (1000000.0 * t_entry); ed->t_update_main = (int) (1000000.0 * t_update_main); ed->t_closed_total = (int) (1000000.0 * t_closed_total); ed->count = (int) vlib_worker_threads[0].barrier_sync_count; /* Reset context for next trace */ vlib_worker_threads[0].barrier_context = NULL; } #else char barrier_trace[65536]; char *btp = barrier_trace; /* * syslog Barrier trace functions, which are nulled out if BARRIER_TRACING * isn't defined */ static inline void barrier_trace_sync (f64 t_entry, f64 t_open, f64 t_closed) { btp += sprintf (btp, "<%u#%s", (unsigned int) vlib_worker_threads[0].barrier_sync_count, vlib_worker_threads[0].barrier_caller); if (vlib_worker_threads[0].barrier_context) { btp += sprintf (btp, "[%s]", vlib_worker_threads[0].barrier_context); } btp += sprintf (btp, "(O:%dus:%dus)(%dus):", (int) (1000000.0 * t_entry), (int) (1000000.0 * t_open), (int) (1000000.0 * t_closed)); } static inline void barrier_trace_sync_rec (f64 t_entry) { btp += sprintf (btp, "<%u(%dus)%s:", (int) vlib_worker_threads[0].recursion_level - 1, (int) (1000000.0 * t_entry), vlib_worker_threads[0].barrier_caller); } static inline void barrier_trace_release_rec (f64 t_entry) { btp += sprintf (btp, ":(%dus)%u>", (int) (1000000.0 * t_entry), (int) vlib_worker_threads[0].recursion_level); } static inline void barrier_trace_release (f64 t_entry, f64 t_closed_total, f64 t_update_main) { btp += sprintf (btp, ":(%dus)", (int) (1000000.0 * t_entry)); if (t_update_main > 0) { btp += sprintf (btp, "{%dus}", (int) (1000000.0 * t_update_main)); } btp += sprintf (btp, "(C:%dus)#%u>", (int) (1000000.0 * t_closed_total), (int) vlib_worker_threads[0].barrier_sync_count); /* Dump buffer to syslog, and reset for next trace */ fformat (stderr, "BTRC %s\n", barrier_trace); btp = barrier_trace; vlib_worker_threads[0].barrier_context = NULL; } #endif #else /* Null functions for default case where barrier tracing isn't used */ static inline void barrier_trace_sync (f64 t_entry, f64 t_open, f64 t_closed) { } static inline void barrier_trace_sync_rec (f64 t_entry) { } static inline void barrier_trace_release_rec (f64 t_entry) { } static inline void barrier_trace_release (f64 t_entry, f64 t_closed_total, f64 t_update_main) { } #endif uword os_get_nthreads (void) { u32 len; len = vec_len (vlib_thread_stacks); if (len == 0) return 1; else return len; } void vlib_set_thread_name (char *name) { int pthread_setname_np (pthread_t __target_thread, const char *__name); int rv; pthread_t thread = pthread_self (); if (thread) { rv = pthread_setname_np (thread, name); if (rv) clib_warning ("pthread_setname_np returned %d", rv); } } static int sort_registrations_by_no_clone (void *a0, void *a1) { vlib_thread_registration_t **tr0 = a0; vlib_thread_registration_t **tr1 = a1; return ((i32) ((*tr0)->no_data_structure_clone) - ((i32) ((*tr1)->no_data_structure_clone))); } static uword * clib_sysfs_list_to_bitmap (char *filename) { FILE *fp; uword *r = 0; fp = fopen (filename, "r"); if (fp != NULL) { u8 *buffer = 0; vec_validate (buffer, 256 - 1); if (fgets ((char *) buffer, 256, fp)) { unformat_input_t in; unformat_init_string (&in, (char *) buffer, strlen ((char *) buffer)); if (unformat (&in, "%U", unformat_bitmap_list, &r) != 1) clib_warning ("unformat_bitmap_list failed"); unformat_free (&in); } vec_free (buffer); fclose (fp); } return r; } /* Called early in the init sequence */ clib_error_t * vlib_thread_init (vlib_main_t * vm) { vlib_thread_main_t *tm = &vlib_thread_main; vlib_worker_thread_t *w; vlib_thread_registration_t *tr; u32 n_vlib_mains = 1; u32 first_index = 1; u32 i; uword *avail_cpu; /* get bitmaps of active cpu cores and sockets */ tm->cpu_core_bitmap = clib_sysfs_list_to_bitmap ("/sys/devices/system/cpu/online"); tm->cpu_socket_bitmap = clib_sysfs_list_to_bitmap ("/sys/devices/system/node/online"); avail_cpu = clib_bitmap_dup (tm->cpu_core_bitmap); /* skip cores */ for (i = 0; i < tm->skip_cores; i++) { uword c = clib_bitmap_first_set (avail_cpu); if (c == ~0) return clib_error_return (0, "no available cpus to skip"); avail_cpu = clib_bitmap_set (avail_cpu, c, 0); } /* grab cpu for main thread */ if (!tm->main_lcore) { tm->main_lcore = clib_bitmap_first_set (avail_cpu); if (tm->main_lcore == (u8) ~ 0) return clib_error_return (0, "no available cpus to be used for the" " main thread"); } else { if (clib_bitmap_get (avail_cpu, tm->main_lcore) == 0) return clib_error_return (0, "cpu %u is not available to be used" " for the main thread", tm->main_lcore); } avail_cpu = clib_bitmap_set (avail_cpu, tm->main_lcore, 0); /* assume that there is socket 0 only if there is no data from sysfs */ if (!tm->cpu_socket_bitmap) tm->cpu_socket_bitmap = clib_bitmap_set (0, 0, 1); /* pin main thread to main_lcore */ if (tm->cb.vlib_thread_set_lcore_cb) { tm->cb.vlib_thread_set_lcore_cb (0, tm->main_lcore); } else { cpu_set_t cpuset; CPU_ZERO (&cpuset); CPU_SET (tm->main_lcore, &cpuset); pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t), &cpuset); } /* as many threads as stacks... */ vec_validate_aligned (vlib_worker_threads, vec_len (vlib_thread_stacks) - 1, CLIB_CACHE_LINE_BYTES); /* Preallocate thread 0 */ _vec_len (vlib_worker_threads) = 1; w = vlib_worker_threads; w->thread_mheap = clib_mem_get_heap (); w->thread_stack = vlib_thread_stacks[0]; w->lcore_id = tm->main_lcore; w->lwp = syscall (SYS_gettid); w->thread_id = pthread_self (); tm->n_vlib_mains = 1; if (tm->sched_policy != ~0) { struct sched_param sched_param; if (!sched_getparam (w->lwp, &sched_param)) { if (tm->sched_priority != ~0) sched_param.sched_priority = tm->sched_priority; sched_setscheduler (w->lwp, tm->sched_policy, &sched_param); } } /* assign threads to cores and set n_vlib_mains */ tr = tm->next; while (tr) { vec_add1 (tm->registrations, tr); tr = tr->next; } vec_sort_with_function (tm->registrations, sort_registrations_by_no_clone); for (i = 0; i < vec_len (tm->registrations); i++) { int j; tr = tm->registrations[i]; tr->first_index = first_index; first_index += tr->count; n_vlib_mains += (tr->no_data_structure_clone == 0) ? tr->count : 0; /* construct coremask */ if (tr->use_pthreads || !tr->count) continue; if (tr->coremask) { uword c; /* *INDENT-OFF* */ clib_bitmap_foreach (c, tr->coremask, ({ if (clib_bitmap_get(avail_cpu, c) == 0) return clib_error_return (0, "cpu %u is not available to be used" " for the '%s' thread",c, tr->name); avail_cpu = clib_bitmap_set(avail_cpu, c, 0); })); /* *INDENT-ON* */ } else { for (j = 0; j < tr->count; j++) { uword c = clib_bitmap_first_set (avail_cpu); if (c == ~0) return clib_error_return (0, "no available cpus to be used for" " the '%s' thread", tr->name); avail_cpu = clib_bitmap_set (avail_cpu, c, 0); tr->coremask = clib_bitmap_set (tr->coremask, c, 1); } } } clib_bitmap_free (avail_cpu); tm->n_vlib_mains = n_vlib_mains; vec_validate_aligned (vlib_worker_threads, first_index - 1, CLIB_CACHE_LINE_BYTES); return 0; } vlib_frame_queue_t * vlib_frame_queue_alloc (int nelts) { vlib_frame_queue_t *fq; fq = clib_mem_alloc_aligned (sizeof (*fq), CLIB_CACHE_LINE_BYTES); memset (fq, 0, sizeof (*fq)); fq->nelts = nelts; fq->vector_threshold = 128; // packets vec_validate_aligned (fq->elts, nelts - 1, CLIB_CACHE_LINE_BYTES); if (1) { if (((uword) & fq->tail) & (CLIB_CACHE_LINE_BYTES - 1)) fformat (stderr, "WARNING: fq->tail unaligned\n"); if (((uword) & fq->head) & (CLIB_CACHE_LINE_BYTES - 1)) fformat (stderr, "WARNING: fq->head unaligned\n"); if (((uword) fq->elts) & (CLIB_CACHE_LINE_BYTES - 1)) fformat (stderr, "WARNING: fq->elts unaligned\n"); if (sizeof (fq->elts[0]) % CLIB_CACHE_LINE_BYTES) fformat (stderr, "WARNING: fq->elts[0] size %d\n", sizeof (fq->elts[0])); if (nelts & (nelts - 1)) { fformat (stderr, "FATAL: nelts MUST be a power of 2\n"); abort (); } } return (fq); } void vl_msg_api_handler_no_free (void *) __attribute__ ((weak)); void vl_msg_api_handler_no_free (void *v) { } /* Turned off, save as reference material... */ #if 0 static inline int vlib_frame_queue_dequeue_internal (int thread_id, vlib_main_t * vm, vlib_node_main_t * nm) { vlib_frame_queue_t *fq = vlib_frame_queues[thread_id]; vlib_frame_queue_elt_t *elt; vlib_frame_t *f; vlib_pending_frame_t *p; vlib_node_runtime_t *r; u32 node_runtime_index; int msg_type; u64 before; int processed = 0; ASSERT (vm == vlib_mains[thread_id]); while (1) { if (fq->head == fq->tail) return processed; elt = fq->elts + ((fq->head + 1) & (fq->nelts - 1)); if (!elt->valid) return processed; before = clib_cpu_time_now (); f = elt->frame; node_runtime_index = elt->node_runtime_index; msg_type = elt->msg_type; switch (msg_type) { case VLIB_FRAME_QUEUE_ELT_FREE_BUFFERS: vlib_buffer_free (vm, vlib_frame_vector_args (f), f->n_vectors); /* note fallthrough... */ case VLIB_FRAME_QUEUE_ELT_FREE_FRAME: r = vec_elt_at_index (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL], node_runtime_index); vlib_frame_free (vm, r, f); break; case VLIB_FRAME_QUEUE_ELT_DISPATCH_FRAME: vec_add2 (vm->node_main.pending_frames, p, 1); f->flags |= (VLIB_FRAME_PENDING | VLIB_FRAME_FREE_AFTER_DISPATCH); p->node_runtime_index = elt->node_runtime_index; p->frame_index = vlib_frame_index (vm, f); p->next_frame_index = VLIB_PENDING_FRAME_NO_NEXT_FRAME; fq->dequeue_vectors += (u64) f->n_vectors; break; case VLIB_FRAME_QUEUE_ELT_API_MSG: vl_msg_api_handler_no_free (f); break; default: clib_warning ("bogus frame queue message, type %d", msg_type); break; } elt->valid = 0; fq->dequeues++; fq->dequeue_ticks += clib_cpu_time_now () - before; CLIB_MEMORY_BARRIER (); fq->head++; processed++; } ASSERT (0); return processed; } int vlib_frame_queue_dequeue (int thread_id, vlib_main_t * vm, vlib_node_main_t * nm) { return vlib_frame_queue_dequeue_internal (thread_id, vm, nm); } int vlib_frame_queue_enqueue (vlib_main_t * vm, u32 node_runtime_index, u32 frame_queue_index, vlib_frame_t * frame, vlib_frame_queue_msg_type_t type) { vlib_frame_queue_t *fq = vlib_frame_queues[frame_queue_index]; vlib_frame_queue_elt_t *elt; u32 save_count; u64 new_tail; u64 before = clib_cpu_time_now (); ASSERT (fq); new_tail = __sync_add_and_fetch (&fq->tail, 1); /* Wait until a ring slot is available */ while (new_tail >= fq->head + fq->nelts) { f64 b4 = vlib_time_now_ticks (vm, before); vlib_worker_thread_barrier_check (vm, b4); /* Bad idea. Dequeue -> enqueue -> dequeue -> trouble */ // vlib_frame_queue_dequeue (vm->thread_index, vm, nm); } elt = fq->elts + (new_tail & (fq->nelts - 1)); /* this would be very bad... */ while (elt->valid) { } /* Once we enqueue the frame, frame->n_vectors is owned elsewhere... */ save_count = frame->n_vectors; elt->frame = frame; elt->node_runtime_index = node_runtime_index; elt->msg_type = type; CLIB_MEMORY_BARRIER (); elt->valid = 1; return save_count; } #endif /* 0 */ /* To be called by vlib worker threads upon startup */ void vlib_worker_thread_init (vlib_worker_thread_t * w) { vlib_thread_main_t *tm = vlib_get_thread_main (); /* * Note: disabling signals in worker threads as follows * prevents the api post-mortem dump scheme from working * { * sigset_t s; * sigfillset (&s); * pthread_sigmask (SIG_SETMASK, &s, 0); * } */ clib_mem_set_heap (w->thread_mheap); if (vec_len (tm->thread_prefix) && w->registration->short_name) { w->name = format (0, "%v_%s_%d%c", tm->thread_prefix, w->registration->short_name, w->instance_id, '\0'); vlib_set_thread_name ((char *) w->name); } if (!w->registration->use_pthreads) { /* Initial barrier sync, for both worker and i/o threads */ clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, 1); while (*vlib_worker_threads->wait_at_barrier) ; clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, -1); } } void * vlib_worker_thread_bootstrap_fn (void *arg) { void *rv; vlib_worker_thread_t *w = arg; w->lwp = syscall (SYS_gettid); w->thread_id = pthread_self (); __os_thread_index = w - vlib_worker_threads; rv = (void *) clib_calljmp ((uword (*)(uword)) w->thread_function, (uword) arg, w->thread_stack + VLIB_THREAD_STACK_SIZE); /* NOTREACHED, we hope */ return rv; } static clib_error_t * vlib_launch_thread_int (void *fp, vlib_worker_thread_t * w, unsigned lcore_id) { vlib_thread_main_t *tm = &vlib_thread_main; void *(*fp_arg) (void *) = fp; w->lcore_id = lcore_id; if (tm->cb.vlib_launch_thread_cb && !w->registration->use_pthreads) return tm->cb.vlib_launch_thread_cb (fp, (void *) w, lcore_id); else { pthread_t worker; cpu_set_t cpuset; CPU_ZERO (&cpuset); CPU_SET (lcore_id, &cp
/*
 * Copyright (c) 2016 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef included_features_h
#define included_features_h

#include <vnet/vnet.h>
#include <vnet/api_errno.h>
#include <vnet/devices/devices.h>

/** feature registration object */
typedef struct _vnet_feature_arc_registration
{
  /** next registration in list of all registrations*/
  struct _vnet_feature_arc_registration *next;
  /** Feature Arc name */
  char *arc_name;
  /** Start nodes */
  char **start_nodes;
  int n_start_nodes;
  /* Feature arc index, assigned by init function */
  u8 feature_arc_index;
  u8 *arc_index_ptr;
} vnet_feature_arc_registration_t;

/* Enable feature callback. */
typedef clib_error_t *(vnet_feature_enable_disable_function_t)
  (u32 sw_if_index, int enable_disable);

/** feature registration object */
typedef struct _vnet_feature_registration
{
  /** next registration in list of all registrations*/
  struct _vnet_feature_registration *next;
  /** Feature arc name */
  char *arc_name;
  /** Graph node name */
  char *node_name;
  /** Pointer to this feature index, filled in by vnet_feature_arc_init */
  u32 *feature_index_ptr;
  u32 feature_index;
  /** Constraints of the form "this feature runs before X" */
  char **runs_before;
  /** Constraints of the form "this feature runs after Y" */
  char **runs_after;

  /** Function to enable/disable feature  **/
  vnet_feature_enable_disable_function_t *enable_disable_cb;
} vnet_feature_registration_t;

typedef struct vnet_feature_config_main_t_
{
  vnet_config_main_t config_main;
  u32 *config_index_by_sw_if_index;
} vnet_feature_config_main_t;

typedef struct
{
  /** feature arc configuration list */
  vnet_feature_arc_registration_t *next_arc;
  uword **arc_index_by_name;

  /** feature path configuration lists */
  vnet_feature_registration_t *next_feature;
  vnet_feature_registration_t **next_feature_by_arc;
  uword **next_feature_by_name;

  /** feature config main objects */
  vnet_feature_config_main_t *feature_config_mains;

  /** Save partial order results for show command */
  char ***feature_nodes;

  /** bitmap of interfaces which have driver rx features configured */
  uword **sw_if_index_has_features;

  /** feature reference counts by interface */
  i16 **feature_count_by_sw_if_index;

  /** Feature arc index for device-input */
  u8 device_input_feature_arc_index;

  /** convenience */
  vlib_main_t *vlib_main;
  vnet_main_t *vnet_main;
} vnet_feature_main_t;

extern vnet_feature_main_t feature_main;

#define VNET_FEATURE_ARC_INIT(x,...)				\
  __VA_ARGS__ vnet_feature_arc_registration_t vnet_feat_arc_##x;\
static void __vnet_add_feature_arc_registration_##x (void)	\
  __attribute__((__constructor__)) ;				\
static void __vnet_add_feature_arc_registration_##x (void)	\
{								\
  vnet_feature_main_t * fm = &feature_main;			\
  vnet_feat_arc_##x.next = fm->next_arc;			\
  fm->next_arc = & vnet_feat_arc_##x;				\
}								\
__VA_ARGS__ vnet_feature_arc_registration_t vnet_feat_arc_##x

#define VNET_FEATURE_INIT(x,...)				\
  __VA_ARGS__ vnet_feature_registration_t vnet_feat_##x;	\
static void __vnet_add_feature_registration_##x (void)		\
  __attribute__((__constructor__)) ;				\
static void __vnet_add_feature_registration_##x (void)		\
{								\
  vnet_feature_main_t * fm = &feature_main;			\
  vnet_feat_##x.next = fm->next_feature;			\
  fm->next_feature = & vnet_feat_##x;				\
}								\
__VA_ARGS__ vnet_feature_registration_t vnet_feat_##x

void
vnet_config_update_feature_count (vnet_feature_main_t * fm, u8 arc,
				  u32 sw_if_index, int is_add);

u32 vnet_get_feature_index (u8 arc, const char *s);
u8 vnet_get_feature_arc_index (const char *s);
vnet_feature_registration_t *vnet_get_feature_reg (const char *arc_name,
						   const char *node_name);


int
vnet_feature_enable_disable_with_index (u8 arc_index, u32 feature_index,
					u32 sw_if_index, int enable_disable,
					void *feature_config,
					u32 n_feature_config_bytes);

int
vnet_feature_enable_disable (const char *arc_name, const char *node_name,
			     u32 sw_if_index, int enable_disable,
			     void *feature_config,
			     u32 n_feature_config_bytes);

static inline vnet_feature_config_main_t *
vnet_get_feature_arc_config_main (u8 arc_index)
{
  vnet_feature_main_t *fm = &feature_main;

  if (arc_index == (u8) ~ 0)
    return 0;

  return &fm->feature_config_mains[arc_index];
}

static_always_inline vnet_feature_config_main_t *
vnet_feature_get_config_main (u16 arc)
{
  vnet_feature_main_t *fm = &feature_main;
  return &fm->feature_config_mains[arc];
}

static_always_inline int
vnet_have_features (u8 arc, u32 sw_if_index)
{
  vnet_feature_main_t *fm = &feature_main;
  return clib_bitmap_get (fm->sw_if_index_has_features[arc], sw_if_index);
}

static_always_inline u32
vnet_get_feature_config_index (u8 arc, u32 sw_if_index)
{
  vnet_feature_main_t *fm = &feature_main;
  vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc];
  return vec_elt (cm->config_index_by_sw_if_index, sw_if_index);
}

static_always_inline void *
vnet_feature_arc_start_with_data (u8 arc, u32 sw_if_index, u32 * next,
				  vlib_buffer_t * b, u32 n_data_bytes)
{
  vnet_feature_main_t *fm = &feature_main;
  vnet_feature_config_main_t *cm;
  cm = &fm->feature_config_mains[arc];

  if (PREDICT_FALSE (vnet_have_features (arc, sw_if_index)))
    {
      b->feature_arc_index = arc;
      b->current_config_index =
	vec_elt (cm->config_index_by_sw_if_index, sw_if_index);
      return vnet_get_config_data (&cm->config_main, &b->current_config_index,
				   next, n_data_bytes);
    }
  return 0;
}

static_always_inline void
vnet_feature_arc_start (u8 arc, u32 sw_if_index, u32 * next0,
			vlib_buffer_t * b0)
{
  vnet_feature_arc_start_with_data (arc, sw_if_index, next0, b0, 0);
}

static_always_inline void *
vnet_feature_next_with_data (u32 sw_if_index, u32 * next0,
			     vlib_buffer_t * b0, u32 n_data_bytes)
{
  vnet_feature_main_t *fm = &feature_main;
  u8 arc = b0->feature_arc_index;
  vnet_feature_config_main_t *cm = &fm->feature_config_mains[arc];

  return vnet_get_config_data (&