diff options
Diffstat (limited to 'plugins/flowperpkt-plugin/flowperpkt')
| -rw-r--r-- | plugins/flowperpkt-plugin/flowperpkt/flowperpkt.api | 42 | ||||
| -rw-r--r-- | plugins/flowperpkt-plugin/flowperpkt/flowperpkt.c | 500 | ||||
| -rw-r--r-- | plugins/flowperpkt-plugin/flowperpkt/flowperpkt.h | 71 | ||||
| -rw-r--r-- | plugins/flowperpkt-plugin/flowperpkt/flowperpkt_all_api_h.h | 18 | ||||
| -rw-r--r-- | plugins/flowperpkt-plugin/flowperpkt/flowperpkt_msg_enum.h | 31 | ||||
| -rw-r--r-- | plugins/flowperpkt-plugin/flowperpkt/flowperpkt_test.c | 231 | ||||
| -rw-r--r-- | plugins/flowperpkt-plugin/flowperpkt/node.c | 504 |
7 files changed, 1397 insertions, 0 deletions
diff --git a/plugins/flowperpkt-plugin/flowperpkt/flowperpkt.api b/plugins/flowperpkt-plugin/flowperpkt/flowperpkt.api new file mode 100644 index 00000000000..8933b5852f4 --- /dev/null +++ b/plugins/flowperpkt-plugin/flowperpkt/flowperpkt.api @@ -0,0 +1,42 @@ +/* Define a simple enable-disable binary API to control the feature */ + +/** \file + This file defines the vpp control-plane API messages + used to control the flowperpkt plugin +*/ + +/** \brief Enable / disable per-packet IPFIX recording on an interface + @param client_index - opaque cookie to identify the sender + @param context - sender context, to match reply w/ request + @param is_add - add address if non-zero, else delete + @param is_ipv6 - if non-zero the address is ipv6, else ipv4 + @param sw_if_index - index of the interface +*/ +manual_print define flowperpkt_tx_interface_add_del +{ + /* Client identifier, set from api_main.my_client_index */ + u32 client_index; + + /* Arbitrary context, so client can match reply to request */ + u32 context; + + /* Enable / disable the feature */ + u8 is_add; + u8 is_ipv6; + + /* Interface handle */ + u32 sw_if_index; +}; + +/** \brief Reply to enable/disable per-packet IPFIX recording messages + @param context - returned sender context, to match reply w/ request + @param retval - return code +*/ +define flowperpkt_tx_interface_add_del_reply +{ + /* From the request */ + u32 context; + + /* Return value, zero means all OK */ + i32 retval; +}; diff --git a/plugins/flowperpkt-plugin/flowperpkt/flowperpkt.c b/plugins/flowperpkt-plugin/flowperpkt/flowperpkt.c new file mode 100644 index 00000000000..3aef24a223a --- /dev/null +++ b/plugins/flowperpkt-plugin/flowperpkt/flowperpkt.c @@ -0,0 +1,500 @@ +/* + * flowperpkt.c - per-packet data capture flow report plugin + * + * Copyright (c) <current-year> <your-organization> + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @file + * @brief Per-packet IPFIX flow record generator plugin + * + * This file implements vpp plugin registration mechanics, + * debug CLI, and binary API handling. + */ + +#include <vnet/vnet.h> +#include <vnet/plugin/plugin.h> +#include <flowperpkt/flowperpkt.h> + +#include <vlibapi/api.h> +#include <vlibmemory/api.h> +#include <vlibsocket/api.h> + +/* define message IDs */ +#include <flowperpkt/flowperpkt_msg_enum.h> + +/* define message structures */ +#define vl_typedefs +#include <flowperpkt/flowperpkt_all_api_h.h> +#undef vl_typedefs + +/* define generated endian-swappers */ +#define vl_endianfun +#include <flowperpkt/flowperpkt_all_api_h.h> +#undef vl_endianfun + +/* instantiate all the print functions we know about */ +#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) +#define vl_printfun +#include <flowperpkt/flowperpkt_all_api_h.h> +#undef vl_printfun + +flowperpkt_main_t flowperpkt_main; + +/* Get the API version number */ +#define vl_api_version(n,v) static u32 api_version=(v); +#include <flowperpkt/flowperpkt_all_api_h.h> +#undef vl_api_version + +/* Define the per-interface configurable feature */ +/* *INDENT-OFF* */ +VNET_IP4_TX_FEATURE_INIT (flow_perpacket, static) = { + .node_name = "flowperpkt", + .runs_before = (char *[]){"interface-output", 0}, + .feature_index = &flowperpkt_main.ip4_tx_feature_index, +}; +/* *INDENT-ON* */ + +/* + * A handy macro to set up a message reply. + * Assumes that the following variables are available: + * mp - pointer to request message + * rmp - pointer to reply message type + * rv - return value + */ +#define REPLY_MACRO(t) \ +do { \ + unix_shared_memory_queue_t * q = \ + vl_api_client_index_to_input_queue (mp->client_index); \ + if (!q) \ + return; \ + \ + rmp = vl_msg_api_alloc (sizeof (*rmp)); \ + rmp->_vl_msg_id = ntohs((t)+fm->msg_id_base); \ + rmp->context = mp->context; \ + rmp->retval = ntohl(rv); \ + \ + vl_msg_api_send_shmem (q, (u8 *)&rmp); \ +} while(0); + +/* Macro to finish up custom dump fns */ +#define FINISH \ + vec_add1 (s, 0); \ + vl_print (handle, (char *)s); \ + vec_free (s); \ + return handle; + +#define VALIDATE_SW_IF_INDEX(mp) \ + do { u32 __sw_if_index = ntohl(mp->sw_if_index); \ + vnet_main_t *__vnm = vnet_get_main(); \ + if (pool_is_free_index(__vnm->interface_main.sw_interfaces, \ + __sw_if_index)) { \ + rv = VNET_API_ERROR_INVALID_SW_IF_INDEX; \ + goto bad_sw_if_index; \ + } \ +} while(0); + +#define BAD_SW_IF_INDEX_LABEL \ +do { \ +bad_sw_if_index: \ + ; \ +} while (0); + +/** + * @brief Create an IPFIX template packet rewrite string + * @param frm flow_report_main_t * + * @param fr flow_report_t * + * @param collector_address ip4_address_t * the IPFIX collector address + * @param src_address ip4_address_t * the source address we should use + * @param collector_port u16 the collector port we should use, host byte order + * @returns u8 * vector containing the indicated IPFIX template packet + */ +u8 * +flowperpkt_template_rewrite (flow_report_main_t * frm, + flow_report_t * fr, + ip4_address_t * collector_address, + ip4_address_t * src_address, u16 collector_port) +{ + ip4_header_t *ip; + udp_header_t *udp; + ipfix_message_header_t *h; + ipfix_set_header_t *s; + ipfix_template_header_t *t; + ipfix_field_specifier_t *f; + ipfix_field_specifier_t *first_field; + u8 *rewrite = 0; + ip4_ipfix_template_packet_t *tp; + u32 field_count = 0; + flow_report_stream_t *stream; + + stream = &frm->streams[fr->stream_index]; + + /* + * Supported Fields: + * + * egressInterface, TLV type 14, u32 + * ipClassOfService, TLV type 5, u8 + * flowStartNanoseconds, TLV type 156, dateTimeNanoseconds (f64) + * Implementation: f64 nanoseconds since VPP started + * warning: wireshark doesn't really understand this TLV + * dataLinkFrameSize, TLV type 312, u16 + * warning: wireshark doesn't understand this TLV at all + */ + + /* Currently 4 fields */ + field_count += 4; + + /* allocate rewrite space */ + vec_validate_aligned (rewrite, + sizeof (ip4_ipfix_template_packet_t) + + field_count * sizeof (ipfix_field_specifier_t) - 1, + CLIB_CACHE_LINE_BYTES); + + tp = (ip4_ipfix_template_packet_t *) rewrite; + ip = (ip4_header_t *) & tp->ip4; + udp = (udp_header_t *) (ip + 1); + h = (ipfix_message_header_t *) (udp + 1); + s = (ipfix_set_header_t *) (h + 1); + t = (ipfix_template_header_t *) (s + 1); + first_field = f = (ipfix_field_specifier_t *) (t + 1); + + ip->ip_version_and_header_length = 0x45; + ip->ttl = 254; + ip->protocol = IP_PROTOCOL_UDP; + ip->src_address.as_u32 = src_address->as_u32; + ip->dst_address.as_u32 = collector_address->as_u32; + udp->src_port = clib_host_to_net_u16 (stream->src_port); + udp->dst_port = clib_host_to_net_u16 (collector_port); + udp->length = clib_host_to_net_u16 (vec_len (rewrite) - sizeof (*ip)); + + /* FIXUP: message header export_time */ + /* FIXUP: message header sequence_number */ + h->domain_id = clib_host_to_net_u32 (stream->domain_id); + + /* Add TLVs to the template */ + f->e_id_length = ipfix_e_id_length (0 /* enterprise */ , egressInterface, + 4); + f++; + f->e_id_length = ipfix_e_id_length (0 /* enterprise */ , ipClassOfService, + 1); + f++; + f->e_id_length = + ipfix_e_id_length (0 /* enterprise */ , flowStartNanoseconds, + 8); + f++; + f->e_id_length = ipfix_e_id_length (0 /* enterprise */ , dataLinkFrameSize, + 2); + f++; + /* Extend in the obvious way, right here... */ + + /* Back to the template packet... */ + ip = (ip4_header_t *) & tp->ip4; + udp = (udp_header_t *) (ip + 1); + + ASSERT (f - first_field); + /* Field count in this template */ + t->id_count = ipfix_id_count (fr->template_id, f - first_field); + + /* set length in octets */ + s->set_id_length = + ipfix_set_id_length (2 /* set_id */ , (u8 *) f - (u8 *) s); + + /* message length in octets */ + h->version_length = version_length ((u8 *) f - (u8 *) h); + + ip->length = clib_host_to_net_u16 ((u8 *) f - (u8 *) ip); + ip->checksum = ip4_header_checksum (ip); + + return rewrite; +} + +/** + * @brief Flush accumulated data + * @param frm flow_report_main_t * + * @param fr flow_report_t * + * @param f vlib_frame_t * + * + * <em>Notes:</em> + * This function must simply return the incoming frame, or no template packets + * will be sent. + */ +vlib_frame_t * +flowperpkt_data_callback (flow_report_main_t * frm, + flow_report_t * fr, + vlib_frame_t * f, u32 * to_next, u32 node_index) +{ + flowperpkt_flush_callback (); + return f; +} + +/** + * @brief configure / deconfigure the IPFIX flow-per-packet + * @param fm flowperpkt_main_t * fm + * @param sw_if_index u32 the desired interface + * @param is_add int 1 to enable the feature, 0 to disable it + * @returns 0 if successful, non-zero otherwise + */ + +static int flowperpkt_tx_interface_add_del_feature + (flowperpkt_main_t * fm, u32 sw_if_index, int is_add) +{ + u32 ci; + ip4_main_t *im = &ip4_main; + ip_lookup_main_t *lm = &im->lookup_main; + ip_config_main_t *cm = &lm->feature_config_mains[VNET_IP_TX_FEAT]; + u32 feature_index; + flow_report_main_t *frm = &flow_report_main; + vnet_flow_report_add_del_args_t _a, *a = &_a; + int rv; + + if (!fm->report_created) + { + memset (a, 0, sizeof (*a)); + a->rewrite_callback = flowperpkt_template_rewrite; + a->flow_data_callback = flowperpkt_data_callback; + a->is_add = 1; + a->domain_id = 1; /*$$$$ config parameter */ + a->src_port = 4739; /*$$$$ config parameter */ + fm->report_created = 1; + + rv = vnet_flow_report_add_del (frm, a); + if (rv) + { + clib_warning ("vnet_flow_report_add_del returned %d", rv); + return -1; + } + } + + feature_index = fm->ip4_tx_feature_index; + + ci = cm->config_index_by_sw_if_index[sw_if_index]; + ci = (is_add + ? vnet_config_add_feature + : vnet_config_del_feature) + (fm->vlib_main, &cm->config_main, + ci, feature_index, 0 /* config struct */ , + 0 /* sizeof config struct */ ); + cm->config_index_by_sw_if_index[sw_if_index] = ci; + + vnet_config_update_tx_feature_count (lm, cm, sw_if_index, is_add); + return 0; +} + +/** + * @brief API message handler + * @param mp vl_api_flowperpkt_tx_interface_add_del_t * mp the api message + */ +void vl_api_flowperpkt_tx_interface_add_del_t_handler + (vl_api_flowperpkt_tx_interface_add_del_t * mp) +{ + flowperpkt_main_t *fm = &flowperpkt_main; + vl_api_flowperpkt_tx_interface_add_del_reply_t *rmp; + u32 sw_if_index = ntohl (mp->sw_if_index); + int rv = 0; + + VALIDATE_SW_IF_INDEX (mp); + + rv = flowperpkt_tx_interface_add_del_feature (fm, sw_if_index, mp->is_add); + + BAD_SW_IF_INDEX_LABEL; + + REPLY_MACRO (VL_API_FLOWPERPKT_TX_INTERFACE_ADD_DEL_REPLY); +} + +/** + * @brief API message custom-dump function + * @param mp vl_api_flowperpkt_tx_interface_add_del_t * mp the api message + * @param handle void * print function handle + * @returns u8 * output string + */ +static void *vl_api_flowperpkt_tx_interface_add_del_t_print + (vl_api_flowperpkt_tx_interface_add_del_t * mp, void *handle) +{ + u8 *s; + + s = format (0, "SCRIPT: flowperpkt_tx_interface_add_del "); + s = format (s, "sw_if_index %d is_add %d is_ipv6 %d ", + clib_host_to_net_u32 (mp->sw_if_index), + (int) mp->is_add, (int) mp->is_ipv6); + FINISH; +} + +/* List of message types that this plugin understands */ +#define foreach_flowperpkt_plugin_api_msg \ +_(FLOWPERPKT_TX_INTERFACE_ADD_DEL, flowperpkt_tx_interface_add_del) + +/** + * @brief plugin-api required function + * @param vm vlib_main_t * vlib main data structure pointer + * @param h vlib_plugin_handoff_t * handoff structure + * @param from_early_init int notused + * + * <em>Notes:</em> + * This routine exists to convince the vlib plugin framework that + * we haven't accidentally copied a random .dll into the plugin directory. + * + * Also collects global variable pointers passed from the vpp engine + */ +clib_error_t * +vlib_plugin_register (vlib_main_t * vm, vnet_plugin_handoff_t * h, + int from_early_init) +{ + flowperpkt_main_t *fm = &flowperpkt_main; + clib_error_t *error = 0; + + fm->vlib_main = vm; + fm->vnet_main = h->vnet_main; + + return error; +} + +static clib_error_t * +flowperpkt_tx_interface_add_del_feature_command_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + flowperpkt_main_t *fm = &flowperpkt_main; + u32 sw_if_index = ~0; + int is_add = 1; + + int rv; + + while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (input, "disable")) + is_add = 0; + else if (unformat (input, "%U", unformat_vnet_sw_interface, + fm->vnet_main, &sw_if_index)) + ; + else + break; + } + + if (sw_if_index == ~0) + return clib_error_return (0, "Please specify an interface..."); + + rv = flowperpkt_tx_interface_add_del_feature (fm, sw_if_index, is_add); + switch (rv) + { + case 0: + break; + + case VNET_API_ERROR_INVALID_SW_IF_INDEX: + return clib_error_return + (0, "Invalid interface, only works on physical ports"); + break; + + case VNET_API_ERROR_UNIMPLEMENTED: + return clib_error_return (0, "ip6 not supported"); + break; + + default: + return clib_error_return (0, "flowperpkt_enable_disable returned %d", + rv); + } + return 0; +} + +/*? + * '<em>flowperpkt feature add-del</em>' commands to enable/disable + * per-packet IPFIX flow record generation on an interface + * + * @cliexpar + * @parblock + * To enable per-packet IPFIX flow-record generation on an interface: + * @cliexcmd{flowperpkt feature add-del GigabitEthernet2/0/0} + * + * To disable per-packet IPFIX flow-record generation on an interface: + * @cliexcmd{flowperpkt feature add-del GigabitEthernet2/0/0 disable} + * @cliexend + * @endparblock +?*/ +/* *INDENT-OFF* */ +VLIB_CLI_COMMAND (flowperpkt_enable_disable_command, static) = { + .path = "flowperpkt feature add-del", + .short_help = + "flowperpkt feature add-del <interface-name> [disable]", + .function = flowperpkt_tx_interface_add_del_feature_command_fn, +}; +/* *INDENT-ON* */ + +/** + * @brief Set up the API message handling tables + * @param vm vlib_main_t * vlib main data structure pointer + * @returns 0 to indicate all is well + */ +static clib_error_t * +flowperpkt_plugin_api_hookup (vlib_main_t * vm) +{ + flowperpkt_main_t *fm = &flowperpkt_main; +#define _(N,n) \ + vl_msg_api_set_handlers((VL_API_##N + fm->msg_id_base), \ + #n, \ + vl_api_##n##_t_handler, \ + vl_noop_handler, \ + vl_api_##n##_t_endian, \ + vl_api_##n##_t_print, \ + sizeof(vl_api_##n##_t), 1); + foreach_flowperpkt_plugin_api_msg; +#undef _ + + return 0; +} + +/** + * @brief Set up the API message handling tables + * @param vm vlib_main_t * vlib main data structure pointer + * @returns 0 to indicate all is well, or a clib_error_t + */ +static clib_error_t * +flowperpkt_init (vlib_main_t * vm) +{ + flowperpkt_main_t *fm = &flowperpkt_main; + vlib_thread_main_t *tm = &vlib_thread_main; + clib_error_t *error = 0; + u32 num_threads; + u8 *name; + + /* Construct the API name */ + name = format (0, "flowperpkt_%08x%c", api_version, 0); + + /* Ask for a correctly-sized block of API message decode slots */ + fm->msg_id_base = vl_msg_api_get_msg_ids + ((char *) name, VL_MSG_FIRST_AVAILABLE); + + /* Hook up message handlers */ + error = flowperpkt_plugin_api_hookup (vm); + + vec_free (name); + + /* Decide how many worker threads we have */ + num_threads = 1 /* main thread */ + tm->n_eal_threads; + + /* Allocate per worker thread vectors */ + vec_validate (fm->buffers_per_worker, num_threads - 1); + vec_validate (fm->frames_per_worker, num_threads - 1); + vec_validate (fm->next_record_offset_per_worker, num_threads - 1); + + return error; +} + +VLIB_INIT_FUNCTION (flowperpkt_init); + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/plugins/flowperpkt-plugin/flowperpkt/flowperpkt.h b/plugins/flowperpkt-plugin/flowperpkt/flowperpkt.h new file mode 100644 index 00000000000..54d978ab1ea --- /dev/null +++ b/plugins/flowperpkt-plugin/flowperpkt/flowperpkt.h @@ -0,0 +1,71 @@ +/* + * flowperpkt.h - skeleton vpp engine plug-in header file + * + * Copyright (c) <current-year> <your-organization> + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef __included_flowperpkt_h__ +#define __included_flowperpkt_h__ + +#include <vnet/vnet.h> +#include <vnet/ip/ip.h> +#include <vnet/ethernet/ethernet.h> + +#include <vppinfra/hash.h> +#include <vppinfra/error.h> +#include <vnet/flow/flow_report.h> +#include <vnet/flow/flow_report_classify.h> + +/** + * @file + * @brief flow-per-packet plugin header file + */ +typedef struct +{ + /** API message ID base */ + u16 msg_id_base; + + /** ip4 feature index */ + u32 ip4_tx_feature_index; + + /** Has the report been created? */ + int report_created; + + /** ipfix buffers under construction, per-worker thread */ + vlib_buffer_t **buffers_per_worker; + /** frames containing ipfix buffers, per-worker thread */ + vlib_frame_t **frames_per_worker; + /** next record offset, per worker thread */ + u16 *next_record_offset_per_worker; + + /** convenience vlib_main_t pointer */ + vlib_main_t *vlib_main; + /** convenience vnet_main_t pointer */ + vnet_main_t *vnet_main; +} flowperpkt_main_t; + +extern flowperpkt_main_t flowperpkt_main; + +vlib_node_registration_t flowperpkt_node; + +void flowperpkt_flush_callback (void); + +#endif /* __included_flowperpkt_h__ */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/plugins/flowperpkt-plugin/flowperpkt/flowperpkt_all_api_h.h b/plugins/flowperpkt-plugin/flowperpkt/flowperpkt_all_api_h.h new file mode 100644 index 00000000000..329c375abca --- /dev/null +++ b/plugins/flowperpkt-plugin/flowperpkt/flowperpkt_all_api_h.h @@ -0,0 +1,18 @@ +/* + * flowperpkt_all_api_h.h - plug-in api #include file + * + * Copyright (c) <current-year> <your-organization> + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* Include the generated file, see BUILT_SOURCES in Makefile.am */ +#include <flowperpkt/flowperpkt.api.h> diff --git a/plugins/flowperpkt-plugin/flowperpkt/flowperpkt_msg_enum.h b/plugins/flowperpkt-plugin/flowperpkt/flowperpkt_msg_enum.h new file mode 100644 index 00000000000..3177e77a63b --- /dev/null +++ b/plugins/flowperpkt-plugin/flowperpkt/flowperpkt_msg_enum.h @@ -0,0 +1,31 @@ +/* + * flowperpkt_msg_enum.h - vpp engine plug-in message enumeration + * + * Copyright (c) <current-year> <your-organization> + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef included_flowperpkt_msg_enum_h +#define included_flowperpkt_msg_enum_h + +#include <vppinfra/byte_order.h> + +#define vl_msg_id(n,h) n, +typedef enum +{ +#include <flowperpkt/flowperpkt_all_api_h.h> + /* We'll want to know how many messages IDs we need... */ + VL_MSG_FIRST_AVAILABLE, +} vl_msg_id_t; +#undef vl_msg_id + +#endif /* included_flowperpkt_msg_enum_h */ diff --git a/plugins/flowperpkt-plugin/flowperpkt/flowperpkt_test.c b/plugins/flowperpkt-plugin/flowperpkt/flowperpkt_test.c new file mode 100644 index 00000000000..3c1cd227f57 --- /dev/null +++ b/plugins/flowperpkt-plugin/flowperpkt/flowperpkt_test.c @@ -0,0 +1,231 @@ +/* + * flowperpkt.c - skeleton vpp-api-test plug-in + * + * Copyright (c) <current-year> <your-organization> + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <vat/vat.h> +#include <vlibapi/api.h> +#include <vlibmemory/api.h> +#include <vlibsocket/api.h> +#include <vppinfra/error.h> + +/** + * @file vpp_api_test plugin + */ + +uword unformat_sw_if_index (unformat_input_t * input, va_list * args); + +/* Declare message IDs */ +#include <flowperpkt/flowperpkt_msg_enum.h> + +/* define message structures */ +#define vl_typedefs +#include <flowperpkt/flowperpkt_all_api_h.h> +#undef vl_typedefs + +/* declare message handlers for each api */ + +#define vl_endianfun /* define message structures */ +#include <flowperpkt/flowperpkt_all_api_h.h> +#undef vl_endianfun + +/* instantiate all the print functions we know about */ +#define vl_print(handle, ...) +#define vl_printfun +#include <flowperpkt/flowperpkt_all_api_h.h> +#undef vl_printfun + +/* Get the API version number. */ +#define vl_api_version(n,v) static u32 api_version=(v); +#include <flowperpkt/flowperpkt_all_api_h.h> +#undef vl_api_version + +typedef struct +{ + /** API message ID base */ + u16 msg_id_base; + /** vat_main_t pointer */ + vat_main_t *vat_main; +} flowperpkt_test_main_t; + +flowperpkt_test_main_t flowperpkt_test_main; + +#define foreach_standard_reply_retval_handler \ +_(flowperpkt_tx_interface_add_del_reply) + +#define _(n) \ + static void vl_api_##n##_t_handler \ + (vl_api_##n##_t * mp) \ + { \ + vat_main_t * vam = flowperpkt_test_main.vat_main; \ + i32 retval = ntohl(mp->retval); \ + if (vam->async_mode) { \ + vam->async_errors += (retval < 0); \ + } else { \ + vam->retval = retval; \ + vam->result_ready = 1; \ + } \ + } +foreach_standard_reply_retval_handler; +#undef _ + +/* + * Table of message reply handlers, must include boilerplate handlers + * we just generated + */ +#define foreach_vpe_api_reply_msg \ +_(FLOWPERPKT_TX_INTERFACE_ADD_DEL_REPLY, \ + flowperpkt_tx_interface_add_del_reply) + + +/* M: construct, but don't yet send a message */ + +#define M(T,t) \ +do { \ + vam->result_ready = 0; \ + mp = vl_msg_api_alloc(sizeof(*mp)); \ + memset (mp, 0, sizeof (*mp)); \ + mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ + mp->client_index = vam->my_client_index; \ +} while(0); + +#define M2(T,t,n) \ +do { \ + vam->result_ready = 0; \ + mp = vl_msg_api_alloc(sizeof(*mp)+(n)); \ + memset (mp, 0, sizeof (*mp)); \ + mp->_vl_msg_id = ntohs (VL_API_##T + sm->msg_id_base); \ + mp->client_index = vam->my_client_index; \ +} while(0); + +/* S: send a message */ +#define S (vl_msg_api_send_shmem (vam->vl_input_queue, (u8 *)&mp)) + +/* W: wait for results, with timeout */ +#define W \ +do { \ + timeout = vat_time_now (vam) + 1.0; \ + \ + while (vat_time_now (vam) < timeout) { \ + if (vam->result_ready == 1) { \ + return (vam->retval); \ + } \ + } \ + return -99; \ +} while(0); + +static int +api_flowperpkt_tx_interface_add_del (vat_main_t * vam) +{ + flowperpkt_test_main_t *sm = &flowperpkt_test_main; + unformat_input_t *i = vam->input; + f64 timeout; + int enable_disable = 1; + u32 sw_if_index = ~0; + vl_api_flowperpkt_tx_interface_add_del_t *mp; + + /* Parse args required to build the message */ + while (unformat_check_input (i) != UNFORMAT_END_OF_INPUT) + { + if (unformat (i, "%U", unformat_sw_if_index, vam, &sw_if_index)) + ; + else if (unformat (i, "sw_if_index %d", &sw_if_index)) + ; + else if (unformat (i, "disable")) + enable_disable = 0; + else + break; + } + + if (sw_if_index == ~0) + { + errmsg ("missing interface name / explicit sw_if_index number \n"); + return -99; + } + + /* Construct the API message */ + M (FLOWPERPKT_TX_INTERFACE_ADD_DEL, flowperpkt_tx_interface_add_del); + mp->sw_if_index = ntohl (sw_if_index); + mp->is_add = enable_disable; + mp->is_ipv6 = 0; /* $$$$ */ + + /* send it... */ + S; + + /* Wait for a reply... */ + W; +} + +/* + * List of messages that the api test plugin sends, + * and that the data plane plugin processes + */ +#define foreach_vpe_api_msg \ +_(flowperpkt_tx_interface_add_del, "<intfc> [disable]") + +void +vat_api_hookup (vat_main_t * vam) +{ + flowperpkt_test_main_t *sm = &flowperpkt_test_main; + /* Hook up handlers for replies from the data plane plug-in */ +#define _(N,n) \ + vl_msg_api_set_handlers((VL_API_##N + sm->msg_id_base), \ + #n, \ + vl_api_##n##_t_handler, \ + vl_noop_handler, \ + vl_api_##n##_t_endian, \ + vl_api_##n##_t_print, \ + sizeof(vl_api_##n##_t), 1); + foreach_vpe_api_reply_msg; +#undef _ + + /* API messages we can send */ +#define _(n,h) hash_set_mem (vam->function_by_name, #n, api_##n); + foreach_vpe_api_msg; +#undef _ + + /* Help strings */ +#define _(n,h) hash_set_mem (vam->help_by_name, #n, h); + foreach_vpe_api_msg; +#undef _ +} + +clib_error_t * +vat_plugin_register (vat_main_t * vam) +{ + flowperpkt_test_main_t *sm = &flowperpkt_test_main; + u8 *name; + + sm->vat_main = vam; + + /* Ask the vpp engine for the first assigned message-id */ + name = format (0, "flowperpkt_%08x%c", api_version, 0); + sm->msg_id_base = vl_client_get_first_plugin_msg_id ((char *) name); + + /* Don't attempt to hook up API messages if the data plane plugin is AWOL */ + if (sm->msg_id_base != (u16) ~ 0) + vat_api_hookup (vam); + + vec_free (name); + + return 0; +} + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ diff --git a/plugins/flowperpkt-plugin/flowperpkt/node.c b/plugins/flowperpkt-plugin/flowperpkt/node.c new file mode 100644 index 00000000000..4deaae1a67c --- /dev/null +++ b/plugins/flowperpkt-plugin/flowperpkt/node.c @@ -0,0 +1,504 @@ +/* + * node.c - skeleton vpp engine plug-in dual-loop node skeleton + * + * Copyright (c) <current-year> <your-organization> + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <vlib/vlib.h> +#include <vnet/vnet.h> +#include <vnet/pg/pg.h> +#include <vppinfra/error.h> +#include <flowperpkt/flowperpkt.h> + +/** + * @file flow record generator graph node + */ + +typedef struct +{ + /** interface handle */ + u32 sw_if_index; + /** ToS bits */ + u8 tos; + /** packet timestamp */ + f64 timestamp; + /** size of the buffer */ + u16 buffer_size; +} flowperpkt_trace_t; + +/* packet trace format function */ +static u8 * +format_flowperpkt_trace (u8 * s, va_list * args) +{ + CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); + CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); + flowperpkt_trace_t *t = va_arg (*args, flowperpkt_trace_t *); + + s = format (s, + "FLOWPERPKT: sw_if_index %d, tos %0x2, timestamp %.6f, size %d", + t->sw_if_index, t->tos, t->timestamp, t->buffer_size); + return s; +} + +vlib_node_registration_t flowperpkt_node; + +#define foreach_flowperpkt_error \ +_(SWAPPED, "Mac swap packets processed") + +typedef enum +{ +#define _(sym,str) FLOWPERPKT_ERROR_##sym, + foreach_flowperpkt_error +#undef _ + FLOWPERPKT_N_ERROR, +} flowperpkt_error_t; + +static char *flowperpkt_error_strings[] = { +#define _(sym,string) string, + foreach_flowperpkt_error +#undef _ +}; + +typedef enum +{ + FLOWPERPKT_NEXT_DROP, + FLOWPERPKT_N_NEXT, +} flowperpkt_next_t; + +/** + * @brief add an entry to the flow record under construction + * @param vm vlib_main_t * current worker thread main structure pointer + * @param fm flowperpkt_main_t * flow-per-packet main structure pointer + * @param sw_if_index u32 interface handle + * @param tos u8 ToS bits from the packet + * @param timestamp f64 timestamp, floating-point seconds since vpp started + * @param length u16 ip length of the packet + * @param do_flush int 1 = flush all cached records, 0 = construct a record + */ + +static inline void +add_to_flow_record (vlib_main_t * vm, + flowperpkt_main_t * fm, + u32 sw_if_index, + u8 tos, f64 timestamp, u16 length, int do_flush) +{ + u32 my_cpu_number = vm->cpu_index; + flow_report_main_t *frm = &flow_report_main; + ip4_header_t *ip; + udp_header_t *udp; + ip4_ipfix_template_packet_t *tp; + ipfix_message_header_t *h; + ipfix_set_header_t *s; + vlib_frame_t *f; + vlib_buffer_t *b0; + u16 offset; + u32 bi0; + + /* Find or allocate a buffer */ + b0 = fm->buffers_per_worker[my_cpu_number]; + + /* Need to allocate a buffer? */ + if (PREDICT_FALSE (b0 == 0)) + { + /* Nothing to flush */ + if (do_flush) + return; + + /* $$$$ drop counter? */ + if (vlib_buffer_alloc (vm, &bi0, 1) != 1) + return; + b0 = fm->buffers_per_worker[my_cpu_number] = vlib_get_buffer (vm, bi0); + offset = 0; + } + else + { + /* use the current buffer */ + bi0 = vlib_get_buffer_index (vm, b0); + offset = fm->next_record_offset_per_worker[my_cpu_number]; + } + + /* Find or allocate a frame */ + f = fm->frames_per_worker[my_cpu_number]; + if (PREDICT_FALSE (f == 0)) + { + u32 *to_next; + f = vlib_get_frame_to_node (vm, ip4_lookup_node.index); + fm->frames_per_worker[my_cpu_number] = f; + + /* Enqueue the buffer */ + to_next = vlib_frame_vector_args (f); + to_next[0] = bi0; + f->n_vectors = 1; + } + + /* Fresh packet, construct header */ + if (PREDICT_FALSE (offset == 0)) + { + flow_report_stream_t *stream; + + stream = &frm->streams[0]; + + b0->current_data = 0; + b0->current_length = sizeof (*ip) + sizeof (*udp) + sizeof (*h) + + sizeof (*s); + b0->flags |= (VLIB_BUFFER_TOTAL_LENGTH_VALID | VLIB_BUFFER_FLOW_REPORT); + vnet_buffer (b0)->sw_if_index[VLIB_RX] = 0; + vnet_buffer (b0)->sw_if_index[VLIB_TX] = frm->fib_index; + + tp = vlib_buffer_get_current (b0); + ip = (ip4_header_t *) & tp->ip4; + udp = (udp_header_t *) (ip + 1); + h = (ipfix_message_header_t *) (udp + 1); + s = (ipfix_set_header_t *) (h + 1); + + ip->ip_version_and_header_length = 0x45; + ip->ttl = 254; + ip->protocol = IP_PROTOCOL_UDP; + ip->src_address.as_u32 = frm->src_address.as_u32; + ip->dst_address.as_u32 = frm->ipfix_collector.as_u32; + udp->src_port = clib_host_to_net_u16 (UDP_DST_PORT_ipfix); + udp->dst_port = clib_host_to_net_u16 (UDP_DST_PORT_ipfix); + + /* FIXUP: message header export_time */ + h->export_time = (u32) + (((f64) frm->unix_time_0) + + (vlib_time_now (frm->vlib_main) - frm->vlib_time_0)); + h->export_time = clib_host_to_net_u32 (h->export_time); + h->domain_id = clib_host_to_net_u32 (stream->domain_id); + + /* FIXUP: message header sequence_number */ + h->sequence_number = stream->sequence_number++; + h->sequence_number = clib_host_to_net_u32 (h->sequence_number); + + offset = (u32) (((u8 *) (s + 1)) - (u8 *) tp); + } + + /* Add data, unless we're flushing stale data */ + if (PREDICT_TRUE (do_flush == 0)) + { + + /* Add data */ + /* Egress interface */ + { + u32 egress_interface = clib_host_to_net_u32 (sw_if_index); + clib_memcpy (b0->data + offset, &egress_interface, + sizeof (egress_interface)); + offset += sizeof (egress_interface); + } + /* ToS */ + b0->data[offset++] = tos; + + /* Timestamp */ + clib_memcpy (b0->data + offset, ×tamp, sizeof (f64)); + offset += sizeof (f64); + + /* pkt size */ + { + u16 pkt_size = clib_host_to_net_u16 (length); + clib_memcpy (b0->data + offset, &pkt_size, sizeof (pkt_size)); + offset += sizeof (pkt_size); + } + + b0->current_length += + /* sw_if_index + tos + timestamp + length = 15 */ + sizeof (u32) + sizeof (u8) + sizeof (f64) + sizeof (u16); + + } + /* Time to flush the buffer? */ + if (PREDICT_FALSE + (do_flush || (offset + sizeof (u32) + sizeof (u8) + + sizeof (f64)) > frm->path_mtu)) + { + tp = vlib_buffer_get_current (b0); + ip = (ip4_header_t *) & tp->ip4; + udp = (udp_header_t *) (ip + 1); + h = (ipfix_message_header_t *) (udp + 1); + s = (ipfix_set_header_t *) (h + 1); + + s->set_id_length = ipfix_set_id_length (256, + b0->current_length - + (sizeof (*ip) + sizeof (*udp) + + sizeof (*h))); + h->version_length = version_length (b0->current_length - + (sizeof (*ip) + sizeof (*udp))); + + ip->length = clib_host_to_net_u16 (b0->current_length); + + ip->checksum = ip4_header_checksum (ip); + udp->length = clib_host_to_net_u16 (b0->current_length - sizeof (*ip)); + + if (frm->udp_checksum) + { + /* RFC 7011 section 10.3.2. */ + udp->checksum = ip4_tcp_udp_compute_checksum (vm, b0, ip); + if (udp->checksum == 0) + udp->checksum = 0xffff; + } + + ASSERT (ip->checksum == ip4_header_checksum (ip)); + + vlib_put_frame_to_node (vm, ip4_lookup_node.index, + fm->frames_per_worker[my_cpu_number]); + fm->frames_per_worker[my_cpu_number] = 0; + fm->buffers_per_worker[my_cpu_number] = 0; + offset = 0; + } + + fm->next_record_offset_per_worker[my_cpu_number] = offset; +} + +void +flowperpkt_flush_callback (void) +{ + vlib_main_t *vm = vlib_get_main (); + flowperpkt_main_t *fm = &flowperpkt_main; + + add_to_flow_record (vm, fm, 0 /* sw_if_index */ , + 0 /* ToS */ , + 0.0 /* timestamp */ , + 0 /* length */ , + 1 /* do_flush */ ); +} + + +static uword +flowperpkt_node_fn (vlib_main_t * vm, + vlib_node_runtime_t * node, vlib_frame_t * frame) +{ + u32 n_left_from, *from, *to_next; + flowperpkt_next_t next_index; + ip4_main_t *im = &ip4_main; + ip_lookup_main_t *lm = &im->lookup_main; + ip_config_main_t *cm = &lm->feature_config_mains[VNET_IP_TX_FEAT]; + flowperpkt_main_t *fm = &flowperpkt_main; + f64 now = vlib_time_now (vm); + + from = vlib_frame_vector_args (frame); + n_left_from = frame->n_vectors; + next_index = node->cached_next_index; + + while (n_left_from > 0) + { + u32 n_left_to_next; + + vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + + while (n_left_from >= 4 && n_left_to_next >= 2) + { + u32 next0 = FLOWPERPKT_NEXT_DROP; + u32 next1 = FLOWPERPKT_NEXT_DROP; + ip4_header_t *ip0, *ip1; + u16 len0, len1; + u32 bi0, bi1; + vlib_buffer_t *b0, *b1; + + /* Prefetch next iteration. */ + { + vlib_buffer_t *p2, *p3; + + p2 = vlib_get_buffer (vm, from[2]); + p3 = vlib_get_buffer (vm, from[3]); + + vlib_prefetch_buffer_header (p2, LOAD); + vlib_prefetch_buffer_header (p3, LOAD); + + CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE); + CLIB_PREFETCH (p3->data, CLIB_CACHE_LINE_BYTES, STORE); + } + + /* speculatively enqueue b0 and b1 to the current next frame */ + to_next[0] = bi0 = from[0]; + to_next[1] = bi1 = from[1]; + from += 2; + to_next += 2; + n_left_from -= 2; + n_left_to_next -= 2; + + b0 = vlib_get_buffer (vm, bi0); + b1 = vlib_get_buffer (vm, bi1); + + + vnet_get_config_data (&cm->config_main, + &b0->current_config_index, + &next0, 0 /* sizeof config data */ ); + + vnet_get_config_data (&cm->config_main, + &b1->current_config_index, + &next1, 0 /* sizeof config data */ ); + + ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) + + vnet_buffer (b0)->ip.save_rewrite_length); + + len0 = vlib_buffer_length_in_chain (vm, b0); + + if (PREDICT_TRUE ((b0->flags & VLIB_BUFFER_FLOW_REPORT) == 0)) + add_to_flow_record (vm, fm, + vnet_buffer (b0)->sw_if_index[VLIB_TX], + ip0->tos, now, len0, 0 /* flush */ ); + + ip1 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b1) + + vnet_buffer (b1)->ip.save_rewrite_length); + len1 = vlib_buffer_length_in_chain (vm, b1); + + if (PREDICT_TRUE ((b1->flags & VLIB_BUFFER_FLOW_REPORT) == 0)) + add_to_flow_record (vm, fm, + vnet_buffer (b1)->sw_if_index[VLIB_TX], + ip1->tos, now, len1, 0 /* flush */ ); + + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE))) + { + if (b0->flags & VLIB_BUFFER_IS_TRACED) + { + flowperpkt_trace_t *t = + vlib_add_trace (vm, node, b0, sizeof (*t)); + t->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX]; + t->tos = ip0->tos; + t->timestamp = now; + t->buffer_size = len0; + + } + if (b1->flags & VLIB_BUFFER_IS_TRACED) + { + flowperpkt_trace_t *t = + vlib_add_trace (vm, node, b1, sizeof (*t)); + + t->sw_if_index = vnet_buffer (b1)->sw_if_index[VLIB_TX]; + t->tos = ip1->tos; + t->timestamp = now; + t->buffer_size = len1; + } + } + + /* verify speculative enqueues, maybe switch current next frame */ + vlib_validate_buffer_enqueue_x2 (vm, node, next_index, + to_next, n_left_to_next, + bi0, bi1, next0, next1); + } + + while (n_left_from > 0 && n_left_to_next > 0) + { + u32 bi0; + vlib_buffer_t *b0; + u32 next0 = FLOWPERPKT_NEXT_DROP; + ip4_header_t *ip0; + u16 len0; + + /* speculatively enqueue b0 to the current next frame */ + bi0 = from[0]; + to_next[0] = bi0; + from += 1; + to_next += 1; + n_left_from -= 1; + n_left_to_next -= 1; + + b0 = vlib_get_buffer (vm, bi0); + + vnet_get_config_data (&cm->config_main, + &b0->current_config_index, + &next0, 0 /* sizeof config data */ ); + + ip0 = (ip4_header_t *) ((u8 *) vlib_buffer_get_current (b0) + + vnet_buffer (b0)->ip.save_rewrite_length); + /* + * egressInterface, TLV type 14, u32 + * ipClassOfService, TLV type 5, u8 + * flowStartNanoseconds, TLV type 156, dateTimeNanoseconds (f64) + * Implementation: f64 nanoseconds since VPP started + * dataLinkFrameSize, TLV type 312, u16 + */ + len0 = vlib_buffer_length_in_chain (vm, b0); + + if (PREDICT_TRUE ((b0->flags & VLIB_BUFFER_FLOW_REPORT) == 0)) + add_to_flow_record (vm, fm, + vnet_buffer (b0)->sw_if_index[VLIB_TX], + ip0->tos, now, len0, 0 /* do flush */ ); + + if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE) + && (b0->flags & VLIB_BUFFER_IS_TRACED))) + { + flowperpkt_trace_t *t = + vlib_add_trace (vm, node, b0, sizeof (*t)); + t->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX]; + t->tos = ip0->tos; + t->timestamp = now; + t->buffer_size = len0; + } + + /* verify speculative enqueue, maybe switch current next frame */ + vlib_validate_buffer_enqueue_x1 (vm, node, next_index, + to_next, n_left_to_next, + bi0, next0); + } + + vlib_put_next_frame (vm, node, next_index, n_left_to_next); + } + return frame->n_vectors; +} + +/** + * @brief IPFIX flow-per-packet graph node + * @node flowperpkt + * + * This is the IPFIX flow-record-per-packet node. + * + * @param vm vlib_main_t corresponding to the current thread. + * @param node vlib_node_runtime_t data for this node. + * @param frame vlib_frame_t whose contents should be dispatched. + * + * @par Graph mechanics: buffer metadata, next index usage + * + * <em>Uses:</em> + * - <code>vnet_buffer(b)->ip.save_rewrite_length</code> + * - tells the node the length of the rewrite which was applied in + * ip4/6_rewrite_inline, allows the code to find the IP header without + * having to parse L2 headers, or make stupid assumptions about their + * length. + * - <code>vnet_buffer(b)->flags & VLIB_BUFFER_FLOW_REPORT</code> + * - Used to suppress flow record generation for flow record packets. + * + * <em>Sets:</em> + * - <code>vnet_buffer(b)->flags & VLIB_BUFFER_FLOW_REPORT</code> + * - To suppress flow record generation for flow record packets + * + * <em>Next Index:</em> + * - Next configured output feature on the interface, usually + * "interface-output." Generated flow records head for ip4-lookup + */ + +/* *INDENT-OFF* */ +VLIB_REGISTER_NODE (flowperpkt_node) = { + .function = flowperpkt_node_fn, + .name = "flowperpkt", + .vector_size = sizeof (u32), + .format_trace = format_flowperpkt_trace, + .type = VLIB_NODE_TYPE_INTERNAL, + + .n_errors = ARRAY_LEN(flowperpkt_error_strings), + .error_strings = flowperpkt_error_strings, + + .n_next_nodes = FLOWPERPKT_N_NEXT, + + /* edit / add dispositions here */ + .next_nodes = { + [FLOWPERPKT_NEXT_DROP] = "error-drop", + }, +}; +/* *INDENT-ON* */ + +/* + * fd.io coding-style-patch-verification: ON + * + * Local Variables: + * eval: (c-set-style "gnu") + * End: + */ |