summaryrefslogtreecommitdiffstats
path: root/src/vnet/ip-neighbor/ip6_neighbor.c
blob: cf14954e96d0a31a0a1f13768724611c52a2c1ef (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
/*
 * ip/ip6_neighbor.c: IP6 neighbor handling
 *
 * Copyright (c) 2010 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <vnet/ip-neighbor/ip6_neighbor.h>
#include <vnet/util/throttle.h>
#include <vnet/fib/fib_sas.h>
#include <vnet/ip/ip_sas.h>

/** ND throttling */
static throttle_t nd_throttle;

VLIB_REGISTER_LOG_CLASS (ip6_neighbor_log, static) = {
  .class_name = "ip6",
  .subclass_name = "neighbor",
};

#define log_debug(fmt, ...)                                                   \
  vlib_log_debug (ip6_neighbor_log.class, fmt, __VA_ARGS__)
void
ip6_neighbor_probe_dst (u32 sw_if_index, const ip6_address_t * dst)
{
  ip6_address_t src;

  if (fib_sas6_get (sw_if_index, dst, &src) ||
      ip6_sas_by_sw_if_index (sw_if_index, dst, &src))
    ip6_neighbor_probe (vlib_get_main (), vnet_get_main (),
			sw_if_index, &src, dst);
}

void
ip6_neighbor_advertise (vlib_main_t * vm,
			vnet_main_t * vnm,
			u32 sw_if_index, const ip6_address_t * addr)
{
  vnet_hw_interface_t *hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
  ip6_main_t *i6m = &ip6_main;
  u8 *rewrite, rewrite_len;
  u8 dst_address[6];

  if (NULL == addr)
    addr = ip6_interface_first_address (i6m, sw_if_index);

  if (addr)
    {
      log_debug ("Sending unsolicitated NA IP6 address %U on sw_if_idex %d",
		 format_ip6_address, addr, sw_if_index);

      /* Form unsolicited neighbor advertisement packet from NS pkt template */
      int bogus_length;
      u32 bi = 0;
      icmp6_neighbor_solicitation_header_t *h =
	vlib_packet_template_get_packet (vm,
					 &ip6_neighbor_packet_template,
					 &bi);
      if (!h)
	return;

      ip6_set_reserved_multicast_address (&h->ip.dst_address,
					  IP6_MULTICAST_SCOPE_link_local,
					  IP6_MULTICAST_GROUP_ID_all_hosts);
      h->ip.src_address = addr[0];
      h->neighbor.icmp.type = ICMP6_neighbor_advertisement;
      h->neighbor.target_address = addr[0];
      h->neighbor.advertisement_flags = clib_host_to_net_u32
	(ICMP6_NEIGHBOR_ADVERTISEMENT_FLAG_OVERRIDE);
      h->link_layer_option.header.type =
	ICMP6_NEIGHBOR_DISCOVERY_OPTION_target_link_layer_address;
      clib_memcpy (h->link_layer_option.ethernet_address,
		   hi->hw_address, vec_len (hi->hw_address));
      h->neighbor.icmp.checksum =
	ip6_tcp_udp_icmp_compute_checksum (vm, 0, &h->ip, &bogus_length);
      ASSERT (bogus_length == 0);

      /* Setup MAC header with IP6 Etype and mcast DMAC */
      vlib_buffer_t *b = vlib_get_buffer (vm, bi);
      ip6_multicast_ethernet_address (dst_address,
				      IP6_MULTICAST_GROUP_ID_all_hosts);
      rewrite =
	ethernet_build_rewrite (vnm, sw_if_index, VNET_LINK_IP6, dst_address);
      rewrite_len = vec_len (rewrite);
      vlib_buffer_advance (b, -rewrite_len);
      ethernet_header_t *e = vlib_buffer_get_current (b);
      clib_memcpy (e->dst_address, rewrite, rewrite_len);
      vec_free (rewrite);

      /* Send unsolicited ND advertisement packet out the specified interface */
      vnet_buffer (b)->sw_if_index[VLIB_RX] =
	vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
      vlib_frame_t *f = vlib_get_frame_to_node (vm, hi->output_node_index);
      u32 *to_next = vlib_frame_vector_args (f);
      to_next[0] = bi;
      f->n_vectors = 1;
      vlib_put_frame_to_node (vm, hi->output_node_index, f);
    }
}

typedef enum
{
  IP6_NBR_NEXT_DROP,
  IP6_NBR_NEXT_REPLY_TX,
  IP6_NBR_N_NEXT,
} ip6_discover_neighbor_next_t;

typedef enum
{
  IP6_NBR_ERROR_DROP,
  IP6_NBR_ERROR_REQUEST_SENT,
  IP6_NBR_ERROR_NO_SOURCE_ADDRESS,
  IP6_NBR_ERROR_NO_BUFFERS,
} ip6_discover_neighbor_error_t;

static uword
ip6_discover_neighbor_inline (vlib_main_t * vm,
			      vlib_node_runtime_t * node,
			      vlib_frame_t * frame, int is_glean)
{
  vnet_main_t *vnm = vnet_get_main ();
  u32 *from, *to_next_drop;
  uword n_left_from, n_left_to_next_drop;
  u64 seed;
  u32 thread_index = vm->thread_index;

  if (node->flags & VLIB_NODE_FLAG_TRACE)
    ip6_forward_next_trace (vm, node, frame, VLIB_TX);

  seed = throttle_seed (&nd_throttle, thread_index, vlib_time_now (vm));

  from = vlib_frame_vector_args (frame);
  n_left_from = frame->n_vectors;

  while (n_left_from > 0)
    {
      vlib_get_next_frame (vm, node, IP6_NBR_NEXT_DROP,
			   to_next_drop, n_left_to_next_drop);

      while (n_left_from > 0 && n_left_to_next_drop > 0)
	{
	  u32 pi0, adj_index0, sw_if_index0, drop0, r0;
	  vnet_hw_interface_t *hw_if0;
	  vlib_buffer_t *p0, *b0;
	  ip_adjacency_t *adj0;
	  ip6_address_t src;
	  ip6_header_t *ip0;

	  pi0 = from[0];

	  p0 = vlib_get_buffer (vm, pi0);

	  adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];

	  ip0 = vlib_buffer_get_current (p0);

	  adj0 = adj_get (adj_index0);

	  if (!is_glean)
	    {
	      ip0->dst_address.as_u64[0] =
		adj0->sub_type.nbr.next_hop.ip6.as_u64[0];
	      ip0->dst_address.as_u64[1] =
		adj0->sub_type.nbr.next_hop.ip6.as_u64[1];
	    }

	  sw_if_index0 = adj0->rewrite_header.sw_if_index;
	  vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0;

	  /* combine the address and interface for a hash */
	  r0 = ip6_address_hash_to_u64 (&ip0->dst_address) ^ sw_if_index0;

	  drop0 = throttle_check (&nd_throttle, thread_index, r0, seed);

	  from += 1;
	  n_left_from -= 1;
	  to_next_drop[0] = pi0;
	  to_next_drop += 1;
	  n_left_to_next_drop -= 1;

	  hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);

	  /* If the interface is link-down, drop the pkt */
	  if (!(hw_if0->flags & VNET_HW_INTERFACE_FLAG_LINK_UP))
	    drop0 = 1;

	  if (!ip6_link_is_enabled (sw_if_index0))
	    drop0 = 1;

	  /*
	   * the adj has been updated to a rewrite but the node the DPO that got
	   * us here hasn't - yet. no big deal. we'll drop while we wait.
	   */
	  if (IP_LOOKUP_NEXT_REWRITE == adj0->lookup_next_index)
	    drop0 = 1;

	  if (drop0)
	    {
	      p0->error = node->errors[IP6_NBR_ERROR_DROP];
	      continue;
	    }

	  /*
	   * Choose source address based on destination lookup
	   * adjacency.
	   */
	  if (!fib_sas6_get (sw_if_index0, &ip0->dst_address, &src) ||
	      !ip6_sas_by_sw_if_index (sw_if_index0, &ip0->dst_address, &src))
	    {
	      /* There is no address on the interface */
	      p0->error = node->errors[IP6_NBR_ERROR_NO_SOURCE_ADDRESS];
	      continue;
	    }

	  b0 = ip6_neighbor_probe (vm, vnm, sw_if_index0,
				   &src, &ip0->dst_address);

	  if (PREDICT_TRUE (NULL != b0))
	    {
	      clib_memcpy_fast (b0->opaque2, p0->opaque2,
				sizeof (p0->opaque2));
	      b0->flags |= p0->flags & VLIB_BUFFER_IS_TRACED;
	      b0->trace_handle = p0->trace_handle;
	      p0->error = node->errors[IP6_NBR_ERROR_REQUEST_SENT];
	    }
	  else
	    {
	      /* There is no address on the interface */
	      p0->error = node->errors[IP6_NBR_ERROR_NO_BUFFERS];
	      continue;
	    }
	}

      vlib_put_next_frame (vm, node, IP6_NBR_NEXT_DROP, n_left_to_next_drop);
    }

  return frame->n_vectors;
}

static uword
ip6_discover_neighbor (vlib_main_t * vm,
		       vlib_node_runtime_t * node, vlib_frame_t * frame)
{
  return (ip6_discover_neighbor_inline (vm, node, frame, 0));
}

static uword
ip6_glean (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * frame)
{
  return (ip6_discover_neighbor_inline (vm, node, frame, 1));
}

static char *ip6_discover_neighbor_error_strings[] = {
  [IP6_NBR_ERROR_DROP] = "address overflow drops",
  [IP6_NBR_ERROR_REQUEST_SENT] = "neighbor solicitations sent",
  [IP6_NBR_ERROR_NO_SOURCE_ADDRESS] = "no source address for ND solicitation",
  [IP6_NBR_ERROR_NO_BUFFERS] = "no buffers",
};

/* *INDENT-OFF* */
VLIB_REGISTER_NODE (ip6_glean_node) =
{
  .function = ip6_glean,
  .name = "ip6-glean",
  .vector_size = sizeof (u32),
  .format_trace = format_ip6_forward_next_trace,
  .n_errors = ARRAY_LEN (ip6_discover_neighbor_error_strings),
  .error_strings = ip6_discover_neighbor_error_strings,
  .n_next_nodes = IP6_NBR_N_NEXT,
  .next_nodes =
  {
    [IP6_NBR_NEXT_DROP] = "ip6-drop",
    [IP6_NBR_NEXT_REPLY_TX] = "ip6-rewrite-mcast",
  },
};
VLIB_REGISTER_NODE (ip6_discover_neighbor_node) =
{
  .function = ip6_discover_neighbor,
  .name = "ip6-discover-neighbor",
  .vector_size = sizeof (u32),
  .format_trace = format_ip6_forward_next_trace,
  .n_errors = ARRAY_LEN (ip6_discover_neighbor_error_strings),
  .error_strings = ip6_discover_neighbor_error_strings,
  .n_next_nodes = IP6_NBR_N_NEXT,
  .next_nodes =
  {
    [IP6_NBR_NEXT_DROP] = "ip6-drop",
    [IP6_NBR_NEXT_REPLY_TX] = "ip6-rewrite-mcast",
  },
};
/* *INDENT-ON* */

/* Template used to generate IP6 neighbor solicitation packets. */
vlib_packet_template_t ip6_neighbor_packet_template;

static clib_error_t *
ip6_neighbor_init (vlib_main_t * vm)
{
  icmp6_neighbor_solicitation_header_t p;

  clib_memset (&p, 0, sizeof (p));

  p.ip.ip_version_traffic_class_and_flow_label =
    clib_host_to_net_u32 (0x6 << 28);
  p.ip.payload_length =
    clib_host_to_net_u16 (sizeof (p) -
			  STRUCT_OFFSET_OF
			  (icmp6_neighbor_solicitation_header_t, neighbor));
  p.ip.protocol = IP_PROTOCOL_ICMP6;
  p.ip.hop_limit = 255;
  ip6_set_solicited_node_multicast_address (&p.ip.dst_address, 0);

  p.neighbor.icmp.type = ICMP6_neighbor_solicitation;

  p.link_layer_option.header.type =
    ICMP6_NEIGHBOR_DISCOVERY_OPTION_source_link_layer_address;
  p.link_layer_option.header.n_data_u64s =
    sizeof (p.link_layer_option) / sizeof (u64);

  vlib_packet_template_init (vm,
			     &ip6_neighbor_packet_template, &p, sizeof (p),
			     /* alloc chunk size */ 8,
			     "ip6 neighbor discovery");

  return NULL;
}

VLIB_INIT_FUNCTION (ip6_neighbor_init);

static clib_error_t *
ip6_nd_main_loop_enter (vlib_main_t * vm)
{
  vlib_thread_main_t *tm = &vlib_thread_main;

  throttle_init (&nd_throttle, tm->n_vlib_mains, 1e-3);

  return 0;
}

VLIB_MAIN_LOOP_ENTER_FUNCTION (ip6_nd_main_loop_enter);

/*
 * fd.io coding-style-patch-verification: ON
 *
 * Local Variables:
 * eval: (c-set-style "gnu")
 * End:
 */
ss="o">= x; serialize (m, serialize_64, y.i); } void serialize_f32 (serialize_main_t * m, va_list * va) { f32 x = va_arg (*va, f64); union { f32 f; u32 i; } y; y.f = x; serialize_integer (m, y.i, sizeof (y.i)); } void unserialize_f64 (serialize_main_t * m, va_list * va) { f64 *x = va_arg (*va, f64 *); union { f64 f; u64 i; } y; unserialize (m, unserialize_64, &y.i); *x = y.f; } void unserialize_f32 (serialize_main_t * m, va_list * va) { f32 *x = va_arg (*va, f32 *); union { f32 f; u32 i; } y; unserialize_integer (m, &y.i, sizeof (y.i)); *x = y.f; } __clib_export void serialize_cstring (serialize_main_t * m, char *s) { u32 len = s ? strlen (s) : 0; void *p; serialize_likely_small_unsigned_integer (m, len); if (len > 0) { p = serialize_get (m, len); clib_memcpy_fast (p, s, len); } } __clib_export void unserialize_cstring (serialize_main_t * m, char **s) { char *p, *r = 0; u32 len; len = unserialize_likely_small_unsigned_integer (m); /* * Given broken enough data, we could get len = 0xFFFFFFFF. * Add one, it overflows, we call vec_new (char, 0), then * memcpy until we bus error. */ if (len > 0 && len != 0xFFFFFFFF) { r = vec_new (char, len + 1); p = unserialize_get (m, len); clib_memcpy_fast (r, p, len); /* Null terminate. */ r[len] = 0; } *s = r; } /* vec_serialize/vec_unserialize helper functions for basic vector types. */ void serialize_vec_8 (serialize_main_t * m, va_list * va) { u8 *s = va_arg (*va, u8 *); u32 n = va_arg (*va, u32); u8 *p = serialize_get (m, n * sizeof (u8)); clib_memcpy_fast (p, s, n * sizeof (u8)); } void unserialize_vec_8 (serialize_main_t * m, va_list * va) { u8 *s = va_arg (*va, u8 *); u32 n = va_arg (*va, u32); u8 *p = unserialize_get (m, n); clib_memcpy_fast (s, p, n); } #define _(n_bits) \ void serialize_vec_##n_bits (serialize_main_t * m, va_list * va) \ { \ u##n_bits * s = va_arg (*va, u##n_bits *); \ u32 n = va_arg (*va, u32); \ u##n_bits * p = serialize_get (m, n * sizeof (s[0])); \ \ while (n >= 4) \ { \ p[0] = clib_host_to_net_u##n_bits (s[0]); \ p[1] = clib_host_to_net_u##n_bits (s[1]); \ p[2] = clib_host_to_net_u##n_bits (s[2]); \ p[3] = clib_host_to_net_u##n_bits (s[3]); \ s += 4; \ p += 4; \ n -= 4; \ } \ \ while (n >= 1) \ { \ p[0] = clib_host_to_net_u##n_bits (s[0]); \ s += 1; \ p += 1; \ n -= 1; \ } \ } \ \ void unserialize_vec_##n_bits (serialize_main_t * m, va_list * va) \ { \ u##n_bits * s = va_arg (*va, u##n_bits *); \ u32 n = va_arg (*va, u32); \ u##n_bits * p = unserialize_get (m, n * sizeof (s[0])); \ \ while (n >= 4) \ { \ s[0] = clib_net_to_host_mem_u##n_bits (&p[0]); \ s[1] = clib_net_to_host_mem_u##n_bits (&p[1]); \ s[2] = clib_net_to_host_mem_u##n_bits (&p[2]); \ s[3] = clib_net_to_host_mem_u##n_bits (&p[3]); \ s += 4; \ p += 4; \ n -= 4; \ } \ \ while (n >= 1) \ { \ s[0] = clib_net_to_host_mem_u##n_bits (&p[0]); \ s += 1; \ p += 1; \ n -= 1; \ } \ } _(16); _(32); _(64); #undef _ #define SERIALIZE_VECTOR_CHUNK_SIZE 64 __clib_export void serialize_vector (serialize_main_t * m, va_list * va) { void *vec = va_arg (*va, void *); u32 elt_bytes = va_arg (*va, u32); serialize_function_t *f = va_arg (*va, serialize_function_t *); u32 l = vec_len (vec); void *p = vec; serialize_integer (m, l, sizeof (l)); /* Serialize vector in chunks for cache locality. */ while (l != 0) { u32 n = clib_min (SERIALIZE_VECTOR_CHUNK_SIZE, l); serialize (m, f, p, n); l -= n; p += SERIALIZE_VECTOR_CHUNK_SIZE * elt_bytes; } } void * unserialize_vector_ha (serialize_main_t * m, u32 elt_bytes, u32 header_bytes, u32 align, u32 max_length, serialize_function_t * f) { void *v, *p; u32 l; vec_attr_t va = { .align = align, .elt_sz = elt_bytes, .hdr_sz = header_bytes }; unserialize_integer (m, &l, sizeof (l)); if (l > max_length) serialize_error (&m->header, clib_error_create ("bad vector length %d", l)); p = v = _vec_alloc_internal (l, &va); while (l != 0) { u32 n = clib_min (SERIALIZE_VECTOR_CHUNK_SIZE, l); unserialize (m, f, p, n); l -= n; p += SERIALIZE_VECTOR_CHUNK_SIZE * elt_bytes; } return v; } void unserialize_aligned_vector (serialize_main_t * m, va_list * va) { void **vec = va_arg (*va, void **); u32 elt_bytes = va_arg (*va, u32); serialize_function_t *f = va_arg (*va, serialize_function_t *); u32 align = va_arg (*va, u32); *vec = unserialize_vector_ha (m, elt_bytes, /* header_bytes */ 0, /* align */ align, /* max_length */ ~0, f); } __clib_export void unserialize_vector (serialize_main_t * m, va_list * va) { void **vec = va_arg (*va, void **); u32 elt_bytes = va_arg (*va, u32); serialize_function_t *f = va_arg (*va, serialize_function_t *); *vec = unserialize_vector_ha (m, elt_bytes, /* header_bytes */ 0, /* align */ 0, /* max_length */ ~0, f); } void serialize_bitmap (serialize_main_t * m, uword * b) { u32 l, i, n_u32s; l = vec_len (b); n_u32s = l * sizeof (b[0]) / sizeof (u32); serialize_integer (m, n_u32s, sizeof (n_u32s)); /* Send 32 bit words, low-order word first on 64 bit. */ for (i = 0; i < l; i++) { serialize_integer (m, b[i], sizeof (u32)); if (BITS (uword) == 64) serialize_integer (m, (u64) b[i] >> (u64) 32, sizeof (u32)); } } uword * unserialize_bitmap (serialize_main_t * m) { uword *b = 0; u32 i, n_u32s; unserialize_integer (m, &n_u32s, sizeof (n_u32s)); if (n_u32s == 0) return b; i = (n_u32s * sizeof (u32) + sizeof (b[0]) - 1) / sizeof (b[0]); vec_resize (b, i); for (i = 0; i < n_u32s; i++) { u32 data; unserialize_integer (m, &data, sizeof (u32)); /* Low-word is first on 64 bit. */ if (BITS (uword) == 64) { if ((i % 2) == 0) b[i / 2] |= (u64) data << (u64) 0; else b[i / 2] |= (u64) data << (u64) 32; } else { b[i] = data; } } return b; } void serialize_pool (serialize_main_t * m, va_list * va) { void *pool = va_arg (*va, void *); u32 elt_bytes = va_arg (*va, u32); serialize_function_t *f = va_arg (*va, serialize_function_t *); u32 l, lo, hi; pool_header_t *p; l = vec_len (pool); serialize_integer (m, l, sizeof (u32)); if (l == 0) return; p = pool_header (pool); /* No need to send free bitmap. Need to send index vector to guarantee that unserialized pool will be identical. */ vec_serialize (m, p->free_indices, serialize_vec_32); pool_foreach_region (lo, hi, pool, serialize (m, f, pool + lo * elt_bytes, hi - lo)); } static void * unserialize_pool_helper (serialize_main_t * m, u32 elt_bytes, u32 align, serialize_function_t * f) { void *v; u32 i, l, lo, hi; pool_header_t *p; vec_attr_t va = { .align = align, .elt_sz = elt_bytes, .hdr_sz = sizeof (pool_header_t) }; unserialize_integer (m, &l, sizeof (l)); if (l == 0) { return 0; } v = _vec_alloc_internal (l, &va); p = pool_header (v); vec_unserialize (m, &p->free_indices, unserialize_vec_32); /* Construct free bitmap. */ p->free_bitmap = 0; for (i = 0; i < vec_len (p->free_indices); i++) p->free_bitmap = clib_bitmap_ori (p->free_bitmap, p->free_indices[i]); pool_foreach_region (lo, hi, v, unserialize (m, f, v + lo * elt_bytes, hi - lo)); return v; } void unserialize_pool (serialize_main_t * m, va_list * va) { void **result = va_arg (*va, void **); u32 elt_bytes = va_arg (*va, u32); serialize_function_t *f = va_arg (*va, serialize_function_t *); *result = unserialize_pool_helper (m, elt_bytes, /* align */ 0, f); } void unserialize_aligned_pool (serialize_main_t * m, va_list * va) { void **result = va_arg (*va, void **); u32 elt_bytes = va_arg (*va, u32); u32 align = va_arg (*va, u32); serialize_function_t *f = va_arg (*va, serialize_function_t *); *result = unserialize_pool_helper (m, elt_bytes, align, f); } static void serialize_vec_heap_elt (serialize_main_t * m, va_list * va) { heap_elt_t *e = va_arg (*va, heap_elt_t *); u32 i, n = va_arg (*va, u32); for (i = 0; i < n; i++) { serialize_integer (m, e[i].offset, sizeof (e[i].offset)); serialize_integer (m, e[i].next, sizeof (e[i].next)); serialize_integer (m, e[i].prev, sizeof (e[i].prev)); } } static void unserialize_vec_heap_elt (serialize_main_t * m, va_list * va) { heap_elt_t *e = va_arg (*va, heap_elt_t *); u32 i, n = va_arg (*va, u32); for (i = 0; i < n; i++) { unserialize_integer (m, &e[i].offset, sizeof (e[i].offset)); unserialize_integer (m, &e[i].next, sizeof (e[i].next)); unserialize_integer (m, &e[i].prev, sizeof (e[i].prev)); } } void serialize_heap (serialize_main_t * m, va_list * va) { void *heap = va_arg (*va, void *); serialize_function_t *f = va_arg (*va, serialize_function_t *); u32 i, l; heap_header_t *h; l = vec_len (heap); serialize_integer (m, l, sizeof (u32)); if (l == 0) return; h = heap_header (heap); #define foreach_serialize_heap_header_integer \ _ (head) _ (tail) _ (used_count) _ (max_len) _ (flags) _ (elt_bytes) #define _(f) serialize_integer (m, h->f, sizeof (h->f)); foreach_serialize_heap_header_integer; #undef _ serialize_integer (m, vec_len (h->free_lists), sizeof (u32)); for (i = 0; i < vec_len (h->free_lists); i++) vec_serialize (m, h->free_lists[i], serialize_vec_32); vec_serialize (m, h->elts, serialize_vec_heap_elt); vec_serialize (m, h->small_free_elt_free_index, serialize_vec_32); vec_serialize (m, h->free_elts, serialize_vec_32); /* Serialize data in heap. */ { heap_elt_t *e, *end; e = h->elts + h->head; end = h->elts + h->tail; while (1) { if (!heap_is_free (e)) { void *v = heap + heap_offset (e) * h->elt_bytes; u32 n = heap_elt_size (heap, e); serialize (m, f, v, n); } if (e == end) break; e = heap_next (e); } } } void unserialize_heap (serialize_main_t * m, va_list * va) { void **result = va_arg (*va, void **); serialize_function_t *f = va_arg (*va, serialize_function_t *); u32 i, vl, fl; heap_header_t h; void *heap; unserialize_integer (m, &vl, sizeof (u32)); if (vl == 0) { *result = 0; return; } clib_memset (&h, 0, sizeof (h)); #define _(f) unserialize_integer (m, &h.f, sizeof (h.f)); foreach_serialize_heap_header_integer; #undef _ unserialize_integer (m, &fl, sizeof (u32)); vec_resize (h.free_lists, fl); for (i = 0; i < vec_len (h.free_lists); i++) vec_unserialize (m, &h.free_lists[i], unserialize_vec_32); vec_unserialize (m, &h.elts, unserialize_vec_heap_elt); vec_unserialize (m, &h.small_free_elt_free_index, unserialize_vec_32); vec_unserialize (m, &h.free_elts, unserialize_vec_32); /* Re-construct used elt bitmap. */ if (CLIB_DEBUG > 0) { heap_elt_t *e; vec_foreach (e, h.elts) { if (!heap_is_free (e)) h.used_elt_bitmap = clib_bitmap_ori (h.used_elt_bitmap, e - h.elts); } } heap = *result = _heap_new (vl, h.elt_bytes); heap_header (heap)[0] = h; /* Unserialize data in heap. */ { heap_elt_t *e, *end; e = h.elts + h.head; end = h.elts + h.tail; while (1) { if (!heap_is_free (e)) { void *v = heap + heap_offset (e) * h.elt_bytes; u32 n = heap_elt_size (heap, e); unserialize (m, f, v, n); } if (e == end) break; e = heap_next (e); } } } void serialize_magic (serialize_main_t * m, void *magic, u32 magic_bytes) { void *p; serialize_integer (m, magic_bytes, sizeof (magic_bytes)); p = serialize_get (m, magic_bytes); clib_memcpy_fast (p, magic, magic_bytes); } void unserialize_check_magic (serialize_main_t * m, void *magic, u32 magic_bytes) { u32 l; void *d; unserialize_integer (m, &l, sizeof (l)); if (l != magic_bytes) { bad: serialize_error_return (m, "bad magic number"); } d = serialize_get (m, magic_bytes); if (memcmp (magic, d, magic_bytes)) goto bad; } __clib_export clib_error_t * va_serialize (serialize_main_t * sm, va_list * va) { serialize_main_header_t *m = &sm->header; serialize_function_t *f = va_arg (*va, serialize_function_t *); clib_error_t *error = 0; m->recursion_level += 1; if (m->recursion_level == 1) { uword r = clib_setjmp (&m->error_longjmp, 0); error = uword_to_pointer (r, clib_error_t *); } if (!error) f (sm, va); m->recursion_level -= 1; return error; } __clib_export clib_error_t * serialize (serialize_main_t * m, ...) { clib_error_t *error; va_list va; va_start (va, m); error = va_serialize (m, &va); va_end (va); return error; } __clib_export clib_error_t * unserialize (serialize_main_t * m, ...) { clib_error_t *error; va_list va; va_start (va, m); error = va_serialize (m, &va); va_end (va); return error; } static void * serialize_write_not_inline (serialize_main_header_t * m, serialize_stream_t * s, uword n_bytes_to_write, uword flags) { uword cur_bi, n_left_b, n_left_o; ASSERT (s->current_buffer_index <= s->n_buffer_bytes); cur_bi = s->current_buffer_index; n_left_b = s->n_buffer_bytes - cur_bi; n_left_o = vec_len (s->overflow_buffer); /* Prepend overflow buffer if present. */ do { if (n_left_o > 0 && n_left_b > 0) { uword n = clib_min (n_left_b, n_left_o); clib_memcpy_fast (s->buffer + cur_bi, s->overflow_buffer, n); cur_bi += n; n_left_b -= n; n_left_o -= n; if (n_left_o == 0) vec_set_len (s->overflow_buffer, 0); else vec_delete (s->overflow_buffer, n, 0); } /* Call data function when buffer is complete. Data function should dispatch with current buffer and give us a new one to write more data into. */ if (n_left_b == 0) { s->current_buffer_index = cur_bi; m->data_function (m, s); cur_bi = s->current_buffer_index; n_left_b = s->n_buffer_bytes - cur_bi; } } while (n_left_o > 0); if (n_left_o > 0 || n_left_b < n_bytes_to_write) { u8 *r; vec_add2 (s->overflow_buffer, r, n_bytes_to_write); return r; } else { s->current_buffer_index = cur_bi + n_bytes_to_write; return s->buffer + cur_bi; } } static void * serialize_read_not_inline (serialize_main_header_t * m, serialize_stream_t * s, uword n_bytes_to_read, uword flags) { uword cur_bi, cur_oi, n_left_b, n_left_o, n_left_to_read; ASSERT (s->current_buffer_index <= s->n_buffer_bytes); cur_bi = s->current_buffer_index; cur_oi = s->current_overflow_index; n_left_b = s->n_buffer_bytes - cur_bi; n_left_o = vec_len (s->overflow_buffer) - cur_oi; /* Read from overflow? */ if (n_left_o >= n_bytes_to_read) { s->current_overflow_index = cur_oi + n_bytes_to_read; return vec_elt_at_index (s->overflow_buffer, cur_oi); } /* Reset overflow buffer. */ if (n_left_o == 0 && s->overflow_buffer) { s->current_overflow_index = 0; vec_set_len (s->overflow_buffer, 0); } n_left_to_read = n_bytes_to_read; while (n_left_to_read > 0) { uword n; /* If we don't have enough data between overflow and normal buffer call read function. */ if (n_left_o + n_left_b < n_bytes_to_read) { /* Save any left over buffer in overflow vector. */ if (n_left_b > 0) { vec_add (s->overflow_buffer, s->buffer + cur_bi, n_left_b); n_left_o += n_left_b; n_left_to_read -= n_left_b; /* Advance buffer to end --- even if SERIALIZE_FLAG_NO_ADVANCE_CURRENT_BUFFER_INDEX is set. */ cur_bi = s->n_buffer_bytes; n_left_b = 0; } if (m->data_function) { m->data_function (m, s); cur_bi = s->current_buffer_index; n_left_b = s->n_buffer_bytes - cur_bi; } } /* For first time through loop return if we have enough data in normal buffer and overflow vector is empty. */ if (n_left_o == 0 && n_left_to_read == n_bytes_to_read && n_left_b >= n_left_to_read) { s->current_buffer_index = cur_bi + n_bytes_to_read; return s->buffer + cur_bi; } if (!m->data_function || serialize_stream_is_end_of_stream (s)) { /* This can happen for a peek at end of file. Pad overflow buffer with 0s. */ vec_resize (s->overflow_buffer, n_left_to_read); n_left_o += n_left_to_read; n_left_to_read = 0; } else { /* Copy from buffer to overflow vector. */ n = clib_min (n_left_to_read, n_left_b); vec_add (s->overflow_buffer, s->buffer + cur_bi, n); cur_bi += n; n_left_b -= n; n_left_o += n; n_left_to_read -= n; } } s->current_buffer_index = cur_bi; s->current_overflow_index = cur_oi + n_bytes_to_read; return vec_elt_at_index (s->overflow_buffer, cur_oi); } __clib_export void * serialize_read_write_not_inline (serialize_main_header_t * m, serialize_stream_t * s, uword n_bytes, uword flags) { return (((flags & SERIALIZE_FLAG_IS_READ) ? serialize_read_not_inline : serialize_write_not_inline) (m, s, n_bytes, flags)); } static void serialize_read_write_close (serialize_main_header_t * m, serialize_stream_t * s, uword flags) { if (serialize_stream_is_end_of_stream (s)) return; if (flags & SERIALIZE_FLAG_IS_WRITE) /* "Write" 0 bytes to flush overflow vector. */ serialize_write_not_inline (m, s, /* n bytes */ 0, flags); serialize_stream_set_end_of_stream (s); /* Call it one last time to flush buffer and close. */ m->data_function (m, s); vec_free (s->overflow_buffer); } __clib_export void serialize_close (serialize_main_t * m) { serialize_read_write_close (&m->header, &m->stream, SERIALIZE_FLAG_IS_WRITE); } __clib_export void unserialize_close (serialize_main_t * m) { serialize_read_write_close (&m->header, &m->stream, SERIALIZE_FLAG_IS_READ); } __clib_export void serialize_open_data (serialize_main_t * m, u8 * data, uword n_data_bytes) { clib_memset (m, 0, sizeof (m[0])); m->stream.buffer = data; m->stream.n_buffer_bytes = n_data_bytes; } __clib_export void unserialize_open_data (serialize_main_t * m, u8 * data, uword n_data_bytes) { serialize_open_data (m, data, n_data_bytes); } static void serialize_vector_write (serialize_main_header_t * m, serialize_stream_t * s) { if (!serialize_stream_is_end_of_stream (s)) { /* Double buffer size. */ uword l = vec_len (s->buffer); vec_resize (s->buffer, l > 0 ? l : 64); s->n_buffer_bytes = vec_len (s->buffer); } } __clib_export void serialize_open_vector (serialize_main_t * m, u8 * vector) { clib_memset (m, 0, sizeof (m[0])); m->header.data_function = serialize_vector_write; m->stream.buffer = vector; m->stream.current_buffer_index = 0; m->stream.n_buffer_bytes = vec_len (vector); } __clib_export void * serialize_close_vector (serialize_main_t * m) { serialize_stream_t *s = &m->stream; void *result; serialize_close (m); /* frees overflow buffer */ if (s->buffer) vec_set_len (s->buffer, s->current_buffer_index); result = s->buffer; clib_memset (m, 0, sizeof (m[0])); return result; } __clib_export void serialize_multiple_1 (serialize_main_t *m, void *data, uword data_stride, uword n_data) { u8 *d = data; u8 *p; uword n_left = n_data; while (n_left >= 4) { p = serialize_get (m, 4 * sizeof (d[0])); p[0] = d[0 * data_stride]; p[1] = d[1 * data_stride]; p[2] = d[2 * data_stride]; p[3] = d[3 * data_stride]; n_left -= 4; d += 4 * data_stride; } if (n_left > 0) { p = serialize_get (m, n_left * sizeof (p[0])); while (n_left > 0) { p[0] = d[0]; p += 1; d += 1 * data_stride; n_left -= 1; } } } __clib_export void serialize_multiple_2 (serialize_main_t *m, void *data, uword data_stride, uword n_data) { void *d = data; u16 *p; uword n_left = n_data; while (n_left >= 4) { p = serialize_get (m, 4 * sizeof (p[0])); clib_mem_unaligned (p + 0, u16) = clib_host_to_net_mem_u16 (d + 0 * data_stride); clib_mem_unaligned (p + 1, u16) = clib_host_to_net_mem_u16 (d + 1 * data_stride); clib_mem_unaligned (p + 2, u16) = clib_host_to_net_mem_u16 (d + 2 * data_stride); clib_mem_unaligned (p + 3, u16) = clib_host_to_net_mem_u16 (d + 3 * data_stride); n_left -= 4; d += 4 * data_stride; } if (n_left > 0) { p = serialize_get (m, n_left * sizeof (p[0])); while (n_left > 0) { clib_mem_unaligned (p + 0, u16) = clib_host_to_net_mem_u16 (d + 0 * data_stride); p += 1; d += 1 * data_stride; n_left -= 1; } } } __clib_export void serialize_multiple_4 (serialize_main_t *m, void *data, uword data_stride, uword n_data) { void *d = data; u32 *p; uword n_left = n_data; while (n_left >= 4) { p = serialize_get (m, 4 * sizeof (p[0])); clib_mem_unaligned (p + 0, u32) = clib_host_to_net_mem_u32 (d + 0 * data_stride); clib_mem_unaligned (p + 1, u32) = clib_host_to_net_mem_u32 (d + 1 * data_stride); clib_mem_unaligned (p + 2, u32) = clib_host_to_net_mem_u32 (d + 2 * data_stride); clib_mem_unaligned (p + 3, u32) = clib_host_to_net_mem_u32 (d + 3 * data_stride); n_left -= 4; d += 4 * data_stride; } if (n_left > 0) { p = serialize_get (m, n_left * sizeof (p[0])); while (n_left > 0) { clib_mem_unaligned (p + 0, u32) = clib_host_to_net_mem_u32 (d + 0 * data_stride); p += 1; d += 1 * data_stride; n_left -= 1; } } } __clib_export void unserialize_multiple_1 (serialize_main_t *m, void *data, uword data_stride, uword n_data) { u8 *d = data; u8 *p; uword n_left = n_data; while (n_left >= 4) { p = unserialize_get (m, 4 * sizeof (d[0])); d[0 * data_stride] = p[0]; d[1 * data_stride] = p[1]; d[2 * data_stride] = p[2]; d[3 * data_stride] = p[3]; n_left -= 4; d += 4 * data_stride; } if (n_left > 0) { p = unserialize_get (m, n_left * sizeof (p[0])); while (n_left > 0) { d[0] = p[0]; p += 1; d += 1 * data_stride; n_left -= 1; } } } __clib_export void unserialize_multiple_2 (serialize_main_t *m, void *data, uword data_stride, uword n_data) { void *d = data; u16 *p; uword n_left = n_data; while (n_left >= 4) { p = unserialize_get (m, 4 * sizeof (p[0])); clib_mem_unaligned (d + 0 * data_stride, u16) = clib_net_to_host_mem_u16 (p + 0); clib_mem_unaligned (d + 1 * data_stride, u16) = clib_net_to_host_mem_u16 (p + 1); clib_mem_unaligned (d + 2 * data_stride, u16) = clib_net_to_host_mem_u16 (p + 2); clib_mem_unaligned (d + 3 * data_stride, u16) = clib_net_to_host_mem_u16 (p + 3); n_left -= 4; d += 4 * data_stride; } if (n_left > 0) { p = unserialize_get (m, n_left * sizeof (p[0])); while (n_left > 0) { clib_mem_unaligned (d + 0 * data_stride, u16) = clib_net_to_host_mem_u16 (p + 0); p += 1; d += 1 * data_stride; n_left -= 1; } } } __clib_export void unserialize_multiple_4 (serialize_main_t *m, void *data, uword data_stride, uword n_data) { void *d = data; u32 *p; uword n_left = n_data; while (n_left >= 4) { p = unserialize_get (m, 4 * sizeof (p[0])); clib_mem_unaligned (d + 0 * data_stride, u32) = clib_net_to_host_mem_u32 (p + 0); clib_mem_unaligned (d + 1 * data_stride, u32) = clib_net_to_host_mem_u32 (p + 1); clib_mem_unaligned (d + 2 * data_stride, u32) = clib_net_to_host_mem_u32 (p + 2); clib_mem_unaligned (d + 3 * data_stride, u32) = clib_net_to_host_mem_u32 (p + 3); n_left -= 4; d += 4 * data_stride; } if (n_left > 0) { p = unserialize_get (m, n_left * sizeof (p[0])); while (n_left > 0) { clib_mem_unaligned (d + 0 * data_stride, u32) = clib_net_to_host_mem_u32 (p + 0); p += 1; d += 1 * data_stride; n_left -= 1; } } } #ifdef CLIB_UNIX #include <unistd.h> #include <fcntl.h> static void clib_file_write (serialize_main_header_t * m, serialize_stream_t * s) { int fd, n; fd = s->data_function_opaque; n = write (fd, s->buffer, s->current_buffer_index); if (n < 0) { if (!unix_error_is_fatal (errno)) n = 0; else serialize_error (m, clib_error_return_unix (0, "write")); } if (n == s->current_buffer_index) vec_set_len (s->buffer, 0); else vec_delete (s->buffer, n, 0); s->current_buffer_index = vec_len (s->buffer); } static void clib_file_read (serialize_main_header_t * m, serialize_stream_t * s) { int fd, n; fd = s->data_function_opaque; n = read (fd, s->buffer, vec_len (s->buffer)); if (n < 0) { if (!unix_error_is_fatal (errno)) n = 0; else serialize_error (m, clib_error_return_unix (0, "read")); } else if (n == 0) serialize_stream_set_end_of_stream (s); s->current_buffer_index = 0; s->n_buffer_bytes = n; } static void serialize_open_clib_file_descriptor_helper (serialize_main_t * m, int fd, uword is_read) { clib_memset (m, 0, sizeof (m[0])); vec_resize (m->stream.buffer, 4096); if (!is_read) { m->stream.n_buffer_bytes = vec_len (m->stream.buffer); vec_set_len (m->stream.buffer, 0); } m->header.data_function = is_read ? clib_file_read : clib_file_write; m->stream.data_function_opaque = fd; } __clib_export void serialize_open_clib_file_descriptor (serialize_main_t * m, int fd) { serialize_open_clib_file_descriptor_helper (m, fd, /* is_read */ 0); } __clib_export void unserialize_open_clib_file_descriptor (serialize_main_t * m, int fd) { serialize_open_clib_file_descriptor_helper (m, fd, /* is_read */ 1); } static clib_error_t * serialize_open_clib_file_helper (serialize_main_t * m, char *file, uword is_read) { int fd, mode; mode = is_read ? O_RDONLY : O_RDWR | O_CREAT | O_TRUNC; fd = open (file, mode, 0666); if (fd < 0) return clib_error_return_unix (0, "open `%s'", file); serialize_open_clib_file_descriptor_helper (m, fd, is_read); return 0; } __clib_export clib_error_t * serialize_open_clib_file (serialize_main_t * m, char *file) { return serialize_open_clib_file_helper (m, file, /* is_read */ 0); } __clib_export clib_error_t * unserialize_open_clib_file (serialize_main_t * m, char *file) { return serialize_open_clib_file_helper (m, file, /* is_read */ 1); } #endif /* CLIB_UNIX */ /* * fd.io coding-style-patch-verification: ON * * Local Variables: * eval: (c-set-style "gnu") * End: */