summaryrefslogtreecommitdiffstats
path: root/src/plugins/srv6-mobile/mobile.h
blob: 5a086b66c6e62e10b29246fff9485733afc40557 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
/*
 * srv6_end.h
 *
 * Copyright (c) 2019 Arrcus Inc and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef __included_srv6_end_h__
#define __included_srv6_end_h__

#include <vnet/vnet.h>
#include <vnet/ip/ip.h>
#include <vnet/srv6/sr.h>
#include <vnet/srv6/sr_packet.h>

#include <vppinfra/error.h>
#include <vppinfra/elog.h>

#define SRV6_GTP_UDP_DST_PORT 2152

#define SRV6_NHTYPE_NONE 	0
#define SRV6_NHTYPE_IPV4 	1
#define SRV6_NHTYPE_IPV6 	2
#define SRV6_NHTYPE_NON_IP	3

#ifndef IP_PROTOCOL_NONE
#define IP_PROTOCOL_NONE	59
#endif

#define SRV6_GTP6_UNKNOW	0
#define SRV6_GTP6_DT4		1
#define SRV6_GTP6_DT6		2
#define SRV6_GTP6_DT46		3

#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
#define BITALIGN2(A,B)          A; B
#define BITALIGN3(A,B,C)        A; B; C
#else
#define BITALIGN2(A,B)          B; A
#define BITALIGN3(A,B,C)        C; B; A
#endif

#define GTPU_EXTHDR_FLAG                0x04
#define GTPU_SEQ_FLAG			0x02
#define GTPU_EXTHDR_PDU_SESSION         0x85

#define SRH_TAG_ECHO_REPLY              0x0008
#define SRH_TAG_ECHO_REQUEST            0x0004
#define SRH_TAG_ERROR_INDICATION        0x0002
#define SRH_TAG_END_MARKER              0x0001

/* *INDENT-OFF* */
typedef struct
{
  u16 seq;
  u8 npdu_num;
  u8 nextexthdr;
} __attribute__ ((packed)) gtpu_exthdr_t;
/* *INDENT-ON* */

/* *INDENT-OFF* */
typedef struct
{
  u8 ver_flags;
  u8 type;
  u16 length;     /* length in octets of the payload */
  u32 teid;
  gtpu_exthdr_t ext[0];
} __attribute__ ((packed)) gtpu_header_t;
/* *INDENT-ON* */

#define GTPU_TYPE_ECHO_REQUEST          1
#define GTPU_TYPE_ECHO_REPLY            2
#define GTPU_TYPE_ERROR_INDICATION      26
#define GTPU_TYPE_END_MARKER            254
#define GTPU_TYPE_GTPU                  255

/* *INDENT-OFF* */
typedef struct
{
  BITALIGN2 (u8 ppi:3,
             u8 spare:5);

  u8 padding[3];
} __attribute__ ((packed)) gtpu_paging_policy_t;
/* *INDENT-ON* */

/* *INDENT-OFF* */
typedef struct
{
  u8 exthdrlen;
  BITALIGN2(u8 type:4,
            u8 spare:4);
  union {
    struct gtpu_qfi_bits {BITALIGN3(u8 p:1,
		                    u8 r:1,
				    u8 qfi:6);
    } bits;

    u8 val;
  } u;

  gtpu_paging_policy_t  paging[0];
  u8 nextexthdr;
} __attribute__ ((packed)) gtpu_pdu_session_t;
/* *INDENT-ON* */

#define GTPU_PDU_SESSION_P_BIT_MASK     0x80
#define GTPU_PDU_SESSION_R_BIT_MASK     0x40
#define GTPU_PDU_SESSION_QFI_MASK       0x3f

#define SRV6_PDU_SESSION_U_BIT_MASK     0x01
#define SRV6_PDU_SESSION_R_BIT_MASK     0x02
#define SRV6_PDU_SESSION_QFI_MASK       0xfC

/* *INDENT-OFF* */
typedef struct
{
  ip4_header_t ip4;            /* 20 bytes */
  udp_header_t udp;            /* 8 bytes */
  gtpu_header_t gtpu;        /* 8 bytes */
} __attribute__ ((packed)) ip4_gtpu_header_t;
/* *INDENT-ON* */

/* *INDENT-OFF* */
typedef struct
{
  ip6_header_t ip6;          /* 40 bytes */
  udp_header_t udp;          /* 8 bytes */
  gtpu_header_t gtpu;        /* 8 bytes */
} __attribute__ ((packed)) ip6_gtpu_header_t;
/* *INDENT-ON* */

#define GTPU_V1_VER   (1<<5)

#define GTPU_PT_GTP   (1<<4)

typedef struct srv6_end_gtp6_param_s
{
  u8 nhtype;

  ip6_address_t sr_prefix;
  u32 sr_prefixlen;
} srv6_end_gtp6_param_t;

typedef struct srv6_end_gtp4_param_s
{
  u8 nhtype;

  ip6_address_t sr_prefix;
  u32 sr_prefixlen;

  ip6_address_t v6src_prefix;
  u32 v6src_prefixlen;

  u32 v4src_position;
} srv6_end_gtp4_param_t;

typedef struct srv6_end_main_v4_s
{
  vlib_main_t *vlib_main;
  vnet_main_t *vnet_main;

  u32 end_m_gtp4_e_node_index;
  u32 error_node_index;

  u32 dst_p_len;		// dst prefix len
  u32 src_p_len;		// src prefix len

  ip4_gtpu_header_t cache_hdr;

} srv6_end_main_v4_t;

typedef struct srv6_t_main_v4_decap_s
{
  vlib_main_t *vlib_main;
  vnet_main_t *vnet_main;

  u32 t_m_gtp4_d_node_index;
  u32 error_node_index;

  ip6_header_t cache_hdr;
} srv6_t_main_v4_decap_t;

extern srv6_end_main_v4_t srv6_end_main_v4;
extern srv6_t_main_v4_decap_t srv6_t_main_v4_decap;
extern vlib_node_registration_t srv6_end_m_gtp4_e;

typedef struct srv6_end_main_v6_s
{
  vlib_main_t *vlib_main;
  vnet_main_t *vnet_main;

  u32 end_m_gtp6_e_node_index;
  u32 error_node_index;

  ip6_gtpu_header_t cache_hdr;
} srv6_end_main_v6_t;

extern srv6_end_main_v6_t srv6_end_main_v6;
extern vlib_node_registration_t srv6_end_m_gtp6_e;

typedef struct srv6_end_main_v6_decap_s
{
  vlib_main_t *vlib_main;
  vnet_main_t *vnet_main;

  u32 end_m_gtp6_d_node_index;
  u32 error_node_index;

  ip6_header_t cache_hdr;
} srv6_end_main_v6_decap_t;

extern srv6_end_main_v6_decap_t srv6_end_main_v6_decap;
extern vlib_node_registration_t srv6_end_m_gtp6_d;

typedef struct srv6_end_main_v6_decap_di_s
{
  vlib_main_t *vlib_main;
  vnet_main_t *vnet_main;

  u32 end_m_gtp6_d_di_node_index;
  u32 error_node_index;

  ip6srv_combo_header_t cache_hdr;
} srv6_end_main_v6_decap_di_t;

extern srv6_end_main_v6_decap_di_t srv6_end_main_v6_decap_di;
extern vlib_node_registration_t srv6_end_m_gtp6_d_di;

#endif /* __included_srv6_end_h__ */

/*
 * fd.io coding-style-patch-verification: ON
 *
 * Local Variables:
 * eval: (c-set-style "gnu")
 * End:
 */
'#n1038'>1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221
/*
 * Copyright (c) 2015 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/*
  Copyright (c) 2005,2009 Eliot Dresselhaus

  Permission is hereby granted, free of charge, to any person obtaining
  a copy of this software and associated documentation files (the
  "Software"), to deal in the Software without restriction, including
  without limitation the rights to use, copy, modify, merge, publish,
  distribute, sublicense, and/or sell copies of the Software, and to
  permit persons to whom the Software is furnished to do so, subject to
  the following conditions:

  The above copyright notice and this permission notice shall be
  included in all copies or substantial portions of the Software.

  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

#include <vppinfra/elog.h>
#include <vppinfra/cache.h>
#include <vppinfra/error.h>
#include <vppinfra/format.h>
#include <vppinfra/hash.h>
#include <vppinfra/math.h>
#include <vppinfra/lock.h>

static inline void
elog_lock (elog_main_t * em)
{
  if (PREDICT_FALSE (em->lock != 0))
    while (clib_atomic_test_and_set (em->lock))
      CLIB_PAUSE ();
}

static inline void
elog_unlock (elog_main_t * em)
{
  if (PREDICT_FALSE (em->lock != 0))
    {
      clib_atomic_release (em->lock);
    }
}

/* Non-inline version. */
__clib_export void *
elog_event_data (elog_main_t * em,
		 elog_event_type_t * type, elog_track_t * track, u64 cpu_time)
{
  return elog_event_data_inline (em, type, track, cpu_time);
}

static void
new_event_type (elog_main_t * em, uword i)
{
  elog_event_type_t *t = vec_elt_at_index (em->event_types, i);

  if (!em->event_type_by_format)
    em->event_type_by_format =
      hash_create_vec ( /* size */ 0, sizeof (u8), sizeof (uword));

  t->type_index_plus_one = i + 1;
  hash_set_mem (em->event_type_by_format, t->format, i);
}

static uword
find_or_create_type (elog_main_t * em, elog_event_type_t * t)
{
  uword *p = hash_get_mem (em->event_type_by_format, t->format);
  uword i;

  if (p)
    i = p[0];
  else
    {
      i = vec_len (em->event_types);
      vec_add1 (em->event_types, t[0]);
      new_event_type (em, i);
    }

  return i;
}

/* External function to register types. */
word
elog_event_type_register (elog_main_t * em, elog_event_type_t * t)
{
  elog_event_type_t *static_type = t;
  word l;

  elog_lock (em);

  /* Multiple simultaneous registration attempts, */
  if (t->type_index_plus_one > 0)
    {
      elog_unlock (em);
      return t->type_index_plus_one - 1;
    }

  l = vec_len (em->event_types);

  t->type_index_plus_one = 1 + l;

  ASSERT (t->format);

  /* If format args are not specified try to be smart about providing defaults
     so most of the time user does not have to specify them. */
  if (!t->format_args)
    {
      uword i, l;
      char *this_arg;

      l = strlen (t->format);
      for (i = 0; i < l; i++)
	{
	  if (t->format[i] != '%')
	    continue;
	  if (i + 1 >= l)
	    continue;
	  if (t->format[i + 1] == '%')	/* %% */
	    continue;

	  switch (t->format[i + 1])
	    {
	    default:
	    case 'd':
	    case 'x':
	    case 'u':
	      this_arg = "i4";	/* size of u32 */
	      break;
	    case 'f':
	      this_arg = "f8";	/* defaults to f64 */
	      break;
	    case 's':
	      this_arg = "s0";	/* defaults to null terminated string. */
	      break;
	    }

	  t->format_args =
	    (char *) format ((u8 *) t->format_args, "%s", this_arg);
	}

      /* Null terminate. */
      vec_add1 (t->format_args, 0);
    }

  vec_add1 (em->event_types, t[0]);

  t = em->event_types + l;

  /* Make copies of strings for hashing etc. */
  if (t->function)
    t->format = (char *) format (0, "%s %s%c", t->function, t->format, 0);
  else
    t->format = (char *) format (0, "%s%c", t->format, 0);

  t->format_args = (char *) format (0, "%s%c", t->format_args, 0);

  /* Construct string table. */
  {
    uword i;
    t->n_enum_strings = static_type->n_enum_strings;
    for (i = 0; i < t->n_enum_strings; i++)
      {
	if (!static_type->enum_strings[i])
	  static_type->enum_strings[i] = "MISSING";
	vec_add1 (t->enum_strings_vector,
		  (char *) format (0, "%s%c", static_type->enum_strings[i],
				   0));
      }
  }

  new_event_type (em, l);
  elog_unlock (em);

  return l;
}

__clib_export word
elog_track_register (elog_main_t * em, elog_track_t * t)
{
  word l;

  elog_lock (em);

  l = vec_len (em->tracks);

  t->track_index_plus_one = 1 + l;

  ASSERT (t->name);

  vec_add1 (em->tracks, t[0]);

  t = em->tracks + l;

  t->name = (char *) format (0, "%s%c", t->name, 0);

  elog_unlock (em);

  return l;
}

static uword
parse_2digit_decimal (char *p, uword * number)
{
  uword i = 0;
  u8 digits[2];

  digits[0] = digits[1] = 0;
  while (p[i] >= '0' && p[i] <= '9')
    {
      if (i >= 2)
	break;
      digits[i] = p[i] - '0';
      i++;
    }

  if (i >= 1 && i <= 2)
    {
      if (i == 1)
	*number = digits[0];
      else
	*number = 10 * digits[0] + digits[1];
      return i;
    }
  else
    return 0;
}

static u8 *
fixed_format (u8 * s, char *fmt, char *result, uword * result_len)
{
  char *f = fmt;
  char *percent;
  uword l = 0;

  while (1)
    {
      if (f[0] == 0)
	break;
      if (f[0] == '%' && f[1] != '%')
	break;
      f++;
    }
  if (f > fmt)
    vec_add (s, fmt, f - fmt);

  if (f[0] != '%')
    goto done;

  /* Skip percent. */
  percent = f++;

  /* Skip possible +-= justification. */
  f += f[0] == '+' || f[0] == '-' || f[0] == '=';

  /* Skip possible X.Y width. */
  while ((f[0] >= '0' && f[0] <= '9') || f[0] == '.')
    f++;

  /* Skip wlL as in e.g. %Ld. */
  f += f[0] == 'w' || f[0] == 'l' || f[0] == 'L';

  /* Finally skip format letter. */
  f += f[0] != 0;

  ASSERT (*result_len > f - percent);
  l = clib_min (f - percent, *result_len - 1);
  clib_memcpy (result, percent, l);
  result[l] = 0;

done:
  *result_len = f - fmt;
  return s;
}

__clib_export u8 *
format_elog_event (u8 * s, va_list * va)
{
  elog_main_t *em = va_arg (*va, elog_main_t *);
  elog_event_t *e = va_arg (*va, elog_event_t *);
  elog_event_type_t *t;
  char *a, *f;
  void *d = (u8 *) e->data;
  char arg_format[64];

  t = vec_elt_at_index (em->event_types, e->event_type);

  f = t->format;
  a = t->format_args;
  while (1)
    {
      uword n_bytes = 0, n_digits, f_bytes = 0;

      f_bytes = sizeof (arg_format);
      s = fixed_format (s, f, arg_format, &f_bytes);
      f += f_bytes;

      if (a == 0 || a[0] == 0)
	{
	  /* Format must also be at end. */
	  ASSERT (f[0] == 0);
	  break;
	}

      /* Don't go past end of event data. */
      ASSERT (d < (void *) (e->data + sizeof (e->data)));

      n_digits = parse_2digit_decimal (a + 1, &n_bytes);
      switch (a[0])
	{
	case 'i':
	case 't':
	case 'T':
	  {
	    u32 i = 0;
	    u64 l = 0;

	    if (n_bytes == 1)
	      i = ((u8 *) d)[0];
	    else if (n_bytes == 2)
	      i = clib_mem_unaligned (d, u16);
	    else if (n_bytes == 4)
	      i = clib_mem_unaligned (d, u32);
	    else if (n_bytes == 8)
	      l = clib_mem_unaligned (d, u64);
	    else
	      ASSERT (0);
	    if (a[0] == 't')
	      {
		char *e =
		  vec_elt (t->enum_strings_vector, n_bytes == 8 ? l : i);
		s = format (s, arg_format, e);
	      }
	    else if (a[0] == 'T')
	      {
		char *e =
		  vec_elt_at_index (em->string_table, n_bytes == 8 ? l : i);
		s = format (s, arg_format, e);
	      }
	    else if (n_bytes == 8)
	      s = format (s, arg_format, l);
	    else
	      s = format (s, arg_format, i);
	  }
	  break;

	case 'f':
	  {
	    f64 x = 0;
	    if (n_bytes == 4)
	      x = clib_mem_unaligned (d, f32);
	    else if (n_bytes == 8)
	      x = clib_mem_unaligned (d, f64);
	    else
	      ASSERT (0);
	    s = format (s, arg_format, x);
	  }
	  break;

	case 's':
	  s = format (s, arg_format, d);
	  if (n_bytes == 0)
	    n_bytes = strlen (d) + 1;
	  break;

	default:
	  ASSERT (0);
	  break;
	}

      ASSERT (n_digits > 0 && n_digits <= 2);
      a += 1 + n_digits;
      d += n_bytes;
    }

  return s;
}

u8 *
format_elog_track_name (u8 * s, va_list * va)
{
  elog_main_t *em = va_arg (*va, elog_main_t *);
  elog_event_t *e = va_arg (*va, elog_event_t *);
  elog_track_t *t = vec_elt_at_index (em->tracks, e->track);
  return format (s, "%s", t->name);
}

__clib_export u8 *
format_elog_track (u8 * s, va_list * args)
{
  elog_main_t *em = va_arg (*args, elog_main_t *);
  f64 dt = va_arg (*args, f64);
  int track_index = va_arg (*args, int);
  elog_event_t *e, *es;
  u8 indent;

  indent = format_get_indent (s) + 1;

  es = elog_peek_events (em);
  vec_foreach (e, es)
  {
    if (e->track != track_index)
      continue;
    s = format (s, "%U%18.9f: %U\n", format_white_space, indent,
		e->time + dt, format_elog_event, em, e);
  }
  vec_free (es);
  return s;
}

__clib_export char *
format_one_elog_event (void *em_arg, void *ep_arg)
{
  elog_main_t *em = (elog_main_t *) em_arg;
  elog_event_t *ep = (elog_event_t *) ep_arg;

  return (char *) format (0, "%U", format_elog_event, em, ep);
}

void
elog_time_now (elog_time_stamp_t * et)
{
  u64 cpu_time_now, os_time_now_nsec;
  struct timespec ts;

#ifdef CLIB_UNIX
  {
#include <sys/syscall.h>
#ifdef __APPLE__
    clock_gettime (CLOCK_REALTIME, &ts);
#else
    syscall (SYS_clock_gettime, CLOCK_REALTIME, &ts);
#endif
    cpu_time_now = clib_cpu_time_now ();
    /* Subtract 3/30/2017's worth of seconds to retain precision */
    os_time_now_nsec = 1e9 * (ts.tv_sec - 1490885108) + ts.tv_nsec;
  }
#else
  cpu_time_now = clib_cpu_time_now ();
  os_time_now_nsec = 0;
#endif

  et->cpu = cpu_time_now;
  et->os_nsec = os_time_now_nsec;
}

always_inline i64
elog_time_stamp_diff_os_nsec (elog_time_stamp_t * t1, elog_time_stamp_t * t2)
{
  return (i64) t1->os_nsec - (i64) t2->os_nsec;
}

always_inline i64
elog_time_stamp_diff_cpu (elog_time_stamp_t * t1, elog_time_stamp_t * t2)
{
  return (i64) t1->cpu - (i64) t2->cpu;
}

always_inline f64
elog_nsec_per_clock (elog_main_t * em)
{
  return ((f64) elog_time_stamp_diff_os_nsec (&em->serialize_time,
					      &em->init_time)
	  / (f64) elog_time_stamp_diff_cpu (&em->serialize_time,
					    &em->init_time));
}

static void
elog_alloc_internal (elog_main_t * em, u32 n_events, int free_ring)
{
  if (free_ring && em->event_ring)
    vec_free (em->event_ring);

  /* Ring size must be a power of 2. */
  em->event_ring_size = n_events = max_pow2 (n_events);

  vec_validate_aligned (em->event_ring, n_events, CLIB_CACHE_LINE_BYTES);
  _vec_len (em->event_ring) = n_events;
}

__clib_export void
elog_alloc (elog_main_t * em, u32 n_events)
{
  elog_alloc_internal (em, n_events, 1 /* free ring */ );
}

__clib_export void
elog_resize (elog_main_t * em, u32 n_events)
{
  elog_alloc_internal (em, n_events, 0 /* do not free ring */ );
}

__clib_export void
elog_init (elog_main_t * em, u32 n_events)
{
  clib_memset (em, 0, sizeof (em[0]));

  em->lock = 0;

  if (n_events > 0)
    elog_alloc (em, n_events);

  clib_time_init (&em->cpu_timer);

  em->n_total_events_disable_limit = ~0;

  /* Make track 0. */
  em->default_track.name = "default";
  elog_track_register (em, &em->default_track);

  elog_time_now (&em->init_time);
  em->string_table_hash = hash_create_string (0, sizeof (uword));
}

/* Returns number of events in ring and start index. */
static uword
elog_event_range (elog_main_t * em, uword * lo)
{
  uword l = em->event_ring_size;
  u64 i = em->n_total_events;

  /* Ring never wrapped? */
  if (i <= (u64) l)
    {
      if (lo)
	*lo = 0;
      return i;
    }
  else
    {
      if (lo)
	*lo = i & (l - 1);
      return l;
    }
}

__clib_export elog_event_t *
elog_peek_events (elog_main_t * em)
{
  elog_event_t *e, *f, *es = 0;
  uword i, j, n;

  n = elog_event_range (em, &j);
  for (i = 0; i < n; i++)
    {
      vec_add2 (es, e, 1);
      f = vec_elt_at_index (em->event_ring, j);
      e[0] = f[0];

      /* Convert absolute time from cycles to seconds from start. */
      e->time =
	(e->time_cycles -
	 em->init_time.cpu) * em->cpu_timer.seconds_per_clock;

      j = (j + 1) & (em->event_ring_size - 1);
    }

  return es;
}

/* Add a formatted string to the string table. */
__clib_export u32
elog_string (elog_main_t * em, char *fmt, ...)
{
  u32 offset;
  uword *p;
  uword len;
  va_list va;

  elog_lock (em);
  vec_reset_length (em->string_table_tmp);
  va_start (va, fmt);
  em->string_table_tmp = va_format (em->string_table_tmp, fmt, &va);
  va_end (va);

  /* String table entries MUST be NULL terminated */
  len = vec_len (em->string_table_tmp);
  ASSERT (len > 0);
  if (em->string_table_tmp[len - 1] != 0)
    vec_add1 (em->string_table_tmp, 0);

  /* See if we already have this string in the string table */
  p = hash_get_mem (em->string_table_hash, em->string_table_tmp);

  /* We already have the string, so give the caller its offset */
  if (p)
    {
      elog_unlock (em);
      return (p[0]);
    }

  /* We don't, so add it. */

  offset = vec_len (em->string_table);
  vec_append (em->string_table, em->string_table_tmp);

  hash_set_mem (em->string_table_hash, em->string_table_tmp, offset);

  /* We gave the key to the string table hash, so we can't reuse it! */
  em->string_table_tmp = 0;
  elog_unlock (em);

  return offset;
}

__clib_export elog_event_t *
elog_get_events (elog_main_t * em)
{
  vec_free (em->events);
  em->events = elog_peek_events (em);
  return em->events;
}

static void
maybe_fix_string_table_offset (elog_event_t * e,
			       elog_event_type_t * t, u32 offset)
{
  void *d = (u8 *) e->data;
  char *a;

  if (offset == 0)
    return;

  a = t->format_args;

  while (1)
    {
      uword n_bytes = 0, n_digits;

      if (a[0] == 0)
	break;

      /* Don't go past end of event data. */
      ASSERT (d < (void *) (e->data + sizeof (e->data)));

      n_digits = parse_2digit_decimal (a + 1, &n_bytes);
      switch (a[0])
	{
	case 'T':
	  ASSERT (n_bytes == 4);
	  clib_mem_unaligned (d, u32) += offset;
	  break;

	case 'i':
	case 't':
	case 'f':
	case 's':
	  break;

	default:
	  ASSERT (0);
	  break;
	}

      ASSERT (n_digits > 0 && n_digits <= 2);
      a += 1 + n_digits;
      d += n_bytes;
    }
}

static int
elog_cmp (void *a1, void *a2)
{
  elog_event_t *e1 = a1;
  elog_event_t *e2 = a2;

  if (e1->time < e2->time)
    return -1;

  if (e1->time > e2->time)
    return 1;

  return 0;
}

/*
 * merge two event logs. Complicated and cranky.
 */
void
elog_merge (elog_main_t * dst, u8 * dst_tag, elog_main_t * src, u8 * src_tag,
	    f64 align_tweak)
{
  elog_event_t *e;
  uword l;
  u32 string_table_offset_for_src_events;
  u32 track_offset_for_src_tracks;
  elog_track_t newt;
  int i;

  clib_memset (&newt, 0, sizeof (newt));

  /* Acquire src and dst events */
  elog_get_events (src);
  elog_get_events (dst);

  string_table_offset_for_src_events = vec_len (dst->string_table);
  vec_append (dst->string_table, src->string_table);

  l = vec_len (dst->events);
  vec_append (dst->events, src->events);

  /* Prepend the supplied tag (if any) to all dst track names */
  if (dst_tag)
    {
      for (i = 0; i < vec_len (dst->tracks); i++)
	{
	  elog_track_t *t = vec_elt_at_index (dst->tracks, i);
	  char *new_name;

	  new_name = (char *) format (0, "%s:%s%c", dst_tag, t->name, 0);
	  vec_free (t->name);
	  t->name = new_name;
	}
    }

  /*
   * Remember where we started allocating new tracks while merging
   */
  track_offset_for_src_tracks = vec_len (dst->tracks);

  /* Copy / tag source tracks */
  for (i = 0; i < vec_len (src->tracks); i++)
    {
      elog_track_t *t = vec_elt_at_index (src->tracks, i);
      if (src_tag)
	newt.name = (char *) format (0, "%s:%s%c", src_tag, t->name, 0);
      else
	newt.name = (char *) format (0, "%s%c", t->name, 0);
      (void) elog_track_register (dst, &newt);
      vec_free (newt.name);
    }

  /* Across all (copied) src events... */
  for (e = dst->events + l; e < vec_end (dst->events); e++)
    {
      elog_event_type_t *t =
	vec_elt_at_index (src->event_types, e->event_type);

      /* Remap type from src -> dst. */
      e->event_type = find_or_create_type (dst, t);

      /* Remap string table offsets for 'T' format args */
      maybe_fix_string_table_offset (e, t,
				     string_table_offset_for_src_events);

      /* Remap track */
      e->track += track_offset_for_src_tracks;
    }

  /* Adjust event times for relative starting times of event streams. */
  {
    f64 dt_event, dt_os_nsec, dt_clock_nsec;

    /* Set clock parameters if dst was not generated by unserialize. */
    if (dst->serialize_time.cpu == 0)
      {
	dst->init_time = src->init_time;
	dst->serialize_time = src->serialize_time;
	dst->nsec_per_cpu_clock = src->nsec_per_cpu_clock;
      }

    dt_os_nsec =
      elog_time_stamp_diff_os_nsec (&src->init_time, &dst->init_time);

    dt_event = dt_os_nsec;
    dt_clock_nsec =
      (elog_time_stamp_diff_cpu (&src->init_time, &dst->init_time) * .5 *
       (dst->nsec_per_cpu_clock + src->nsec_per_cpu_clock));

    /*
     * Heuristic to see if src/dst came from same time source.
     * If frequencies are "the same" and os clock and cpu clock agree
     * to within 100e-9 secs about time difference between src/dst
     * init_time, then we use cpu clock.  Otherwise we use OS clock.
     *
     * When merging event logs from different systems, time paradoxes
     * at the O(1ms) level are to be expected. Hence, the "align_tweak"
     * parameter. If two events logged on different processors are known
     * to occur in a specific order - and with a reasonably-estimated
     * interval - supply a non-zero "align_tweak" parameter
     */
    if (fabs (src->nsec_per_cpu_clock - dst->nsec_per_cpu_clock) < 1e-2
	&& fabs (dt_os_nsec - dt_clock_nsec) < 100)
      dt_event = dt_clock_nsec;

    /* Convert to seconds. */
    dt_event *= 1e-9;

    /*
     * Move the earlier set of events later, to avoid creating
     * events which precede the Big Bang (aka have negative timestamps).
     *
     * Not to any scale, we have something like the following picture:
     *
     * DST capture start point
     *       ^
     *       +--- dt_event --+
     *                       v
     *                 SRC capture start point
     *
     * In this case dt_event is positive, src started after dst,
     * to put src events onto a common timebase we have to move them
     * forward in time. Naturally, the opposite case is
     * possible, too: dt_event will be negative, and so we have to
     * move dst events forward in time by the |dt_event|.
     * In both cases, we add align_tweak.
     */
    if (dt_event > 0)
      {
	/* Src started after dst. */
	for (e = dst->events + l; e < vec_end (dst->events); e++)
	  e->time += dt_event + align_tweak;
      }
    else
      {
	/* Dst started after src. */
	dt_event = -dt_event;
	for (e = dst->events + 0; e < dst->events + l; e++)
	  e->time += dt_event + align_tweak;
      }
  }

  /* Sort events by increasing time. */
  vec_sort_with_function (dst->events, elog_cmp);

  dst->n_total_events = vec_len (dst->events);

  /* Recreate the event ring or the results won't serialize */
  {
    int i;

    ASSERT (dst->cpu_timer.seconds_per_clock);

    elog_alloc (dst, vec_len (dst->events));
    for (i = 0; i < vec_len (dst->events); i++)
      {
	elog_event_t *es, *ed;

	es = dst->events + i;
	ed = dst->event_ring + i;

	ed[0] = es[0];
      }
  }
}

static void
serialize_elog_event (serialize_main_t * m, va_list * va)
{
  elog_main_t *em = va_arg (*va, elog_main_t *);
  elog_event_t *e = va_arg (*va, elog_event_t *);
  elog_event_type_t *t = vec_elt_at_index (em->event_types, e->event_type);
  u8 *d = e->data;
  u8 *p = (u8 *) t->format_args;

  serialize_integer (m, e->event_type, sizeof (e->event_type));
  serialize_integer (m, e->track, sizeof (e->track));
  serialize (m, serialize_f64, e->time);

  while (*p)
    {
      uword n_digits, n_bytes = 0;

      n_digits = parse_2digit_decimal ((char *) p + 1, &n_bytes);

      switch (p[0])
	{
	case 'i':
	case 't':
	case 'T':
	  if (n_bytes == 1)
	    serialize_integer (m, d[0], sizeof (u8));
	  else if (n_bytes == 2)
	    serialize_integer (m, clib_mem_unaligned (d, u16), sizeof (u16));
	  else if (n_bytes == 4)
	    serialize_integer (m, clib_mem_unaligned (d, u32), sizeof (u32));
	  else if (n_bytes == 8)
	    serialize (m, serialize_64, clib_mem_unaligned (d, u64));
	  else
	    ASSERT (0);
	  break;

	case 's':
	  serialize_cstring (m, (char *) d);
	  if (n_bytes == 0)
	    n_bytes = strlen ((char *) d) + 1;
	  break;

	case 'f':
	  if (n_bytes == 4)
	    serialize (m, serialize_f32, clib_mem_unaligned (d, f32));
	  else if (n_bytes == 8)
	    serialize (m, serialize_f64, clib_mem_unaligned (d, f64));
	  else
	    ASSERT (0);
	  break;

	default:
	  ASSERT (0);
	  break;
	}

      p += 1 + n_digits;
      d += n_bytes;
    }
}

static void
unserialize_elog_event (serialize_main_t * m, va_list * va)
{
  elog_main_t *em = va_arg (*va, elog_main_t *);
  elog_event_t *e = va_arg (*va, elog_event_t *);
  elog_event_type_t *t;
  u8 *p, *d;

  {
    u16 tmp[2];

    unserialize_integer (m, &tmp[0], sizeof (e->event_type));
    unserialize_integer (m, &tmp[1], sizeof (e->track));

    e->event_type = tmp[0];
    e->track = tmp[1];

    /* Make sure it fits. */
    ASSERT (e->event_type == tmp[0]);
    ASSERT (e->track == tmp[1]);
  }

  t = vec_elt_at_index (em->event_types, e->event_type);

  unserialize (m, unserialize_f64, &e->time);

  d = e->data;
  p = (u8 *) t->format_args;

  while (p && *p)
    {
      uword n_digits, n_bytes = 0;
      u32 tmp;

      n_digits = parse_2digit_decimal ((char *) p + 1, &n_bytes);

      switch (p[0])
	{
	case 'i':
	case 't':
	case 'T':
	  if (n_bytes == 1)
	    {
	      unserialize_integer (m, &tmp, sizeof (u8));
	      d[0] = tmp;
	    }
	  else if (n_bytes == 2)
	    {
	      unserialize_integer (m, &tmp, sizeof (u16));
	      clib_mem_unaligned (d, u16) = tmp;
	    }
	  else if (n_bytes == 4)
	    {
	      unserialize_integer (m, &tmp, sizeof (u32));
	      clib_mem_unaligned (d, u32) = tmp;
	    }
	  else if (n_bytes == 8)
	    {
	      u64 x;
	      unserialize (m, unserialize_64, &x);
	      clib_mem_unaligned (d, u64) = x;
	    }
	  else
	    ASSERT (0);
	  break;

	case 's':
	  {
	    char *t;
	    unserialize_cstring (m, &t);
	    if (n_bytes == 0)
	      n_bytes = strlen (t) + 1;
	    clib_memcpy (d, t, clib_min (n_bytes, vec_len (t)));
	    vec_free (t);
	    break;
	  }

	case 'f':
	  if (n_bytes == 4)
	    {
	      f32 x;
	      unserialize (m, unserialize_f32, &x);
	      clib_mem_unaligned (d, f32) = x;
	    }
	  else if (n_bytes == 8)
	    {
	      f64 x;
	      unserialize (m, unserialize_f64, &x);
	      clib_mem_unaligned (d, f64) = x;
	    }
	  else
	    ASSERT (0);
	  break;

	default:
	  ASSERT (0);
	  break;
	}

      p += 1 + n_digits;
      d += n_bytes;
    }
}

static void
serialize_elog_event_type (serialize_main_t * m, va_list * va)
{
  elog_event_type_t *t = va_arg (*va, elog_event_type_t *);
  int n = va_arg (*va, int);
  int i, j;
  for (i = 0; i < n; i++)
    {
      serialize_cstring (m, t[i].format);
      serialize_cstring (m, t[i].format_args);
      serialize_integer (m, t[i].type_index_plus_one,
			 sizeof (t->type_index_plus_one));
      serialize_integer (m, t[i].n_enum_strings,
			 sizeof (t[i].n_enum_strings));
      for (j = 0; j < t[i].n_enum_strings; j++)
	serialize_cstring (m, t[i].enum_strings_vector[j]);
    }
}

static void
unserialize_elog_event_type (serialize_main_t * m, va_list * va)
{
  elog_event_type_t *t = va_arg (*va, elog_event_type_t *);
  int n = va_arg (*va, int);
  int i, j;
  for (i = 0; i < n; i++)
    {
      unserialize_cstring (m, &t[i].format);
      unserialize_cstring (m, &t[i].format_args);
      unserialize_integer (m, &t[i].type_index_plus_one,
			   sizeof (t->type_index_plus_one));
      unserialize_integer (m, &t[i].n_enum_strings,
			   sizeof (t[i].n_enum_strings));
      vec_resize (t[i].enum_strings_vector, t[i].n_enum_strings);
      for (j = 0; j < t[i].n_enum_strings; j++)
	unserialize_cstring (m, &t[i].enum_strings_vector[j]);
    }
}

static void
serialize_elog_track (serialize_main_t * m, va_list * va)
{
  elog_track_t *t = va_arg (*va, elog_track_t *);
  int n = va_arg (*va, int);
  int i;
  for (i = 0; i < n; i++)
    {
      serialize_cstring (m, t[i].name);
    }
}

static void
unserialize_elog_track (serialize_main_t * m, va_list * va)
{
  elog_track_t *t = va_arg (*va, elog_track_t *);
  int n = va_arg (*va, int);
  int i;
  for (i = 0; i < n; i++)
    {
      unserialize_cstring (m, &t[i].name);
    }
}

static void
serialize_elog_time_stamp (serialize_main_t * m, va_list * va)
{
  elog_time_stamp_t *st = va_arg (*va, elog_time_stamp_t *);
  serialize (m, serialize_64, st->os_nsec);
  serialize (m, serialize_64, st->cpu);
}

static void
unserialize_elog_time_stamp (serialize_main_t * m, va_list * va)
{
  elog_time_stamp_t *st = va_arg (*va, elog_time_stamp_t *);
  unserialize (m, unserialize_64, &st->os_nsec);
  unserialize (m, unserialize_64, &st->cpu);
}

static char *elog_serialize_magic = "elog v0";

__clib_export void
serialize_elog_main (serialize_main_t * m, va_list * va)
{
  elog_main_t *em = va_arg (*va, elog_main_t *);
  int flush_ring = va_arg (*va, int);
  elog_event_t *e;

  serialize_magic (m, elog_serialize_magic, strlen (elog_serialize_magic));

  serialize_integer (m, em->event_ring_size, sizeof (u32));

  elog_time_now (&em->serialize_time);
  serialize (m, serialize_elog_time_stamp, &em->serialize_time);
  serialize (m, serialize_elog_time_stamp, &em->init_time);

  vec_serialize (m, em->event_types, serialize_elog_event_type);
  vec_serialize (m, em->tracks, serialize_elog_track);
  vec_serialize (m, em->string_table, serialize_vec_8);

  /* Free old events (cached) in case they have changed. */
  if (flush_ring)
    {
      vec_free (em->events);
      elog_get_events (em);
    }

  serialize_integer (m, vec_len (em->events), sizeof (u32));

  /* SMP logs can easily have local time paradoxes... */
  vec_sort_with_function (em->events, elog_cmp);

  vec_foreach (e, em->events) serialize (m, serialize_elog_event, em, e);
}

void
unserialize_elog_main (serialize_main_t * m, va_list * va)
{
  elog_main_t *em = va_arg (*va, elog_main_t *);
  uword i;
  u32 rs;

  unserialize_check_magic (m, elog_serialize_magic,
			   strlen (elog_serialize_magic));

  unserialize_integer (m, &rs, sizeof (u32));
  em->event_ring_size = rs;
  elog_init (em, em->event_ring_size);

  unserialize (m, unserialize_elog_time_stamp, &em->serialize_time);
  unserialize (m, unserialize_elog_time_stamp, &em->init_time);
  em->nsec_per_cpu_clock = elog_nsec_per_clock (em);

  vec_unserialize (m, &em->event_types, unserialize_elog_event_type);
  for (i = 0; i < vec_len (em->event_types); i++)
    new_event_type (em, i);

  vec_unserialize (m, &em->tracks, unserialize_elog_track);
  vec_unserialize (m, &em->string_table, unserialize_vec_8);

  {
    u32 ne;
    elog_event_t *e;

    unserialize_integer (m, &ne, sizeof (u32));
    vec_resize (em->events, ne);
    vec_foreach (e, em->events)
      unserialize (m, unserialize_elog_event, em, e);
  }
}

#ifdef CLIB_UNIX
clib_error_t *
elog_write_file_not_inline (elog_main_t * em, char *clib_file, int flush_ring)
{
  serialize_main_t m;
  clib_error_t *error;

  error = serialize_open_clib_file (&m, clib_file);
  if (error)
    return error;
  error = serialize (&m, serialize_elog_main, em, flush_ring);
  if (!error)
    serialize_close (&m);
  return error;
}

__clib_export clib_error_t *
elog_read_file_not_inline (elog_main_t * em, char *clib_file)
{
  serialize_main_t m;
  clib_error_t *error;

  error = unserialize_open_clib_file (&m, clib_file);
  if (error)
    return error;
  error = unserialize (&m, unserialize_elog_main, em);
  if (!error)
    unserialize_close (&m);
  return error;
}
#endif /* CLIB_UNIX */


/*
 * fd.io coding-style-patch-verification: ON
 *
 * Local Variables:
 * eval: (c-set-style "gnu")
 * End:
 */