aboutsummaryrefslogtreecommitdiffstats
path: root/test/test_pppoe.py
blob: d69058f1aeb84cf57ec6046300279117935262d7 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
#!/usr/bin/env python

import unittest
from logging import *

from framework import VppTestCase, VppTestRunner
from vpp_ip_route import VppIpRoute, VppRoutePath
from vpp_pppoe_interface import VppPppoeInterface
from vpp_papi_provider import L2_VTR_OP

from scapy.packet import Raw
from scapy.layers.l2 import Ether
from scapy.layers.ppp import PPPoE, PPPoED, PPP
from scapy.layers.inet import IP, UDP
from scapy.layers.inet6 import IPv6
from scapy.volatile import RandMAC, RandIP

from util import ppp, ppc, mactobinary
import socket


class TestPPPoE(VppTestCase):
    """ PPPoE Test Case """

    @classmethod
    def setUpClass(cls):
        super(TestPPPoE, cls).setUpClass()

        cls.session_id = 1
        cls.dst_ip = "100.1.1.100"
        cls.dst_ipn = socket.inet_pton(socket.AF_INET, cls.dst_ip)

    def setUp(self):
        super(TestPPPoE, self).setUp()

        # create 2 pg interfaces
        self.create_pg_interfaces(range(3))

        for i in self.pg_interfaces:
            i.admin_up()
            i.config_ip4()
            i.resolve_arp()

    def tearDown(self):
        super(TestPPPoE, self).tearDown()

        self.logger.info(self.vapi.cli("show int"))
        self.logger.info(self.vapi.cli("show pppoe fib"))
        self.logger.info(self.vapi.cli("show pppoe session"))
        self.logger.info(self.vapi.cli("show ip fib"))
        self.logger.info(self.vapi.cli("show trace"))

        for i in self.pg_interfaces:
            i.unconfig_ip4()
            i.admin_down()

    def create_stream_pppoe_discovery(self, src_if, dst_if,
                                      client_mac, count=1):
        packets = []
        for i in range(count):
            # create packet info stored in the test case instance
            info = self.create_packet_info(src_if, dst_if)
            # convert the info into packet payload
            payload = self.info_to_payload(info)
            # create the packet itself
            p = (Ether(dst=src_if.local_mac, src=client_mac) /
                 PPPoED(sessionid=0) /
                 Raw(payload))
            # store a copy of the packet in the packet info
            info.data = p.copy()
            # append the packet to the list
            packets.append(p)

        # return the created packet list
        return packets

    def create_stream_pppoe_lcp(self, src_if, dst_if,
                                client_mac, session_id, count=1):
        packets = []
        for i in range(count):
            # create packet info stored in the test case instance
            info = self.create_packet_info(src_if, dst_if)
            # convert the info into packet payload
            payload = self.info_to_payload(info)
            # create the packet itself
            p = (Ether(dst=src_if.local_mac, src=client_mac) /
                 PPPoE(sessionid=session_id) /
                 PPP(proto=0xc021) /
                 Raw(payload))
            # store a copy of the packet in the packet info
            info.data = p.copy()
            # append the packet to the list
            packets.append(p)

        # return the created packet list
        return packets

    def create_stream_pppoe_ip4(self, src_if, dst_if,
                                client_mac, session_id, client_ip, count=1):
        packets = []
        for i in range(count):
            # create packet info stored in the test case instance
            info = self.create_packet_info(src_if, dst_if)
            # convert the info into packet payload
            payload = self.info_to_payload(info)
            # create the packet itself
            p = (Ether(dst=src_if.local_mac, src=client_mac) /
                 PPPoE(sessionid=session_id) /
                 PPP(proto=0x0021) /
                 IP(src=client_ip, dst=self.dst_ip) /
                 Raw(payload))
            # store a copy of the packet in the packet info
            info.data = p.copy()
            # append the packet to the list
            packets.append(p)

        # return the created packet list
        return packets

    def create_stream_ip4(self, src_if, dst_if, client_ip, dst_ip, count=1):
        pkts = []
        for i in range(count):
            # create packet info stored in the test case instance
            info = self.create_packet_info(src_if, dst_if)
            # convert the info into packet payload
            payload = self.info_to_payload(info)
            # create the packet itself
            p = (Ether(dst=src_if.local_mac, src=src_if.remote_mac) /
                 IP(src=dst_ip, dst=client_ip) /
                 Raw(payload))
            # store a copy of the packet in the packet info
            info.data = p.copy()
            # append the packet to the list
            pkts.append(p)

        # return the created packet list
        return pkts

    def verify_decapped_pppoe(self, src_if, capture, sent):
        self.assertEqual(len(capture), len(sent))

        for i in range(len(capture)):
            try:
                tx = sent[i]
                rx = capture[i]

                tx_ip = tx[IP]
                rx_ip = rx[IP]

                self.assertEqual(rx_ip.src, tx_ip.src)
                self.assertEqual(rx_ip.dst, tx_ip.dst)

            except:
                self.logger.error(ppp("Rx:", rx))
                self.logger.error(ppp("Tx:", tx))
                raise

    def verify_encaped_pppoe(self, src_if, capture, sent, session_id):

        self.assertEqual(len(capture), len(sent))

        for i in range(len(capture)):
            try:
                tx = sent[i]
                rx = capture[i]

                tx_ip = tx[IP]
                rx_ip = rx[IP]

                self.assertEqual(rx_ip.src, tx_ip.src)
                self.assertEqual(rx_ip.dst, tx_ip.dst)

                rx_pppoe = rx[PPPoE]

                self.assertEqual(rx_pppoe.sessionid, session_id)

            except:
                self.logger.error(ppp("Rx:", rx))
                self.logger.error(ppp("Tx:", tx))
                raise

    def test_PPPoE_Decap(self):
        """ PPPoE Decap Test """

        self.vapi.cli("clear trace")

        #
        # Add a route that resolves the server's destination
        #
        route_sever_dst = VppIpRoute(self, "100.1.1.100", 32,
                                     [VppRoutePath(self.pg1.remote_ip4,
                                                   self.pg1.sw_if_index)])
        route_sever_dst.add_vpp_config()

        # Send PPPoE Discovery
        tx0 = self.create_stream_pppoe_discovery(self.pg0, self.pg1,
                                                 self.pg0.remote_mac)
        self.pg0.add_stream(tx0)
        self.pg_start()

        # Send PPPoE PPP LCP
        tx1 = self.create_stream_pppoe_lcp(self.pg0, self.pg1,
                                           self.pg0.remote_mac,
                                           self.session_id)
        self.pg0.add_stream(tx1)
        self.pg_start()

        # Create PPPoE session
        pppoe_if = VppPppoeInterface(self,
                                     self.pg0.remote_ip4,
                                     self.pg0.remote_mac,
                                     self.session_id)
        pppoe_if.add_vpp_config()

        #
        # Send tunneled packets that match the created tunnel and
        # are decapped and forwarded
        #
        tx2 = self.create_stream_pppoe_ip4(self.pg0, self.pg1,
                                           self.pg0.remote_mac,
                                           self.session_id,
                                           self.pg0.remote_ip4)
        self.pg0.add_stream(tx2)

        self.pg_enable_capture(self.pg_interfaces)
        self.pg_start()

        rx2 = self.pg1.get_capture(len(tx2))
        self.verify_decapped_pppoe(self.pg0, rx2, tx2)

        self.logger.info(self.vapi.cli("show pppoe fib"))
        self.logger.info(self.vapi.cli("show pppoe session"))
        self.logger.info(self.vapi.cli("show ip fib"))

        #
        # test case cleanup
        #

        # Delete PPPoE session
        pppoe_if.remove_vpp_config()

        # Delete a route that resolves the server's destination
        route_sever_dst.remove_vpp_config()

    def test_PPPoE_Encap(self):
        """ PPPoE Encap Test """

        self.vapi.cli("clear trace")

        #
        # Add a route that resolves the server's destination
        #
        route_sever_dst = VppIpRoute(self, "100.1.1.100", 32,
                                     [VppRoutePath(self.pg1.remote_ip4,
                                                   self.pg1.sw_if_index)])
        route_sever_dst.add_vpp_config()

        # Send PPPoE Discovery
        tx0 = self.create_stream_pppoe_discovery(self.pg0, self.pg1,
                                                 self.pg0.remote_mac)
        self.pg0.add_stream(tx0)
        self.pg_start()

        # Send PPPoE PPP LCP
        tx1 = self.create_stream_pppoe_lcp(self.pg0, self.pg1,
                                           self.pg0.remote_mac,
                                           self.session_id)
        self.pg0.add_stream(tx1)
        self.pg_start()

        # Create PPPoE session
        pppoe_if = VppPppoeInterface(self,
                                     self.pg0.remote_ip4,
                                     self.pg0.remote_mac,
                                     self.session_id)
        pppoe_if.add_vpp_config()

        #
        # Send a packet stream that is routed into the session
        #  - packets are PPPoE encapped
        #
        self.vapi.cli("clear trace")
        tx2 = self.create_stream_ip4(self.pg1, self.pg0,
                                     self.pg0.remote_ip4, self.dst_ip, 65)
        self.pg1.add_stream(tx2)

        self.pg_enable_capture(self.pg_interfaces)
        self.pg_start()

        rx2 = self.pg0.get_capture(len(tx2))
        self.verify_encaped_pppoe(self.pg1, rx2, tx2, self.session_id)

        self.logger.info(self.vapi.cli("show pppoe fib"))
        self.logger.info(self.vapi.cli("show pppoe session"))
        self.logger.info(self.vapi.cli("show ip fib"))
        self.logger.info(self.vapi.cli("show adj"))

        #
        # test case cleanup
        #

        # Delete PPPoE session
        pppoe_if.remove_vpp_config()

        # Delete a route that resolves the server's destination
        route_sever_dst.remove_vpp_config()

    def test_PPPoE_Add_Twice(self):
        """ PPPoE Add Same Session Twice Test """

        self.vapi.cli("clear trace")

        #
        # Add a route that resolves the server's destination
        #
        route_sever_dst = VppIpRoute(self, "100.1.1.100", 32,
                                     [VppRoutePath(self.pg1.remote_ip4,
                                                   self.pg1.sw_if_index)])
        route_sever_dst.add_vpp_config()

        # Send PPPoE Discovery
        tx0 = self.create_stream_pppoe_discovery(self.pg0, self.pg1,
                                                 self.pg0.remote_mac)
        self.pg0.add_stream(tx0)
        self.pg_start()

        # Send PPPoE PPP LCP
        tx1 = self.create_stream_pppoe_lcp(self.pg0, self.pg1,
                                           self.pg0.remote_mac,
                                           self.session_id)
        self.pg0.add_stream(tx1)
        self.pg_start()

        # Create PPPoE session
        pppoe_if = VppPppoeInterface(self,
                                     self.pg0.remote_ip4,
                                     self.pg0.remote_mac,
                                     self.session_id)
        pppoe_if.add_vpp_config()

        #
        # The double create (create the same session twice) should fail,
        # and we should still be able to use the original
        #
        try:
            pppoe_if.add_vpp_config()
        except Exception:
            pass
        else:
            self.fail("Double GRE tunnel add does not fail")

        #
        # test case cleanup
        #

        # Delete PPPoE session
        pppoe_if.remove_vpp_config()

        # Delete a route that resolves the server's destination
        route_sever_dst.remove_vpp_config()

    def test_PPPoE_Del_Twice(self):
        """ PPPoE Delete Same Session Twice Test """

        self.vapi.cli("clear trace")

        #
        # Add a route that resolves the server's destination
        #
        route_sever_dst = VppIpRoute(self, "100.1.1.100", 32,
                                     [VppRoutePath(self.pg1.remote_ip4,
                                                   self.pg1.sw_if_index)])
        route_sever_dst.add_vpp_config()

        # Send PPPoE Discovery
        tx0 = self.create_stream_pppoe_discovery(self.pg0, self.pg1,
                                                 self.pg0.remote_mac)
        self.pg0.add_stream(tx0)
        self.pg_start()

        # Send PPPoE PPP LCP
        tx1 = self.create_stream_pppoe_lcp(self.pg0, self.pg1,
                                           self.pg0.remote_mac,
                                           self.session_id)
        self.pg0.add_stream(tx1)
        self.pg_start()

        # Create PPPoE session
        pppoe_if = VppPppoeInterface(self,
                                     self.pg0.remote_ip4,
                                     self.pg0.remote_mac,
                                     self.session_id)
        pppoe_if.add_vpp_config()

        # Delete PPPoE session
        pppoe_if.remove_vpp_config()

        #
        # The double del (del the same session twice) should fail,
        # and we should still be able to use the original
        #
        try:
            pppoe_if.remove_vpp_config()
        except Exception:
            pass
        else:
            self.fail("Double GRE tunnel del does not fail")

        #
        # test case cleanup
        #

        # Delete a route that resolves the server's destination
        route_sever_dst.remove_vpp_config()

    def test_PPPoE_Decap_Multiple(self):
        """ PPPoE Decap Multiple Sessions Test """

        self.vapi.cli("clear trace")

        #
        # Add a route that resolves the server's destination
        #
        route_sever_dst = VppIpRoute(self, "100.1.1.100", 32,
                                     [VppRoutePath(self.pg1.remote_ip4,
                                                   self.pg1.sw_if_index)])
        route_sever_dst.add_vpp_config()

        # Send PPPoE Discovery 1
        tx0 = self.create_stream_pppoe_discovery(self.pg0, self.pg1,
                                                 self.pg0.remote_mac)
        self.pg0.add_stream(tx0)
        self.pg_start()

        # Send PPPoE PPP LCP 1
        tx1 = self.create_stream_pppoe_lcp(self.pg0, self.pg1,
                                           self.pg0.remote_mac,
                                           self.session_id)
        self.pg0.add_stream(tx1)
        self.pg_start()

        # Create PPPoE session 1
        pppoe_if1 = VppPppoeInterface(self,
                                      self.pg0.remote_ip4,
                                      self.pg0.remote_mac,
                                      self.session_id)
        pppoe_if1.add_vpp_config()

        # Send PPPoE Discovery 2
        tx3 = self.create_stream_pppoe_discovery(self.pg2, self.pg1,
                                                 self.pg2.remote_mac)
        self.pg2.add_stream(tx3)
        self.pg_start()

        # Send PPPoE PPP LCP 2
        tx4 = self.create_stream_pppoe_lcp(self.pg2, self.pg1,
                                           self.pg2.remote_mac,
                                           self.session_id + 1)
        self.pg2.add_stream(tx4)
        self.pg_start()

        # Create PPPoE session 2
        pppoe_if2 = VppPppoeInterface(self,
                                      self.pg2.remote_ip4,
                                      self.pg2.remote_mac,
                                      self.session_id + 1)
        pppoe_if2.add_vpp_config()

        #
        # Send tunneled packets that match the created tunnel and
        # are decapped and forwarded
        #
        tx2 = self.create_stream_pppoe_ip4(self.pg0, self.pg1,
                                           self.pg0.remote_mac,
                                           self.session_id,
                                           self.pg0.remote_ip4)
        self.pg0.add_stream(tx2)

        self.pg_enable_capture(self.pg_interfaces)
        self.pg_start()

        rx2 = self.pg1.get_capture(len(tx2))
        self.verify_decapped_pppoe(self.pg0, rx2, tx2)

        tx5 = self.create_stream_pppoe_ip4(self.pg2, self.pg1,
                                           self.pg2.remote_mac,
                                           self.session_id + 1,
                                           self.pg2.remote_ip4)
        self.pg2.add_stream(tx5)

        self.pg_enable_capture(self.pg_interfaces)
        self.pg_start()

        rx5 = self.pg1.get_capture(len(tx5))
        self.verify_decapped_pppoe(self.pg2, rx5, tx5)

        self.logger.info(self.vapi.cli("show pppoe fib"))
        self.logger.info(self.vapi.cli("show pppoe session"))
        self.logger.info(self.vapi.cli("show ip fib"))

        #
        # test case cleanup
        #

        # Delete PPPoE session
        pppoe_if1.remove_vpp_config()
        pppoe_if2.remove_vpp_config()

        # Delete a route that resolves the server's destination
        route_sever_dst.remove_vpp_config()

    def test_PPPoE_Encap_Multiple(self):
        """ PPPoE Encap Multiple Sessions Test """

        self.vapi.cli("clear trace")

        #
        # Add a route that resolves the server's destination
        #
        route_sever_dst = VppIpRoute(self, "100.1.1.100", 32,
                                     [VppRoutePath(self.pg1.remote_ip4,
                                                   self.pg1.sw_if_index)])
        route_sever_dst.add_vpp_config()

        # Send PPPoE Discovery 1
        tx0 = self.create_stream_pppoe_discovery(self.pg0, self.pg1,
                                                 self.pg0.remote_mac)
        self.pg0.add_stream(tx0)
        self.pg_start()

        # Send PPPoE PPP LCP 1
        tx1 = self.create_stream_pppoe_lcp(self.pg0, self.pg1,
                                           self.pg0.remote_mac,
                                           self.session_id)
        self.pg0.add_stream(tx1)
        self.pg_start()

        # Create PPPoE session 1
        pppoe_if1 = VppPppoeInterface(self,
                                      self.pg0.remote_ip4,
                                      self.pg0.remote_mac,
                                      self.session_id)
        pppoe_if1.add_vpp_config()

        # Send PPPoE Discovery 2
        tx3 = self.create_stream_pppoe_discovery(self.pg2, self.pg1,
                                                 self.pg2.remote_mac)
        self.pg2.add_stream(tx3)
        self.pg_start()

        # Send PPPoE PPP LCP 2
        tx4 = self.create_stream_pppoe_lcp(self.pg2, self.pg1,
                                           self.pg2.remote_mac,
                                           self.session_id + 1)
        self.pg2.add_stream(tx4)
        self.pg_start()

        # Create PPPoE session 2
        pppoe_if2 = VppPppoeInterface(self,
                                      self.pg2.remote_ip4,
                                      self.pg2.remote_mac,
                                      self.session_id + 1)
        pppoe_if2.add_vpp_config()

        #
        # Send a packet stream that is routed into the session
        #  - packets are PPPoE encapped
        #
        self.vapi.cli("clear trace")
        tx2 = self.create_stream_ip4(self.pg1, self.pg0,
                                     self.pg0.remote_ip4, self.dst_ip)
        self.pg1.add_stream(tx2)

        self.pg_enable_capture(self.pg_interfaces)
        self.pg_start()

        rx2 = self.pg0.get_capture(len(tx2))
        self.verify_encaped_pppoe(self.pg1, rx2, tx2, self.session_id)

        tx5 = self.create_stream_ip4(self.pg1, self.pg2,
                                     self.pg2.remote_ip4, self.dst_ip)
        self.pg1.add_stream(tx5)

        self.pg_enable_capture(self.pg_interfaces)
        self.pg_start()

        rx5 = self.pg2.get_capture(len(tx5))
        self.verify_encaped_pppoe(self.pg1, rx5, tx5, self.session_id + 1)

        self.logger.info(self.vapi.cli("show pppoe fib"))
        self.logger.info(self.vapi.cli("show pppoe session"))
        self.logger.info(self.vapi.cli("show ip fib"))

        #
        # test case cleanup
        #

        # Delete PPPoE session
        pppoe_if1.remove_vpp_config()
        pppoe_if2.remove_vpp_config()

        # Delete a route that resolves the server's destination
        route_sever_dst.remove_vpp_config()

if __name__ == '__main__':
    unittest.main(testRunner=VppTestRunner)
n">buffer_known_hash, buffer_index); clib_spinlock_unlock (&bm->buffer_known_hash_lockp); return p ? p[0] : VLIB_BUFFER_UNKNOWN; } /* Validates sanity of a single buffer. Returns format'ed vector with error message if any. */ u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index, uword follow_chain); static_always_inline vlib_buffer_pool_t * vlib_get_buffer_pool (vlib_main_t * vm, u8 buffer_pool_index) { vlib_buffer_main_t *bm = vm->buffer_main; return vec_elt_at_index (bm->buffer_pools, buffer_pool_index); } static_always_inline uword vlib_buffer_pool_get (vlib_main_t * vm, u8 buffer_pool_index, u32 * buffers, u32 n_buffers) { vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index); u32 len; ASSERT (bp->buffers); clib_spinlock_lock (&bp->lock); len = vec_len (bp->buffers); if (PREDICT_TRUE (n_buffers < len)) { len -= n_buffers; vlib_buffer_copy_indices (buffers, bp->buffers + len, n_buffers); _vec_len (bp->buffers) = len; clib_spinlock_unlock (&bp->lock); return n_buffers; } else { vlib_buffer_copy_indices (buffers, bp->buffers, len); _vec_len (bp->buffers) = 0; clib_spinlock_unlock (&bp->lock); return len; } } /** \brief Allocate buffers from specific pool into supplied array @param vm - (vlib_main_t *) vlib main data structure pointer @param buffers - (u32 * ) buffer index array @param n_buffers - (u32) number of buffers requested @return - (u32) number of buffers actually allocated, may be less than the number requested or zero */ always_inline u32 vlib_buffer_alloc_from_pool (vlib_main_t * vm, u32 * buffers, u32 n_buffers, u8 buffer_pool_index) { vlib_buffer_main_t *bm = vm->buffer_main; vlib_buffer_pool_t *bp; vlib_buffer_pool_thread_t *bpt; u32 *src, *dst, len, n_left; bp = vec_elt_at_index (bm->buffer_pools, buffer_pool_index); bpt = vec_elt_at_index (bp->threads, vm->thread_index); dst = buffers; n_left = n_buffers; len = vec_len (bpt->cached_buffers); /* per-thread cache contains enough buffers */ if (len >= n_buffers) { src = bpt->cached_buffers + len - n_buffers; vlib_buffer_copy_indices (dst, src, n_buffers); _vec_len (bpt->cached_buffers) -= n_buffers; if (CLIB_DEBUG > 0) vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, VLIB_BUFFER_KNOWN_FREE); return n_buffers; } /* take everything available in the cache */ if (len) { vlib_buffer_copy_indices (dst, bpt->cached_buffers, len); _vec_len (bpt->cached_buffers) = 0; dst += len; n_left -= len; } len = round_pow2 (n_left, 32); vec_validate_aligned (bpt->cached_buffers, len - 1, CLIB_CACHE_LINE_BYTES); len = vlib_buffer_pool_get (vm, buffer_pool_index, bpt->cached_buffers, len); _vec_len (bpt->cached_buffers) = len; if (len) { u32 n_copy = clib_min (len, n_left); src = bpt->cached_buffers + len - n_copy; vlib_buffer_copy_indices (dst, src, n_copy); _vec_len (bpt->cached_buffers) -= n_copy; n_left -= n_copy; } n_buffers -= n_left; /* Verify that buffers are known free. */ if (CLIB_DEBUG > 0) vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, VLIB_BUFFER_KNOWN_FREE); return n_buffers; } /** \brief Allocate buffers from specific numa node into supplied array @param vm - (vlib_main_t *) vlib main data structure pointer @param buffers - (u32 * ) buffer index array @param n_buffers - (u32) number of buffers requested @param numa_node - (u32) numa node @return - (u32) number of buffers actually allocated, may be less than the number requested or zero */ always_inline u32 vlib_buffer_alloc_on_numa (vlib_main_t * vm, u32 * buffers, u32 n_buffers, u32 numa_node) { u8 index = vlib_buffer_pool_get_default_for_numa (vm, numa_node); return vlib_buffer_alloc_from_pool (vm, buffers, n_buffers, index); } /** \brief Allocate buffers into supplied array @param vm - (vlib_main_t *) vlib main data structure pointer @param buffers - (u32 * ) buffer index array @param n_buffers - (u32) number of buffers requested @return - (u32) number of buffers actually allocated, may be less than the number requested or zero */ always_inline u32 vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers) { return vlib_buffer_alloc_on_numa (vm, buffers, n_buffers, vm->numa_node); } /** \brief Allocate buffers into ring @param vm - (vlib_main_t *) vlib main data structure pointer @param buffers - (u32 * ) buffer index ring @param start - (u32) first slot in the ring @param ring_size - (u32) ring size @param n_buffers - (u32) number of buffers requested @return - (u32) number of buffers actually allocated, may be less than the number requested or zero */ always_inline u32 vlib_buffer_alloc_to_ring (vlib_main_t * vm, u32 * ring, u32 start, u32 ring_size, u32 n_buffers) { u32 n_alloc; ASSERT (n_buffers <= ring_size); if (PREDICT_TRUE (start + n_buffers <= ring_size)) return vlib_buffer_alloc (vm, ring + start, n_buffers); n_alloc = vlib_buffer_alloc (vm, ring + start, ring_size - start); if (PREDICT_TRUE (n_alloc == ring_size - start)) n_alloc += vlib_buffer_alloc (vm, ring, n_buffers - n_alloc); return n_alloc; } /** \brief Allocate buffers into ring from specific buffer pool @param vm - (vlib_main_t *) vlib main data structure pointer @param buffers - (u32 * ) buffer index ring @param start - (u32) first slot in the ring @param ring_size - (u32) ring size @param n_buffers - (u32) number of buffers requested @return - (u32) number of buffers actually allocated, may be less than the number requested or zero */ always_inline u32 vlib_buffer_alloc_to_ring_from_pool (vlib_main_t * vm, u32 * ring, u32 start, u32 ring_size, u32 n_buffers, u8 buffer_pool_index) { u32 n_alloc; ASSERT (n_buffers <= ring_size); if (PREDICT_TRUE (start + n_buffers <= ring_size)) return vlib_buffer_alloc_from_pool (vm, ring + start, n_buffers, buffer_pool_index); n_alloc = vlib_buffer_alloc_from_pool (vm, ring + start, ring_size - start, buffer_pool_index); if (PREDICT_TRUE (n_alloc == ring_size - start)) n_alloc += vlib_buffer_alloc_from_pool (vm, ring, n_buffers - n_alloc, buffer_pool_index); return n_alloc; } static_always_inline void vlib_buffer_pool_put (vlib_main_t * vm, u8 buffer_pool_index, u32 * buffers, u32 n_buffers) { vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index); vlib_buffer_pool_thread_t *bpt = vec_elt_at_index (bp->threads, vm->thread_index); if (CLIB_DEBUG > 0) vlib_buffer_validate_alloc_free (vm, buffers, n_buffers, VLIB_BUFFER_KNOWN_ALLOCATED); vec_add_aligned (bpt->cached_buffers, buffers, n_buffers, CLIB_CACHE_LINE_BYTES); if (vec_len (bpt->cached_buffers) > 4 * VLIB_FRAME_SIZE) { clib_spinlock_lock (&bp->lock); /* keep last stored buffers, as they are more likely hot in the cache */ vec_add_aligned (bp->buffers, bpt->cached_buffers, VLIB_FRAME_SIZE, CLIB_CACHE_LINE_BYTES); vec_delete (bpt->cached_buffers, VLIB_FRAME_SIZE, 0); bpt->n_alloc -= VLIB_FRAME_SIZE; clib_spinlock_unlock (&bp->lock); } } static_always_inline void vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers, int maybe_next) { const int queue_size = 128; vlib_buffer_pool_t *bp = 0; u8 buffer_pool_index = ~0; u32 n_queue = 0, queue[queue_size + 4]; vlib_buffer_t bt = { }; #if defined(CLIB_HAVE_VEC128) vlib_buffer_t bpi_mask = {.buffer_pool_index = ~0 }; vlib_buffer_t bpi_vec = {.buffer_pool_index = ~0 }; vlib_buffer_t flags_refs_mask = { .flags = VLIB_BUFFER_NEXT_PRESENT, .ref_count = ~1 }; #endif while (n_buffers) { vlib_buffer_t *b[8]; u32 bi, sum = 0, flags, next; if (n_buffers < 12) goto one_by_one; vlib_get_buffers (vm, buffers, b, 4); vlib_get_buffers (vm, buffers + 8, b + 4, 4); vlib_prefetch_buffer_header (b[4], LOAD); vlib_prefetch_buffer_header (b[5], LOAD); vlib_prefetch_buffer_header (b[6], LOAD); vlib_prefetch_buffer_header (b[7], LOAD); #if defined(CLIB_HAVE_VEC128) u8x16 p0, p1, p2, p3, r; p0 = u8x16_load_unaligned (b[0]); p1 = u8x16_load_unaligned (b[1]); p2 = u8x16_load_unaligned (b[2]); p3 = u8x16_load_unaligned (b[3]); r = p0 ^ bpi_vec.as_u8x16[0]; r |= p1 ^ bpi_vec.as_u8x16[0]; r |= p2 ^ bpi_vec.as_u8x16[0]; r |= p3 ^ bpi_vec.as_u8x16[0]; r &= bpi_mask.as_u8x16[0]; r |= (p0 | p1 | p2 | p3) & flags_refs_mask.as_u8x16[0]; sum = !u8x16_is_all_zero (r); #else sum |= b[0]->flags; sum |= b[1]->flags; sum |= b[2]->flags; sum |= b[3]->flags; sum &= VLIB_BUFFER_NEXT_PRESENT; sum += b[0]->ref_count - 1; sum += b[1]->ref_count - 1; sum += b[2]->ref_count - 1; sum += b[3]->ref_count - 1; sum |= b[0]->buffer_pool_index ^ buffer_pool_index; sum |= b[1]->buffer_pool_index ^ buffer_pool_index; sum |= b[2]->buffer_pool_index ^ buffer_pool_index; sum |= b[3]->buffer_pool_index ^ buffer_pool_index; #endif if (sum) goto one_by_one; vlib_buffer_copy_indices (queue + n_queue, buffers, 4); vlib_buffer_copy_template (b[0], &bt); vlib_buffer_copy_template (b[1], &bt); vlib_buffer_copy_template (b[2], &bt); vlib_buffer_copy_template (b[3], &bt); n_queue += 4; vlib_buffer_validate (vm, b[0]); vlib_buffer_validate (vm, b[1]); vlib_buffer_validate (vm, b[2]); vlib_buffer_validate (vm, b[3]); VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]); VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[1]); VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[2]); VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[3]); if (n_queue >= queue_size) { vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue); n_queue = 0; } buffers += 4; n_buffers -= 4; continue; one_by_one: bi = buffers[0]; next_in_chain: b[0] = vlib_get_buffer (vm, bi); flags = b[0]->flags; next = b[0]->next_buffer; if (PREDICT_FALSE (buffer_pool_index != b[0]->buffer_pool_index)) { if (n_queue) { vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue); n_queue = 0; } buffer_pool_index = b[0]->buffer_pool_index; #if defined(CLIB_HAVE_VEC128) bpi_vec.buffer_pool_index = buffer_pool_index; #endif bp = vlib_get_buffer_pool (vm, buffer_pool_index); vlib_buffer_copy_template (&bt, &bp->buffer_template); } vlib_buffer_validate (vm, b[0]); VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b[0]); if (clib_atomic_sub_fetch (&b[0]->ref_count, 1) == 0) { vlib_buffer_copy_template (b[0], &bt); queue[n_queue++] = bi; } if (n_queue == queue_size) { vlib_buffer_pool_put (vm, buffer_pool_index, queue, queue_size); n_queue = 0; } if (maybe_next && (flags & VLIB_BUFFER_NEXT_PRESENT)) { bi = next; goto next_in_chain; } buffers++; n_buffers--; } if (n_queue) vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue); } /** \brief Free buffers Frees the entire buffer chain for each buffer @param vm - (vlib_main_t *) vlib main data structure pointer @param buffers - (u32 * ) buffer index array @param n_buffers - (u32) number of buffers to free */ always_inline void vlib_buffer_free (vlib_main_t * vm, /* pointer to first buffer */ u32 * buffers, /* number of buffers to free */ u32 n_buffers) { vlib_buffer_free_inline (vm, buffers, n_buffers, /* maybe next */ 1); } /** \brief Free buffers, does not free the buffer chain for each buffer @param vm - (vlib_main_t *) vlib main data structure pointer @param buffers - (u32 * ) buffer index array @param n_buffers - (u32) number of buffers to free */ always_inline void vlib_buffer_free_no_next (vlib_main_t * vm, /* pointer to first buffer */ u32 * buffers, /* number of buffers to free */ u32 n_buffers) { vlib_buffer_free_inline (vm, buffers, n_buffers, /* maybe next */ 0); } /** \brief Free one buffer Shorthand to free a single buffer chain. @param vm - (vlib_main_t *) vlib main data structure pointer @param buffer_index - (u32) buffer index to free */ always_inline void vlib_buffer_free_one (vlib_main_t * vm, u32 buffer_index) { vlib_buffer_free_inline (vm, &buffer_index, 1, /* maybe next */ 1); } /** \brief Free buffers from ring @param vm - (vlib_main_t *) vlib main data structure pointer @param buffers - (u32 * ) buffer index ring @param start - (u32) first slot in the ring @param ring_size - (u32) ring size @param n_buffers - (u32) number of buffers */ always_inline void vlib_buffer_free_from_ring (vlib_main_t * vm, u32 * ring, u32 start, u32 ring_size, u32 n_buffers) { ASSERT (n_buffers <= ring_size); if (PREDICT_TRUE (start + n_buffers <= ring_size)) { vlib_buffer_free (vm, ring + start, n_buffers); } else { vlib_buffer_free (vm, ring + start, ring_size - start); vlib_buffer_free (vm, ring, n_buffers - (ring_size - start)); } } /** \brief Free buffers from ring without freeing tail buffers @param vm - (vlib_main_t *) vlib main data structure pointer @param buffers - (u32 * ) buffer index ring @param start - (u32) first slot in the ring @param ring_size - (u32) ring size @param n_buffers - (u32) number of buffers */ always_inline void vlib_buffer_free_from_ring_no_next (vlib_main_t * vm, u32 * ring, u32 start, u32 ring_size, u32 n_buffers) { ASSERT (n_buffers <= ring_size); if (PREDICT_TRUE (start + n_buffers <= ring_size)) { vlib_buffer_free_no_next (vm, ring + start, n_buffers); } else { vlib_buffer_free_no_next (vm, ring + start, ring_size - start); vlib_buffer_free_no_next (vm, ring, n_buffers - (ring_size - start)); } } /* Append given data to end of buffer, possibly allocating new buffers. */ int vlib_buffer_add_data (vlib_main_t * vm, u32 * buffer_index, void *data, u32 n_data_bytes); /* Define vlib_buffer and vnet_buffer flags bits preserved for copy/clone */ #define VLIB_BUFFER_COPY_CLONE_FLAGS_MASK \ (VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID | \ VLIB_BUFFER_IS_TRACED | ~VLIB_BUFFER_FLAGS_ALL) /* duplicate all buffers in chain */ always_inline vlib_buffer_t * vlib_buffer_copy (vlib_main_t * vm, vlib_buffer_t * b) { vlib_buffer_t *s, *d, *fd; uword n_alloc, n_buffers = 1; u32 flag_mask = VLIB_BUFFER_COPY_CLONE_FLAGS_MASK; int i; s = b; while (s->flags & VLIB_BUFFER_NEXT_PRESENT) { n_buffers++; s = vlib_get_buffer (vm, s->next_buffer); } u32 new_buffers[n_buffers]; n_alloc = vlib_buffer_alloc (vm, new_buffers, n_buffers); /* No guarantee that we'll get all the buffers we asked for */ if (PREDICT_FALSE (n_alloc < n_buffers)) { if (n_alloc > 0) vlib_buffer_free (vm, new_buffers, n_alloc); return 0; } /* 1st segment */ s = b; fd = d = vlib_get_buffer (vm, new_buffers[0]); d->current_data = s->current_data; d->current_length = s->current_length; d->flags = s->flags & flag_mask; d->trace_handle = s->trace_handle; d->total_length_not_including_first_buffer = s->total_length_not_including_first_buffer; clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque)); clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2)); clib_memcpy_fast (vlib_buffer_get_current (d), vlib_buffer_get_current (s), s->current_length); /* next segments */ for (i = 1; i < n_buffers; i++) { /* previous */ d->next_buffer = new_buffers[i]; /* current */ s = vlib_get_buffer (vm, s->next_buffer); d = vlib_get_buffer (vm, new_buffers[i]); d->current_data = s->current_data; d->current_length = s->current_length; clib_memcpy_fast (vlib_buffer_get_current (d), vlib_buffer_get_current (s), s->current_length); d->flags = s->flags & flag_mask; } return fd; } /* duplicate first buffer in chain */ always_inline vlib_buffer_t * vlib_buffer_copy_no_chain (vlib_main_t * vm, vlib_buffer_t * b, u32 * di) { vlib_buffer_t *d; if ((vlib_buffer_alloc (vm, di, 1)) != 1) return 0; d = vlib_get_buffer (vm, *di); /* 1st segment */ d->current_data = b->current_data; d->current_length = b->current_length; clib_memcpy_fast (d->opaque, b->opaque, sizeof (b->opaque)); clib_memcpy_fast (d->opaque2, b->opaque2, sizeof (b->opaque2)); clib_memcpy_fast (vlib_buffer_get_current (d), vlib_buffer_get_current (b), b->current_length); return d; } /* \brief Move packet from current position to offset position in buffer. Only work for small packet using one buffer with room to fit the move @param vm - (vlib_main_t *) vlib main data structure pointer @param b - (vlib_buffer_t *) pointer to buffer @param offset - (i16) position to move the packet in buffer */ always_inline void vlib_buffer_move (vlib_main_t * vm, vlib_buffer_t * b, i16 offset) { ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0); ASSERT (offset + VLIB_BUFFER_PRE_DATA_SIZE >= 0); ASSERT (offset + b->current_length < vlib_buffer_get_default_data_size (vm)); u8 *source = vlib_buffer_get_current (b); b->current_data = offset; u8 *destination = vlib_buffer_get_current (b); u16 length = b->current_length; if (source + length <= destination) /* no overlap */ clib_memcpy_fast (destination, source, length); else memmove (destination, source, length); } /** \brief Create a maximum of 256 clones of buffer and store them in the supplied array @param vm - (vlib_main_t *) vlib main data structure pointer @param src_buffer - (u32) source buffer index @param buffers - (u32 * ) buffer index array @param n_buffers - (u16) number of buffer clones requested (<=256) @param head_end_offset - (u16) offset relative to current position where packet head ends @param offset - (i16) copy packet head at current position if 0, else at offset position to change headroom space as specified @return - (u16) number of buffers actually cloned, may be less than the number requested or zero */ always_inline u16 vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers, u16 n_buffers, u16 head_end_offset, i16 offset) { u16 i; vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer); ASSERT (s->ref_count == 1); ASSERT (n_buffers); ASSERT (n_buffers <= 256); ASSERT (offset + VLIB_BUFFER_PRE_DATA_SIZE >= 0); ASSERT ((offset + head_end_offset) < vlib_buffer_get_default_data_size (vm)); if (s->current_length <= head_end_offset + CLIB_CACHE_LINE_BYTES * 2) { buffers[0] = src_buffer; if (offset) vlib_buffer_move (vm, s, offset); for (i = 1; i < n_buffers; i++) { vlib_buffer_t *d; d = vlib_buffer_copy (vm, s); if (d == 0) return i; buffers[i] = vlib_get_buffer_index (vm, d); } return n_buffers; } if (PREDICT_FALSE ((n_buffers == 1) && (offset == 0))) { buffers[0] = src_buffer; return 1; } n_buffers = vlib_buffer_alloc_from_pool (vm, buffers, n_buffers, s->buffer_pool_index); for (i = 0; i < n_buffers; i++) { vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]); if (offset) d->current_data = offset; else d->current_data = s->current_data; d->current_length = head_end_offset; ASSERT (d->buffer_pool_index == s->buffer_pool_index); d->total_length_not_including_first_buffer = s->current_length - head_end_offset; if (PREDICT_FALSE (s->flags & VLIB_BUFFER_NEXT_PRESENT)) { d->total_length_not_including_first_buffer += s->total_length_not_including_first_buffer; } d->flags = (s->flags & VLIB_BUFFER_COPY_CLONE_FLAGS_MASK) | VLIB_BUFFER_NEXT_PRESENT; d->trace_handle = s->trace_handle; clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque)); clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2)); clib_memcpy_fast (vlib_buffer_get_current (d), vlib_buffer_get_current (s), head_end_offset); d->next_buffer = src_buffer; } vlib_buffer_advance (s, head_end_offset); s->ref_count = n_buffers; while (s->flags & VLIB_BUFFER_NEXT_PRESENT) { s = vlib_get_buffer (vm, s->next_buffer); s->ref_count = n_buffers; } return n_buffers; } /** \brief Create multiple clones of buffer and store them in the supplied array @param vm - (vlib_main_t *) vlib main data structure pointer @param src_buffer - (u32) source buffer index @param buffers - (u32 * ) buffer index array @param n_buffers - (u16) number of buffer clones requested (<=256) @param head_end_offset - (u16) offset relative to current position where packet head ends @param offset - (i16) copy packet head at current position if 0, else at offset position to change headroom space as specified @return - (u16) number of buffers actually cloned, may be less than the number requested or zero */ always_inline u16 vlib_buffer_clone_at_offset (vlib_main_t * vm, u32 src_buffer, u32 * buffers, u16 n_buffers, u16 head_end_offset, i16 offset) { vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer); u16 n_cloned = 0; while (n_buffers > 256) { vlib_buffer_t *copy; copy = vlib_buffer_copy (vm, s); n_cloned += vlib_buffer_clone_256 (vm, vlib_get_buffer_index (vm, copy), (buffers + n_cloned), 256, head_end_offset, offset); n_buffers -= 256; } n_cloned += vlib_buffer_clone_256 (vm, src_buffer, buffers + n_cloned, n_buffers, head_end_offset, offset); return n_cloned; } /** \brief Create multiple clones of buffer and store them in the supplied array @param vm - (vlib_main_t *) vlib main data structure pointer @param src_buffer - (u32) source buffer index @param buffers - (u32 * ) buffer index array @param n_buffers - (u16) number of buffer clones requested (<=256) @param head_end_offset - (u16) offset relative to current position where packet head ends @return - (u16) number of buffers actually cloned, may be less than the number requested or zero */ always_inline u16 vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers, u16 n_buffers, u16 head_end_offset) { return vlib_buffer_clone_at_offset (vm, src_buffer, buffers, n_buffers, head_end_offset, 0); } /** \brief Attach cloned tail to the buffer @param vm - (vlib_main_t *) vlib main data structure pointer @param head - (vlib_buffer_t *) head buffer @param tail - (Vlib buffer_t *) tail buffer to clone and attach to head */ always_inline void vlib_buffer_attach_clone (vlib_main_t * vm, vlib_buffer_t * head, vlib_buffer_t * tail) { ASSERT ((head->flags & VLIB_BUFFER_NEXT_PRESENT) == 0); ASSERT (head->buffer_pool_index == tail->buffer_pool_index); head->flags |= VLIB_BUFFER_NEXT_PRESENT; head->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID; head->flags &= ~VLIB_BUFFER_EXT_HDR_VALID; head->flags |= (tail->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID); head->next_buffer = vlib_get_buffer_index (vm, tail); head->total_length_not_including_first_buffer = tail->current_length + tail->total_length_not_including_first_buffer; next_segment: clib_atomic_add_fetch (&tail->ref_count, 1); if (tail->flags & VLIB_BUFFER_NEXT_PRESENT) { tail = vlib_get_buffer (vm, tail->next_buffer); goto next_segment; } } /* Initializes the buffer as an empty packet with no chained buffers. */ always_inline void vlib_buffer_chain_init (vlib_buffer_t * first) { first->total_length_not_including_first_buffer = 0; first->current_length = 0; first->flags &= ~VLIB_BUFFER_NEXT_PRESENT; first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; } /* The provided next_bi buffer index is appended to the end of the packet. */ always_inline vlib_buffer_t * vlib_buffer_chain_buffer (vlib_main_t * vm, vlib_buffer_t * last, u32 next_bi) { vlib_buffer_t *next_buffer = vlib_get_buffer (vm, next_bi); last->next_buffer = next_bi; last->flags |= VLIB_BUFFER_NEXT_PRESENT; next_buffer->current_length = 0; next_buffer->flags &= ~VLIB_BUFFER_NEXT_PRESENT; return next_buffer; } /* Increases or decreases the packet length. * It does not allocate or deallocate new buffers. * Therefore, the added length must be compatible * with the last buffer. */ always_inline void vlib_buffer_chain_increase_length (vlib_buffer_t * first, vlib_buffer_t * last, i32 len) { last->current_length += len; if (first != last) first->total_length_not_including_first_buffer += len; } /* Copy data to the end of the packet and increases its length. * It does not allocate new buffers. * Returns the number of copied bytes. */ always_inline u16 vlib_buffer_chain_append_data (vlib_main_t * vm, vlib_buffer_t * first, vlib_buffer_t * last, void *data, u16 data_len) { u32 n_buffer_bytes = vlib_buffer_get_default_data_size (vm); ASSERT (n_buffer_bytes >= last->current_length + last->current_data); u16 len = clib_min (data_len, n_buffer_bytes - last->current_length - last->current_data); clib_memcpy_fast (vlib_buffer_get_current (last) + last->current_length, data, len); vlib_buffer_chain_increase_length (first, last, len); return len; } /* Copy data to the end of the packet and increases its length. * Allocates additional buffers from the free list if necessary. * Returns the number of copied bytes. * 'last' value is modified whenever new buffers are allocated and * chained and points to the last buffer in the chain. */ u16 vlib_buffer_chain_append_data_with_alloc (vlib_main_t * vm, vlib_buffer_t * first, vlib_buffer_t ** last, void *data, u16 data_len); void vlib_buffer_chain_validate (vlib_main_t * vm, vlib_buffer_t * first); format_function_t format_vlib_buffer, format_vlib_buffer_and_data, format_vlib_buffer_contents; typedef struct { /* Vector of packet data. */ u8 *packet_data; /* Number of buffers to allocate in each call to allocator. */ u32 min_n_buffers_each_alloc; u8 *name; } vlib_packet_template_t; void vlib_packet_template_init (vlib_main_t * vm, vlib_packet_template_t * t, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_alloc, char *fmt, ...); void *vlib_packet_template_get_packet (vlib_main_t * vm, vlib_packet_template_t * t, u32 * bi_result); always_inline void vlib_packet_template_free (vlib_main_t * vm, vlib_packet_template_t * t) { vec_free (t->packet_data); } always_inline u32 vlib_buffer_space_left_at_end (vlib_main_t * vm, vlib_buffer_t * b) { return b->data + vlib_buffer_get_default_data_size (vm) - ((u8 *) vlib_buffer_get_current (b) + b->current_length); } always_inline u32 vlib_buffer_chain_linearize (vlib_main_t * vm, vlib_buffer_t * b) { vlib_buffer_t *db = b, *sb, *first = b; int is_cloned = 0; u32 bytes_left = 0, data_size; u16 src_left, dst_left, n_buffers = 1; u8 *dp, *sp; u32 to_free = 0; if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0)) return 1; data_size = vlib_buffer_get_default_data_size (vm); dst_left = vlib_buffer_space_left_at_end (vm, b); while (b->flags & VLIB_BUFFER_NEXT_PRESENT) { b = vlib_get_buffer (vm, b->next_buffer); if (b->ref_count > 1) is_cloned = 1; bytes_left += b->current_length; n_buffers++; } /* if buffer is cloned, create completely new chain - unless everything fits * into one buffer */ if (is_cloned && bytes_left >= dst_left) { u32 len = 0; u32 space_needed = bytes_left - dst_left; u32 tail; if (vlib_buffer_alloc (vm, &tail, 1) == 0) return 0; ++n_buffers; len += data_size; b = vlib_get_buffer (vm, tail); while (len < space_needed) { u32 bi; if (vlib_buffer_alloc (vm, &bi, 1) == 0) { vlib_buffer_free_one (vm, tail); return 0; } b->flags = VLIB_BUFFER_NEXT_PRESENT; b->next_buffer = bi; b = vlib_get_buffer (vm, bi); len += data_size; n_buffers++; } sb = vlib_get_buffer (vm, first->next_buffer); to_free = first->next_buffer; first->next_buffer = tail; } else sb = vlib_get_buffer (vm, first->next_buffer); src_left = sb->current_length; sp = vlib_buffer_get_current (sb); dp = vlib_buffer_get_tail (db); while (bytes_left) { u16 bytes_to_copy; if (dst_left == 0) { db->current_length = dp - (u8 *) vlib_buffer_get_current (db); ASSERT (db->flags & VLIB_BUFFER_NEXT_PRESENT); db = vlib_get_buffer (vm, db->next_buffer); dst_left = data_size; if (db->current_data > 0) { db->current_data = 0; } else { dst_left += -db->current_data; } dp = vlib_buffer_get_current (db); } while (src_left == 0) { ASSERT (sb->flags & VLIB_BUFFER_NEXT_PRESENT); sb = vlib_get_buffer (vm, sb->next_buffer); src_left = sb->current_length; sp = vlib_buffer_get_current (sb); } bytes_to_copy = clib_min (dst_left, src_left); if (dp != sp) { if (sb == db) bytes_to_copy = clib_min (bytes_to_copy, sp - dp); clib_memcpy_fast (dp, sp, bytes_to_copy); } src_left -= bytes_to_copy; dst_left -= bytes_to_copy; dp += bytes_to_copy; sp += bytes_to_copy; bytes_left -= bytes_to_copy; } if (db != first) db->current_data = 0; db->current_length = dp - (u8 *) vlib_buffer_get_current (db); if (is_cloned && to_free) vlib_buffer_free_one (vm, to_free); else { if (db->flags & VLIB_BUFFER_NEXT_PRESENT) vlib_buffer_free_one (vm, db->next_buffer); db->flags &= ~VLIB_BUFFER_NEXT_PRESENT; b = first; n_buffers = 1; while (b->flags & VLIB_BUFFER_NEXT_PRESENT) { b = vlib_get_buffer (vm, b->next_buffer); ++n_buffers; } } first->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID; return n_buffers; } #endif /* included_vlib_buffer_funcs_h */ /* * fd.io coding-style-patch-verification: ON * * Local Variables: * eval: (c-set-style "gnu") * End: */