summaryrefslogtreecommitdiffstats
path: root/src/vcl/ldp.c
blob: 5f1eeb9d540b41b6cbb1a376c6ec9dfa372c3ba5 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
@media only all and (prefers-color-scheme: dark) {
.highlight .hll { background-color: #49483e }
.highlight .c { color: #75715e } /* Comment */
.highlight .err { color: #960050; background-color: #1e0010 } /* Error */
.highlight .k { color: #66d9ef } /* Keyword */
.highlight .l { color: #ae81ff } /* Literal */
.highlight .n { color: #f8f8f2 } /* Name */
.highlight .o { color: #f92672 } /* Operator */
.highlight .p { color: #f8f8f2 } /* Punctuation */
.highlight .ch { color: #75715e } /* Comment.Hashbang */
.highlight .cm { color: #75715e } /* Comment.Multiline */
.highlight .cp { color: #75715e } /* Comment.Preproc */
.highlight .cpf { color: #75715e } /* Comment.PreprocFile */
.highlight .c1 { color: #75715e } /* Comment.Single */
.highlight .cs { color: #75715e } /* Comment.Special */
.highlight .gd { color: #f92672 } /* Generic.Deleted */
.highlight .ge { font-style: italic } /* Generic.Emph */
.highlight .gi { color: #a6e22e } /* Generic.Inserted */
.highlight .gs { font-weight: bold } /* Generic.Strong */
.highlight .gu { color: #75715e } /* Generic.Subheading */
.highlight .kc { color: #66d9ef } /* Keyword.Constant */
.highlight .kd { color: #66d9ef } /* Keyword.Declaration */
.highlight .kn { color: #f92672 } /* Keyword.Namespace */
.highlight .kp { color: #66d9ef } /* Keyword.Pseudo */
.highlight .kr { color: #66d9ef } /* Keyword.Reserved */
.highlight .kt { color: #66d9ef } /* Keyword.Type */
.highlight .ld { color: #e6db74 } /* Literal.Date */
.highlight .m { color: #ae81ff } /* Literal.Number */
.highlight .s { color: #e6db74 } /* Literal.String */
.highlight .na { color: #a6e22e } /* Name.Attribute */
.highlight .nb { color: #f8f8f2 } /* Name.Builtin */
.highlight .nc { color: #a6e22e } /* Name.Class */
.highlight .no { color: #66d9ef } /* Name.Constant */
.highlight .nd { color: #a6e22e } /* Name.Decorator */
.highlight .ni { color: #f8f8f2 } /* Name.Entity */
.highlight .ne { color: #a6e22e } /* Name.Exception */
.highlight .nf { color: #a6e22e } /* Name.Function */
.highlight .nl { color: #f8f8f2 } /* Name.Label */
.highlight .nn { color: #f8f8f2 } /* Name.Namespace */
.highlight .nx { color: #a6e22e } /* Name.Other */
.highlight .py { color: #f8f8f2 } /* Name.Property */
.highlight .nt { color: #f92672 } /* Name.Tag */
.highlight .nv { color: #f8f8f2 } /* Name.Variable */
.highlight .ow { color: #f92672 } /* Operator.Word */
.highlight .w { color: #f8f8f2 } /* Text.Whitespace */
.highlight .mb { color: #ae81ff } /* Literal.Number.Bin */
.highlight .mf { color: #ae81ff } /* Literal.Number.Float */
.highlight .mh { color: #ae81ff } /* Literal.Number.Hex */
.highlight .mi { color: #ae81ff } /* Literal.Number.Integer */
.highlight .mo { color: #ae81ff } /* Literal.Number.Oct */
.highlight .sa { color: #e6db74 } /* Literal.String.Affix */
.highlight .sb { color: #e6db74 } /* Literal.String.Backtick */
.highlight .sc { color: #e6db74 } /* Literal.String.Char */
.highlight .dl { color: #e6db74 } /* Literal.String.Delimiter */
.highlight .sd { color: #e6db74 } /* Literal.String.Doc */
.highlight .s2 { color: #e6db74 } /* Literal.String.Double */
.highlight .se { color: #ae81ff } /* Literal.String.Escape */
.highlight .sh { color: #e6db74 } /* Literal.String.Heredoc */
.highlight .si { color: #e6db74 } /* Literal.String.Interpol */
.highlight .sx { color: #e6db74 } /* Literal.String.Other */
.highlight .sr { color: #e6db74 } /* Literal.String.Regex */
.highlight .s1 { color: #e6db74 } /* Literal.String.Single */
.highlight .ss { color: #e6db74 } /* Literal.String.Symbol */
.highlight .bp { color: #f8f8f2 } /* Name.Builtin.Pseudo */
.highlight .fm { color: #a6e22e } /* Name.Function.Magic */
.highlight .vc { color: #f8f8f2 } /* Name.Variable.Class */
.highlight .vg { color: #f8f8f2 } /* Name.Variable.Global */
.highlight .vi { color: #f8f8f2 } /* Name.Variable.Instance */
.highlight .vm { color: #f8f8f2 } /* Name.Variable.Magic */
.highlight .il { color: #ae81ff } /* Literal.Number.Integer.Long */
}
@media (prefers-color-scheme: light) {
.highlight .hll { background-color: #ffffcc }
.highlight .c { color: #888888 } /* Comment */
.highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */
.highlight .k { color: #008800; font-weight: bold } /* Keyword */
.highlight .ch { color: #888888 } /* Comment.Hashbang */
.highlight .cm { color: #888888 } /* Comment.Multiline */
.highlight .cp { color: #cc0000; font-weight: bold } /* Comment.Preproc */
.highlight .cpf { color: #888888 } /* Comment.PreprocFile */
.highlight .c1 { color: #888888 } /* Comment.Single */
.highlight .cs { color: #cc0000; font-weight: bold; background-color: #fff0f0 } /* Comment.Special */
.highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */
.highlight .ge { font-style: italic } /* Generic.Emph */
.highlight .gr { color: #aa0000 } /* Generic.Error */
.highlight .gh { color: #333333 } /* Generic.Heading */
.highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */
.highlight .go { color: #888888 } /* Generic.Output */
.highlight .gp { color: #555555 } /* Generic.Prompt */
.highlight .gs { font-weight: bold } /* Generic.Strong */
.highlight .gu { color: #666666 } /* Generic.Subheading */
.highlight .gt { color: #aa0000 } /* Generic.Traceback */
.highlight .kc { color: #008800; font-weight: bold } /* Keyword.Constant */
.highlight .kd { color: #008800; font-weight: bold } /* Keyword.Declaration */
.highlight .kn { color: #008800; font-weight: bold } /* Keyword.Namespace */
.highlight .kp { color: #008800 } /* Keyword.Pseudo */
.highlight .kr { color: #008800; font-weight: bold } /* Keyword.Reserved */
.highlight .kt { color: #888888; font-weight: bold } /* Keyword.Type */
.highlight .m { color: #0000DD; font-weight: bold } /* Literal.Number */
.highlight .s { color: #dd2200; background-color: #fff0f0 } /* Literal.String */
.highlight .na { color: #336699 } /* Name.Attribute */
.highlight .nb { color: #003388 } /* Name.Builtin */
.highlight .nc { color: #bb0066; font-weight: bold } /* Name.Class */
.highlight .no { color: #003366; font-weight: bold } /* Name.Constant */
.highlight .nd { color: #555555 } /* Name.Decorator */
.highlight .ne { color: #bb0066; font-weight: bold } /* Name.Exception */
.highlight .nf { color: #0066bb; font-weight: bold } /* Name.Function */
.highlight .nl { color: #336699; font-style: italic } /* Name.Label */
.highlight .nn { color: #bb0066; font-weight: bold } /* Name.Namespace */
.highlight .py { color: #336699; font-weight: bold } /* Name.Property */
.highlight .nt { color: #bb0066; font-weight: bold } /* Name.Tag */
.highlight .nv { color: #336699 } /* Name.Variable */
.highlight .ow { color: #008800 } /* Operator.Word */
.highlight .w { color: #bbbbbb } /* Text.Whitespace */
.highlight .mb { color: #0000DD; font-weight: bold } /* Literal.Number.Bin */
.highlight .mf { color: #0000DD; font-weight: bold } /* Literal.Number.Float */
.highlight .mh { color: #0000DD; font-weight: bold } /* Literal.Number.Hex */
.highlight .mi { color: #0000DD; font-weight: bold } /* Literal.Number.Integer */
.highlight .mo { color: #0000DD; font-weight: bold } /* Literal.Number.Oct */
.highlight .sa { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Affix */
.highlight .sb { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Backtick */
.highlight .sc { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Char */
.highlight .dl { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Delimiter */
.highlight .sd { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Doc */
.highlight .s2 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Double */
.highlight .se { color: #0044dd; background-color: #fff0f0 } /* Literal.String.Escape */
.highlight .sh { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Heredoc */
.highlight .si { color: #3333bb; background-color: #fff0f0 } /* Literal.String.Interpol */
.highlight .sx { color: #22bb22; background-color: #f0fff0 } /* Literal.String.Other */
.highlight .sr { color: #008800; background-color: #fff0ff } /* Literal.String.Regex */
.highlight .s1 { color: #dd2200; background-color: #fff0f0 } /* Literal.String.Single */
.highlight .ss { color: #aa6600; background-color: #fff0f0 } /* Literal.String.Symbol */
.highlight .bp { color: #003388 } /* Name.Builtin.Pseudo */
.highlight .fm { color: #0066bb; font-weight: bold } /* Name.Function.Magic */
.highlight .vc { color: #336699 } /* Name.Variable.Class */
.highlight .vg { color: #dd7700 } /* Name.Variable.Global */
.highlight .vi { color: #3333bb } /* Name.Variable.Instance */
.highlight .vm { color: #336699 } /* Name.Variable.Magic */
.highlight .il { color: #0000DD; font-weight: bold } /* Literal.Number.Integer.Long */
}
#!/usr/bin/env python3
"""L2XC Multi-instance Test Case HLD:

**NOTES:**
    - higher number (more than 15) of pg-l2 interfaces causes problems => only
      14 pg-l2 interfaces and 10 cross-connects are tested
    - jumbo packets in configuration with 14 l2-pg interfaces leads to
      problems too

**config 1**
    - add 14 pg-l2 interfaces
    - add 10 cross-connects (two cross-connects per pair of l2-pg interfaces)

**test 1**
    - send L2 MAC frames between all pairs of pg-l2 interfaces

**verify 1**
    - all packets received correctly in case of cross-connected l2-pg
      interfaces
    - no packet received in case of not cross-connected l2-pg interfaces

**config 2**
    - delete 4 cross-connects

**test 2**
    - send L2 MAC frames between all pairs of pg-l2 interfaces

**verify 2**
    - all packets received correctly in case of cross-connected l2-pg
      interfaces
    - no packet received in case of not cross-connected l2-pg interfaces

**config 3**
    - add new 4 cross-connects

**test 3**
    - send L2 MAC frames between all pairs of pg-l2 interfaces

**verify 3**
    - all packets received correctly in case of cross-connected l2-pg
      interfaces
    - no packet received in case of not cross-connected l2-pg interfaces

**config 4**
    - delete 10 cross-connects

**test 4**
    - send L2 MAC frames between all pairs of pg-l2 interfaces

**verify 4**
    - no packet received on all of l2-pg interfaces (no cross-connect created)
"""

import unittest
import random

from scapy.packet import Raw
from scapy.layers.l2 import Ether
from scapy.layers.inet import IP, UDP

from framework import VppTestCase, VppTestRunner
from util import Host, ppp


class TestL2xcMultiInst(VppTestCase):
    """ L2XC Multi-instance Test Case """

    @classmethod
    def setUpClass(cls):
        """
        Perform standard class setup (defined by class method setUpClass in
        class VppTestCase) before running the test case, set test case related
        variables and configure VPP.
        """
        super(TestL2xcMultiInst, cls).setUpClass()

        try:
            # Create pg interfaces
            cls.create_pg_interfaces(range(14))

            # Packet flows mapping pg0 -> pg1 etc.
            cls.flows = dict()
            for i in range(len(cls.pg_interfaces)):
                delta = 1 if i % 2 == 0 else -1
                cls.flows[cls.pg_interfaces[i]] =\
                    [cls.pg_interfaces[i + delta]]

            # Mapping between packet-generator index and lists of test hosts
            cls.hosts_by_pg_idx = dict()
            for pg_if in cls.pg_interfaces:
                cls.hosts_by_pg_idx[pg_if.sw_if_index] = []

            # Create test host entries
            cls.create_hosts(70)

            # Packet sizes - jumbo packet (9018 bytes) skipped
            cls.pg_if_packet_sizes = [64, 512, 1518]

            # Set up all interfaces
            for i in cls.pg_interfaces:
                i.admin_up()

            # Create list of x-connected pg_interfaces
            cls.pg_in_xc = list()

            # Create list of not x-connected pg_interfaces
            cls.pg_not_in_xc = list()
            for pg_if in cls.pg_interfaces:
                cls.pg_not_in_xc.append(pg_if)

        except Exception:
            super(TestL2xcMultiInst, cls).tearDownClass()
            raise

    @classmethod
    def tearDownClass(cls):
        super(TestL2xcMultiInst, cls).tearDownClass()

    def setUp(self):
        """
        Clear trace and packet infos before running each test.
        """
        super(TestL2xcMultiInst, self).setUp()
        self.reset_packet_infos()

    def tearDown(self):
        """
        Show various debug prints after each test.
        """
        super(TestL2xcMultiInst, self).tearDown()

    def show_commands_at_teardown(self):
        self.logger.info(self.vapi.ppcli("show l2patch"))

    @classmethod
    def create_hosts(cls, count):
        """
        Create required number of host MAC addresses and distribute them among
        interfaces. Create host IPv4 address for every host MAC address.

        :param int count: Number of hosts to create MAC/IPv4 addresses for.
        """
        n_int = len(cls.pg_interfaces)
        macs_per_if = count // n_int
        i = -1
        for pg_if in cls.pg_interfaces:
            i += 1
            start_nr = macs_per_if * i
            end_nr = count if i == (n_int - 1) else macs_per_if * (i + 1)
            hosts = cls.hosts_by_pg_idx[pg_if.sw_if_index]
            for j in range(start_nr, end_nr):
                host = Host(
                    "00:00:00:ff:%02x:%02x" % (pg_if.sw_if_index, j),
                    "172.17.1%02u.%u" % (pg_if.sw_if_index, j))
                hosts.append(host)

    def create_xconnects(self, count, start=0):
        """
        Create required number of cross-connects (always two cross-connects per
        pair of packet-generator interfaces).

        :param int count: Number of cross-connects to be created.
        :param int start: Starting index of packet-generator interfaces. \
        (Default value = 0)
        """
        for i in range(count):
            rx_if = self.pg_interfaces[i + start]
            delta = 1 if i % 2 == 0 else -1
            tx_if = self.pg_interfaces[i + start + delta]
            self.vapi.sw_interface_set_l2_xconnect(rx_if.sw_if_index,
                                                   tx_if.sw_if_index, 1)
            self.logger.info("Cross-connect from %s to %s created"
                             % (tx_if.name, rx_if.name))
            if self.pg_in_xc.count(rx_if) == 0:
                self.pg_in_xc.append(rx_if)
            if self.pg_not_in_xc.count(rx_if) == 1:
                self.pg_not_in_xc.remove(rx_if)

    def delete_xconnects(self, count, start=0):
        """
        Delete required number of cross-connects (always two cross-connects per
        pair of packet-generator interfaces).

        :param int count: Number of cross-connects to be deleted.
        :param int start: Starting index of packet-generator interfaces. \
        (Default value = 0)
        """
        for i in range(count):
            rx_if = self.pg_interfaces[i + start]
            delta = 1 if i % 2 == 0 else -1
            tx_if = self.pg_interfaces[i + start + delta]
            self.vapi.sw_interface_set_l2_xconnect(rx_if.sw_if_index,
                                                   tx_if.sw_if_index, 0)
            self.logger.info("Cross-connect from %s to %s deleted"
                             % (tx_if.name, rx_if.name))
            if self.pg_not_in_xc.count(rx_if) == 0:
                self.pg_not_in_xc.append(rx_if)
            if self.pg_in_xc.count(rx_if) == 1:
                self.pg_in_xc.remove(rx_if)

    def create_stream(self, src_if, packet_sizes):
        """
        Create input packet stream for defined interface using hosts list.

        :param object src_if: Interface to create packet stream for.
        :param list packet_sizes: List of required packet sizes.
        :return: Stream of packets.
        """
        pkts = []
        src_hosts = self.hosts_by_pg_idx[src_if.sw_if_index]
        for dst_if in self.flows[src_if]:
            dst_hosts = self.hosts_by_pg_idx[dst_if.sw_if_index]
            n_int = len(dst_hosts)
            for i in range(0, n_int):
                dst_host = dst_hosts[i]
                src_host = random.choice(src_hosts)
                pkt_info = self.create_packet_info(src_if, dst_if)
                payload = self.info_to_payload(pkt_info)
                p = (Ether(dst=dst_host.mac, src=src_host.mac) /
                     IP(src=src_host.ip4, dst=dst_host.ip4) /
                     UDP(sport=1234, dport=1234) /
                     Raw(payload))
                pkt_info.data = p.copy()
                size = random.choice(packet_sizes)
                self.extend_packet(p, size)
                pkts.append(p)
        self.logger.debug("Input stream created for port %s. Length: %u pkt(s)"
                          % (src_if.name, len(pkts)))
        return pkts

    def verify_capture(self, pg_if, capture):
        """
        Verify captured input packet stream for defined interface.

        :param object pg_if: Interface to verify captured packet stream for.
        :param list capture: Captured packet stream.
        """
        last_info = dict()
        for i in self.pg_interfaces:
            last_info[i.sw_if_index] = None
        dst_sw_if_index = pg_if.sw_if_index
        for packet in capture:
            payload_info = self.payload_to_info(packet[Raw])
            try:
                ip = packet[IP]
                udp = packet[UDP]
                packet_index = payload_info.index
                self.assertEqual(payload_info.dst, dst_sw_if_index)
                self.logger.debug("Got packet on port %s: src=%u (id=%u)" %
                                  (pg_if.name, payload_info.src, packet_index))
                next_info = self.get_next_packet_info_for_interface2(
                    payload_info.src, dst_sw_if_index,
                    last_info[payload_info.src])
                last_info[payload_info.src] = next_info
                self.assertTrue(next_info is not None)
                self.assertEqual(packet_index, next_info.index)
                saved_packet = next_info.data
                # Check standard fields
                self.assertEqual(ip.src, saved_packet[IP].src)
                self.assertEqual(ip.dst, saved_packet[IP].dst)
                self.assertEqual(udp.sport, saved_packet[UDP].sport)
                self.assertEqual(udp.dport, saved_packet[UDP].dport)
            except:
                self.logger.error(ppp("Unexpected or invalid packet:", packet))
                raise
        for i in self.pg_interfaces:
            remaining_packet = self.get_next_packet_info_for_interface2(
                i, dst_sw_if_index, last_info[i.sw_if_index])
            self.assertTrue(
                remaining_packet is None,
                "Port %u: Packet expected from source %u didn't arrive" %
                (dst_sw_if_index, i.sw_if_index))

    def run_verify_test(self):
        """
        Create packet streams for all configured l2-pg interfaces, send all \
        prepared packet streams and verify that:
            - all packets received correctly on all pg-l2 interfaces assigned
              to cross-connects
            - no packet received on all pg-l2 interfaces not assigned to
              cross-connects

        :raise RuntimeError: if no packet captured on l2-pg interface assigned
                             to the cross-connect or if any packet is captured
                             on l2-pg interface not assigned to the
                             cross-connect.
        """
        # Test
        # Create incoming packet streams for packet-generator interfaces
        for pg_if in self.pg_interfaces:
            pkts = self.create_stream(pg_if, self.pg_if_packet_sizes)
            pg_if.add_stream(pkts)

        # Enable packet capture and start packet sending
        self.pg_enable_capture(self.pg_interfaces)
        self.pg_start()

        # Verify
        # Verify outgoing packet streams per packet-generator interface
        for pg_if in self.pg_interfaces:
            if pg_if in self.pg_in_xc:
                capture = pg_if.get_capture(
                    remark="interface is a cross-connect sink")
                self.verify_capture(pg_if, capture)
            elif pg_if in self.pg_not_in_xc:
                pg_if.assert_nothing_captured(
                    remark="interface is not a cross-connect sink")
            else:
                raise Exception("Unexpected interface: %s" % pg_if.name)

    def test_l2xc_inst_01(self):
        """ L2XC Multi-instance test 1 - create 10 cross-connects
        """
        # Config 1
        # Create 10 cross-connects
        self.create_xconnects(10)

        # Test 1
        self.run_verify_test()

    def test_l2xc_inst_02(self):
        """ L2XC Multi-instance test 2 - delete 4 cross-connects
        """
        # Config 2
        # Delete 4 cross-connects
        self.delete_xconnects(4)

        # Test 2
        self.run_verify_test()

    def test_l2xc_inst_03(self):
        """ L2BD Multi-instance 3 - add new 4 cross-connects
        """
        # Config 3
        # Add new 4 cross-connects
        self.create_xconnects(4, start=10)

        # Test 3
        self.run_verify_test()

    def test_l2xc_inst_04(self):
        """ L2XC Multi-instance test 4 - delete 10 cross-connects
        """
        # Config 4
        # Delete 10 cross-connects
        self.delete_xconnects(10, start=4)

        # Test 4
        self.run_verify_test()


if __name__ == '__main__':
    unittest.main(testRunner=VppTestRunner)
a id='n1925' href='#n1925'>1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714
/*
 * Copyright (c) 2016-2019 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include <unistd.h>
#include <stdio.h>
#include <signal.h>
#include <dlfcn.h>
#include <pthread.h>
#include <time.h>
#include <stdarg.h>
#include <sys/resource.h>
#include <netinet/tcp.h>

#include <vcl/ldp_socket_wrapper.h>
#include <vcl/ldp.h>
#include <sys/time.h>

#include <vcl/vcl_locked.h>
#include <vppinfra/time.h>
#include <vppinfra/bitmap.h>
#include <vppinfra/lock.h>
#include <vppinfra/pool.h>
#include <vppinfra/hash.h>

#define HAVE_CONSTRUCTOR_ATTRIBUTE
#ifdef HAVE_CONSTRUCTOR_ATTRIBUTE
#define CONSTRUCTOR_ATTRIBUTE                       \
    __attribute__ ((constructor))
#else
#define CONSTRUCTOR_ATTRIBUTE
#endif /* HAVE_CONSTRUCTOR_ATTRIBUTE */

#define HAVE_DESTRUCTOR_ATTRIBUTE
#ifdef HAVE_DESTRUCTOR_ATTRIBUTE
#define DESTRUCTOR_ATTRIBUTE                        \
    __attribute__ ((destructor))
#else
#define DESTRUCTOR_ATTRIBUTE
#endif

#define LDP_MAX_NWORKERS 32

typedef struct ldp_worker_ctx_
{
  u8 *io_buffer;
  clib_time_t clib_time;

  /*
   * Select state
   */
  clib_bitmap_t *rd_bitmap;
  clib_bitmap_t *wr_bitmap;
  clib_bitmap_t *ex_bitmap;
  clib_bitmap_t *si_rd_bitmap;
  clib_bitmap_t *si_wr_bitmap;
  clib_bitmap_t *si_ex_bitmap;
  clib_bitmap_t *libc_rd_bitmap;
  clib_bitmap_t *libc_wr_bitmap;
  clib_bitmap_t *libc_ex_bitmap;

  /*
   * Poll state
   */
  vcl_poll_t *vcl_poll;
  struct pollfd *libc_poll;
  u16 *libc_poll_idxs;

  /*
   * Epoll state
   */
  u8 epoll_wait_vcl;
  u8 mq_epfd_added;
  int vcl_mq_epfd;

} ldp_worker_ctx_t;

/* clib_bitmap_t, fd_mask and vcl_si_set are used interchangeably. Make sure
 * they are the same size */
STATIC_ASSERT (sizeof (clib_bitmap_t) == sizeof (fd_mask),
	       "ldp bitmap size mismatch");
STATIC_ASSERT (sizeof (vcl_si_set) == sizeof (fd_mask),
	       "ldp bitmap size mismatch");

typedef struct
{
  ldp_worker_ctx_t *workers;
  int init;
  char app_name[LDP_APP_NAME_MAX];
  u32 vlsh_bit_val;
  u32 vlsh_bit_mask;
  u32 debug;

  /** vcl needs next epoll_create to go to libc_epoll */
  u8 vcl_needs_real_epoll;

  /**
   * crypto state used only for testing
   */
  u8 transparent_tls;
  u32 ckpair_index;
} ldp_main_t;

#define LDP_DEBUG ldp->debug

#define LDBG(_lvl, _fmt, _args...) 					\
  if (ldp->debug > _lvl)						\
    {									\
      int errno_saved = errno;						\
      fprintf (stderr, "ldp<%d>: " _fmt "\n", getpid(), ##_args);	\
      errno = errno_saved;						\
    }

static ldp_main_t ldp_main = {
  .vlsh_bit_val = (1 << LDP_SID_BIT_MIN),
  .vlsh_bit_mask = (1 << LDP_SID_BIT_MIN) - 1,
  .debug = LDP_DEBUG_INIT,
  .transparent_tls = 0,
  .ckpair_index = ~0,
};

static ldp_main_t *ldp = &ldp_main;

static inline ldp_worker_ctx_t *
ldp_worker_get_current (void)
{
  return (ldp->workers + vppcom_worker_index ());
}

/*
 * RETURN:  0 on success or -1 on error.
 * */
static inline void
ldp_set_app_name (char *app_name)
{
  snprintf (ldp->app_name, LDP_APP_NAME_MAX,
	    "ldp-%d-%s", getpid (), app_name);
}

static inline char *
ldp_get_app_name ()
{
  if (ldp->app_name[0] == '\0')
    ldp_set_app_name ("app");

  return ldp->app_name;
}

static inline int
ldp_vlsh_to_fd (vls_handle_t vlsh)
{
  return (vlsh + ldp->vlsh_bit_val);
}

static inline vls_handle_t
ldp_fd_to_vlsh (int fd)
{
  if (fd < ldp->vlsh_bit_val)
    return VLS_INVALID_HANDLE;

  return (fd - ldp->vlsh_bit_val);
}

static void
ldp_alloc_workers (void)
{
  if (ldp->workers)
    return;
  pool_alloc (ldp->workers, LDP_MAX_NWORKERS);
}

static int
ldp_init (void)
{
  ldp_worker_ctx_t *ldpw;
  int rv;

  ASSERT (!ldp->init);

  ldp->init = 1;
  ldp->vcl_needs_real_epoll = 1;
  rv = vls_app_create (ldp_get_app_name ());
  if (rv != VPPCOM_OK)
    {
      ldp->vcl_needs_real_epoll = 0;
      if (rv == VPPCOM_EEXIST)
	return 0;
      LDBG (2, "\nERROR: ldp_init: vppcom_app_create()"
	    " failed!  rv = %d (%s)\n", rv, vppcom_retval_str (rv));
      ldp->init = 0;
      return rv;
    }
  ldp->vcl_needs_real_epoll = 0;
  ldp_alloc_workers ();
  ldpw = ldp_worker_get_current ();

  char *env_var_str = getenv (LDP_ENV_DEBUG);
  if (env_var_str)
    {
      u32 tmp;
      if (sscanf (env_var_str, "%u", &tmp) != 1)
	clib_warning ("LDP<%d>: WARNING: Invalid LDP debug level specified in"
		      " the env var " LDP_ENV_DEBUG " (%s)!", getpid (),
		      env_var_str);
      else
	{
	  ldp->debug = tmp;
	  LDBG (0, "configured LDP debug level (%u) from env var "
		LDP_ENV_DEBUG "!", ldp->debug);
	}
    }

  env_var_str = getenv (LDP_ENV_APP_NAME);
  if (env_var_str)
    {
      ldp_set_app_name (env_var_str);
      LDBG (0, "configured LDP app name (%s) from the env var "
	    LDP_ENV_APP_NAME "!", ldp->app_name);
    }

  env_var_str = getenv (LDP_ENV_SID_BIT);
  if (env_var_str)
    {
      u32 sb;
      if (sscanf (env_var_str, "%u", &sb) != 1)
	{
	  LDBG (0, "WARNING: Invalid LDP sid bit specified in the env var "
		LDP_ENV_SID_BIT " (%s)! sid bit value %d (0x%x)", env_var_str,
		ldp->vlsh_bit_val, ldp->vlsh_bit_val);
	}
      else if (sb < LDP_SID_BIT_MIN)
	{
	  ldp->vlsh_bit_val = (1 << LDP_SID_BIT_MIN);
	  ldp->vlsh_bit_mask = ldp->vlsh_bit_val - 1;

	  LDBG (0, "WARNING: LDP sid bit (%u) specified in the env var "
		LDP_ENV_SID_BIT " (%s) is too small. Using LDP_SID_BIT_MIN"
		" (%d)! sid bit value %d (0x%x)", sb, env_var_str,
		LDP_SID_BIT_MIN, ldp->vlsh_bit_val, ldp->vlsh_bit_val);
	}
      else if (sb > LDP_SID_BIT_MAX)
	{
	  ldp->vlsh_bit_val = (1 << LDP_SID_BIT_MAX);
	  ldp->vlsh_bit_mask = ldp->vlsh_bit_val - 1;

	  LDBG (0, "WARNING: LDP sid bit (%u) specified in the env var "
		LDP_ENV_SID_BIT " (%s) is too big. Using LDP_SID_BIT_MAX"
		" (%d)! sid bit value %d (0x%x)", sb, env_var_str,
		LDP_SID_BIT_MAX, ldp->vlsh_bit_val, ldp->vlsh_bit_val);
	}
      else
	{
	  ldp->vlsh_bit_val = (1 << sb);
	  ldp->vlsh_bit_mask = ldp->vlsh_bit_val - 1;

	  LDBG (0, "configured LDP sid bit (%u) from "
		LDP_ENV_SID_BIT "!  sid bit value %d (0x%x)", sb,
		ldp->vlsh_bit_val, ldp->vlsh_bit_val);
	}

      /* Make sure there are enough bits in the fd set for vcl sessions */
      if (ldp->vlsh_bit_val > FD_SETSIZE / 2)
	{
	  LDBG (0, "ERROR: LDP vlsh bit value %d > FD_SETSIZE/2 %d!",
		ldp->vlsh_bit_val, FD_SETSIZE / 2);
	  ldp->init = 0;
	  return -1;
	}
    }
  env_var_str = getenv (LDP_ENV_TLS_TRANS);
  if (env_var_str)
    {
      ldp->transparent_tls = 1;
    }

  /* *INDENT-OFF* */
  pool_foreach (ldpw, ldp->workers)  {
    clib_memset (&ldpw->clib_time, 0, sizeof (ldpw->clib_time));
  }
  /* *INDENT-ON* */

  LDBG (0, "LDP initialization: done!");

  return 0;
}

#define ldp_init_check()                                                      \
  if (PREDICT_FALSE (!ldp->init))                                             \
    {                                                                         \
      if ((errno = -ldp_init ()))                                             \
	return -1;                                                            \
    }

int
close (int fd)
{
  vls_handle_t vlsh;
  int rv, epfd;

  ldp_init_check ();

  vlsh = ldp_fd_to_vlsh (fd);
  if (vlsh != VLS_INVALID_HANDLE)
    {
      epfd = vls_attr (vlsh, VPPCOM_ATTR_GET_LIBC_EPFD, 0, 0);
      if (epfd > 0)
	{
	  LDBG (0, "fd %d: calling libc_close: epfd %u", fd, epfd);

	  rv = libc_close (epfd);
	  if (rv < 0)
	    {
	      u32 size = sizeof (epfd);
	      epfd = 0;

	      (void) vls_attr (vlsh, VPPCOM_ATTR_SET_LIBC_EPFD, &epfd, &size);
	    }
	}
      else if (PREDICT_FALSE (epfd < 0))
	{
	  errno = -epfd;
	  rv = -1;
	  goto done;
	}

      LDBG (0, "fd %d: calling vls_close: vlsh %u", fd, vlsh);

      rv = vls_close (vlsh);
      if (rv != VPPCOM_OK)
	{
	  errno = -rv;
	  rv = -1;
	}
    }
  else
    {
      LDBG (0, "fd %d: calling libc_close", fd);
      rv = libc_close (fd);
    }

done:
  return rv;
}

ssize_t
read (int fd, void *buf, size_t nbytes)
{
  vls_handle_t vlsh;
  ssize_t size;

  ldp_init_check ();

  vlsh = ldp_fd_to_vlsh (fd);
  if (vlsh != VLS_INVALID_HANDLE)
    {
      size = vls_read (vlsh, buf, nbytes);
      if (size < 0)
	{
	  errno = -size;
	  size = -1;
	}
    }
  else
    {
      size = libc_read (fd, buf, nbytes);
    }

  return size;
}

ssize_t
readv (int fd, const struct iovec * iov, int iovcnt)
{
  int rv = 0, i, total = 0;
  vls_handle_t vlsh;
  ssize_t size = 0;

  ldp_init_check ();

  vlsh = ldp_fd_to_vlsh (fd);
  if (vlsh != VLS_INVALID_HANDLE)
    {
      for (i = 0; i < iovcnt; ++i)
	{
	  rv = vls_read (vlsh, iov[i].iov_base, iov[i].iov_len);
	  if (rv <= 0)
	    break;
	  else
	    {
	      total += rv;
	      if (rv < iov[i].iov_len)
		break;
	    }
	}
      if (rv < 0 && total == 0)
	{
	  errno = -rv;
	  size = -1;
	}
      else
	size = total;
    }
  else
    {
      size = libc_readv (fd, iov, iovcnt);
    }

  return size;
}

ssize_t
write (int fd, const void *buf, size_t nbytes)
{
  vls_handle_t vlsh;
  ssize_t size = 0;

  ldp_init_check ();

  vlsh = ldp_fd_to_vlsh (fd);
  if (vlsh != VLS_INVALID_HANDLE)
    {
      size = vls_write_msg (vlsh, (void *) buf, nbytes);
      if (size < 0)
	{
	  errno = -size;
	  size = -1;
	}
    }
  else
    {
      size = libc_write (fd, buf, nbytes);
    }

  return size;
}

ssize_t
writev (int fd, const struct iovec * iov, int iovcnt)
{
  ssize_t size = 0, total = 0;
  vls_handle_t vlsh;
  int i, rv = 0;

  ldp_init_check ();

  vlsh = ldp_fd_to_vlsh (fd);
  if (vlsh != VLS_INVALID_HANDLE)
    {
      for (i = 0; i < iovcnt; ++i)
	{
	  rv = vls_write_msg (vlsh, iov[i].iov_base, iov[i].iov_len);
	  if (rv < 0)
	    break;
	  else
	    {
	      total += rv;
	      if (rv < iov[i].iov_len)
		break;
	    }
	}

      if (rv < 0 && total == 0)
	{
	  errno = -rv;
	  size = -1;
	}
      else
	size = total;
    }
  else
    {
      size = libc_writev (fd, iov, iovcnt);
    }

  return size;
}

static int
fcntl_internal (int fd, int cmd, va_list ap)
{
  vls_handle_t vlsh;
  int rv = 0;

  vlsh = ldp_fd_to_vlsh (fd);
  LDBG (0, "fd %u vlsh %d, cmd %u", fd, vlsh, cmd);
  if (vlsh != VLS_INVALID_HANDLE)
    {
      int flags = va_arg (ap, int);
      u32 size;

      size = sizeof (flags);
      rv = -EOPNOTSUPP;
      switch (cmd)
	{
	case F_SETFL:
	  rv = vls_attr (vlsh, VPPCOM_ATTR_SET_FLAGS, &flags, &size);
	  break;

	case F_GETFL:
	  rv = vls_attr (vlsh, VPPCOM_ATTR_GET_FLAGS, &flags, &size);
	  if (rv == VPPCOM_OK)
	    rv = flags;
	  break;
	case F_SETFD:
	  /* TODO handle this */
	  LDBG (0, "F_SETFD ignored flags %u", flags);
	  rv = 0;
	  break;
	default:
	  rv = -EOPNOTSUPP;
	  break;
	}
      if (rv < 0)
	{
	  errno = -rv;
	  rv = -1;
	}
    }
  else
    {
#ifdef HAVE_FCNTL64
      rv = libc_vfcntl64 (fd, cmd, ap);
#else
      rv = libc_vfcntl (fd, cmd, ap);
#endif
    }

  return rv;
}

int
fcntl (int fd, int cmd, ...)
{
  va_list ap;
  int rv;

  ldp_init_check ();

  va_start (ap, cmd);
  rv = fcntl_internal (fd, cmd, ap);
  va_end (ap);

  return rv;
}

int
fcntl64 (int fd, int cmd, ...)
{
  va_list ap;
  int rv;

  ldp_init_check ();

  va_start (ap, cmd);
  rv = fcntl_internal (fd, cmd, ap);
  va_end (ap);
  return rv;
}

int
ioctl (int fd, unsigned long int cmd, ...)
{
  vls_handle_t vlsh;
  va_list ap;
  int rv;

  ldp_init_check ();

  va_start (ap, cmd);

  vlsh = ldp_fd_to_vlsh (fd);
  if (vlsh != VLS_INVALID_HANDLE)
    {
      switch (cmd)
	{
	case FIONREAD:
	  rv = vls_attr (vlsh, VPPCOM_ATTR_GET_NREAD, 0, 0);
	  break;

	case FIONBIO:
	  {
	    u32 flags = va_arg (ap, int) ? O_NONBLOCK : 0;
	    u32 size = sizeof (flags);

	    /* TBD: When VPPCOM_ATTR_[GS]ET_FLAGS supports flags other than
	     *      non-blocking, the flags should be read here and merged
	     *      with O_NONBLOCK.
	     */
	    rv = vls_attr (vlsh, VPPCOM_ATTR_SET_FLAGS, &flags, &size);
	  }
	  break;

	default:
	  rv = -EOPNOTSUPP;
	  break;
	}
      if (rv < 0)
	{
	  errno = -rv;
	  rv = -1;
	}
    }
  else
    {
      rv = libc_vioctl (fd, cmd, ap);
    }

  va_end (ap);
  return rv;
}

always_inline void
ldp_select_init_maps (fd_set * __restrict original,
		      clib_bitmap_t ** resultb, clib_bitmap_t ** libcb,
		      clib_bitmap_t ** vclb, int nfds, u32 minbits,
		      u32 n_bytes, uword * si_bits, uword * libc_bits)
{
  uword si_bits_set, libc_bits_set;
  vls_handle_t vlsh;
  int fd;

  clib_bitmap_validate (*vclb, minbits);
  clib_bitmap_validate (*libcb, minbits);
  clib_bitmap_validate (*resultb, minbits);
  clib_memcpy_fast (*resultb, original, n_bytes);
  memset (original, 0, n_bytes);

  /* *INDENT-OFF* */
  clib_bitmap_foreach (fd, *resultb)  {
    if (fd > nfds)
      break;
    vlsh = ldp_fd_to_vlsh (fd);
    if (vlsh == VLS_INVALID_HANDLE)
      clib_bitmap_set_no_check (*libcb, fd, 1);
    else
      *vclb = clib_bitmap_set (*vclb, vlsh_to_session_index (vlsh), 1);
  }
  /* *INDENT-ON* */

  si_bits_set = clib_bitmap_last_set (*vclb) + 1;
  *si_bits = (si_bits_set > *si_bits) ? si_bits_set : *si_bits;
  clib_bitmap_validate (*resultb, *si_bits);

  libc_bits_set = clib_bitmap_last_set (*libcb) + 1;
  *libc_bits = (libc_bits_set > *libc_bits) ? libc_bits_set : *libc_bits;
}

always_inline int
ldp_select_vcl_map_to_libc (clib_bitmap_t * vclb, fd_set * __restrict libcb)
{
  vls_handle_t vlsh;
  uword si;
  int fd;

  if (!libcb)
    return 0;

  /* *INDENT-OFF* */
  clib_bitmap_foreach (si, vclb)  {
    vlsh = vls_session_index_to_vlsh (si);
    ASSERT (vlsh != VLS_INVALID_HANDLE);
    fd = ldp_vlsh_to_fd (vlsh);
    if (PREDICT_FALSE (fd < 0))
      {
        errno = EBADFD;
        return -1;
      }
    FD_SET (fd, libcb);
  }
  /* *INDENT-ON* */

  return 0;
}

always_inline void
ldp_select_libc_map_merge (clib_bitmap_t * result, fd_set * __restrict libcb)
{
  uword fd;

  if (!libcb)
    return;

  /* *INDENT-OFF* */
  clib_bitmap_foreach (fd, result)
    FD_SET ((int)fd, libcb);
  /* *INDENT-ON* */
}

int
ldp_pselect (int nfds, fd_set * __restrict readfds,
	     fd_set * __restrict writefds,
	     fd_set * __restrict exceptfds,
	     const struct timespec *__restrict timeout,
	     const __sigset_t * __restrict sigmask)
{
  u32 minbits = clib_max (nfds, BITS (uword)), n_bytes;
  ldp_worker_ctx_t *ldpw = ldp_worker_get_current ();
  struct timespec libc_tspec = { 0 };
  f64 time_out, vcl_timeout = 0;
  uword si_bits, libc_bits;
  int rv, bits_set = 0;

  if (nfds < 0)
    {
      errno = EINVAL;
      return -1;
    }

  if (PREDICT_FALSE (ldpw->clib_time.init_cpu_time == 0))
    clib_time_init (&ldpw->clib_time);

  if (timeout)
    {
      time_out = (timeout->tv_sec == 0 && timeout->tv_nsec == 0) ?
	(f64) 0 : (f64) timeout->tv_sec + (f64) timeout->tv_nsec / (f64) 1e9;

      time_out += clib_time_now (&ldpw->clib_time);

      /* select as fine grained sleep */
      if (!nfds)
	{
	  while (clib_time_now (&ldpw->clib_time) < time_out)
	    ;
	  return 0;
	}
    }
  else if (!nfds)
    {
      errno = EINVAL;
      return -1;
    }
  else
    time_out = -1;

  if (nfds <= ldp->vlsh_bit_val)
    {
      rv = libc_pselect (nfds, readfds, writefds, exceptfds,
			 timeout, sigmask);
      goto done;
    }

  si_bits = libc_bits = 0;
  n_bytes = nfds / 8 + ((nfds % 8) ? 1 : 0);

  if (readfds)
    ldp_select_init_maps (readfds, &ldpw->rd_bitmap, &ldpw->libc_rd_bitmap,
			  &ldpw->si_rd_bitmap, nfds, minbits, n_bytes,
			  &si_bits, &libc_bits);
  if (writefds)
    ldp_select_init_maps (writefds, &ldpw->wr_bitmap,
			  &ldpw->libc_wr_bitmap, &ldpw->si_wr_bitmap, nfds,
			  minbits, n_bytes, &si_bits, &libc_bits);
  if (exceptfds)
    ldp_select_init_maps (exceptfds, &ldpw->ex_bitmap,
			  &ldpw->libc_ex_bitmap, &ldpw->si_ex_bitmap, nfds,
			  minbits, n_bytes, &si_bits, &libc_bits);

  if (PREDICT_FALSE (!si_bits && !libc_bits))
    {
      errno = EINVAL;
      rv = -1;
      goto done;
    }

  if (!si_bits)
    libc_tspec = timeout ? *timeout : libc_tspec;

  do
    {
      if (si_bits)
	{
	  if (readfds)
	    clib_memcpy_fast (ldpw->rd_bitmap, ldpw->si_rd_bitmap,
			      vec_len (ldpw->si_rd_bitmap) *
			      sizeof (clib_bitmap_t));
	  if (writefds)
	    clib_memcpy_fast (ldpw->wr_bitmap, ldpw->si_wr_bitmap,
			      vec_len (ldpw->si_wr_bitmap) *
			      sizeof (clib_bitmap_t));
	  if (exceptfds)
	    clib_memcpy_fast (ldpw->ex_bitmap, ldpw->si_ex_bitmap,
			      vec_len (ldpw->si_ex_bitmap) *
			      sizeof (clib_bitmap_t));

	  rv = vls_select (si_bits, readfds ? ldpw->rd_bitmap : NULL,
			   writefds ? ldpw->wr_bitmap : NULL,
			   exceptfds ? ldpw->ex_bitmap : NULL, vcl_timeout);
	  if (rv < 0)
	    {
	      errno = -rv;
	      rv = -1;
	      goto done;
	    }
	  else if (rv > 0)
	    {
	      if (ldp_select_vcl_map_to_libc (ldpw->rd_bitmap, readfds))
		{
		  rv = -1;
		  goto done;
		}

	      if (ldp_select_vcl_map_to_libc (ldpw->wr_bitmap, writefds))
		{
		  rv = -1;
		  goto done;
		}

	      if (ldp_select_vcl_map_to_libc (ldpw->ex_bitmap, exceptfds))
		{
		  rv = -1;
		  goto done;
		}
	      bits_set = rv;
	    }
	}
      if (libc_bits)
	{
	  if (readfds)
	    clib_memcpy_fast (ldpw->rd_bitmap, ldpw->libc_rd_bitmap,
			      vec_len (ldpw->libc_rd_bitmap) *
			      sizeof (clib_bitmap_t));
	  if (writefds)
	    clib_memcpy_fast (ldpw->wr_bitmap, ldpw->libc_wr_bitmap,
			      vec_len (ldpw->libc_wr_bitmap) *
			      sizeof (clib_bitmap_t));
	  if (exceptfds)
	    clib_memcpy_fast (ldpw->ex_bitmap, ldpw->libc_ex_bitmap,
			      vec_len (ldpw->libc_ex_bitmap) *
			      sizeof (clib_bitmap_t));

	  rv = libc_pselect (libc_bits,
			     readfds ? (fd_set *) ldpw->rd_bitmap : NULL,
			     writefds ? (fd_set *) ldpw->wr_bitmap : NULL,
			     exceptfds ? (fd_set *) ldpw->ex_bitmap : NULL,
			     &libc_tspec, sigmask);
	  if (rv > 0)
	    {
	      ldp_select_libc_map_merge (ldpw->rd_bitmap, readfds);
	      ldp_select_libc_map_merge (ldpw->wr_bitmap, writefds);
	      ldp_select_libc_map_merge (ldpw->ex_bitmap, exceptfds);
	      bits_set += rv;
	    }
	}

      if (bits_set)
	{
	  rv = bits_set;
	  goto done;
	}
    }
  while ((time_out == -1) || (clib_time_now (&ldpw->clib_time) < time_out));
  rv = 0;

done:
  /* TBD: set timeout to amount of time left */
  clib_bitmap_zero (ldpw->rd_bitmap);
  clib_bitmap_zero (ldpw->si_rd_bitmap);
  clib_bitmap_zero (ldpw->libc_rd_bitmap);
  clib_bitmap_zero (ldpw->wr_bitmap);
  clib_bitmap_zero (ldpw->si_wr_bitmap);
  clib_bitmap_zero (ldpw->libc_wr_bitmap);
  clib_bitmap_zero (ldpw->ex_bitmap);
  clib_bitmap_zero (ldpw->si_ex_bitmap);
  clib_bitmap_zero (ldpw->libc_ex_bitmap);

  return rv;
}

int
select (int nfds, fd_set * __restrict readfds,
	fd_set * __restrict writefds,
	fd_set * __restrict exceptfds, struct timeval *__restrict timeout)
{
  struct timespec tspec;

  if (timeout)
    {
      tspec.tv_sec = timeout->tv_sec;
      tspec.tv_nsec = timeout->tv_usec * 1000;
    }
  return ldp_pselect (nfds, readfds, writefds, exceptfds,
		      timeout ? &tspec : NULL, NULL);
}

#ifdef __USE_XOPEN2K
int
pselect (int nfds, fd_set * __restrict readfds,
	 fd_set * __restrict writefds,
	 fd_set * __restrict exceptfds,
	 const struct timespec *__restrict timeout,
	 const __sigset_t * __restrict sigmask)
{
  return ldp_pselect (nfds, readfds, writefds, exceptfds, timeout, 0);
}
#endif

/* If transparent TLS mode is turned on, then ldp will load key and cert.
 */
static int
load_cert_key_pair (void)
{
  char *cert_str = getenv (LDP_ENV_TLS_CERT);
  char *key_str = getenv (LDP_ENV_TLS_KEY);
  char cert_buf[4096], key_buf[4096];
  int cert_size, key_size;
  vppcom_cert_key_pair_t crypto;
  int ckp_index;
  FILE *fp;

  if (!cert_str || !key_str)
    {
      LDBG (0, "ERROR: failed to read LDP environment %s\n",
	    LDP_ENV_TLS_CERT);
      return -1;
    }

  fp = fopen (cert_str, "r");
  if (fp == NULL)
    {
      LDBG (0, "ERROR: failed to open cert file %s \n", cert_str);
      return -1;
    }
  cert_size = fread (cert_buf, sizeof (char), sizeof (cert_buf), fp);
  fclose (fp);

  fp = fopen (key_str, "r");
  if (fp == NULL)
    {
      LDBG (0, "ERROR: failed to open key file %s \n", key_str);
      return -1;
    }
  key_size = fread (key_buf, sizeof (char), sizeof (key_buf), fp);
  fclose (fp);

  crypto.cert = cert_buf;
  crypto.key = key_buf;
  crypto.cert_len = cert_size;
  crypto.key_len = key_size;
  ckp_index = vppcom_add_cert_key_pair (&crypto);
  if (ckp_index < 0)
    {
      LDBG (0, "ERROR: failed to add cert key pair\n");
      return -1;
    }

  ldp->ckpair_index = ckp_index;

  return 0;
}

static int
assign_cert_key_pair (vls_handle_t vlsh)
{
  uint32_t ckp_len;

  if (ldp->ckpair_index == ~0 && load_cert_key_pair () < 0)
    return -1;

  ckp_len = sizeof (ldp->ckpair_index);
  return vls_attr (vlsh, VPPCOM_ATTR_SET_CKPAIR, &ldp->ckpair_index, &ckp_len);
}

int
socket (int domain, int type, int protocol)
{
  int rv, sock_type = type & ~(SOCK_CLOEXEC | SOCK_NONBLOCK);
  u8 is_nonblocking = type & SOCK_NONBLOCK ? 1 : 0;
  vls_handle_t vlsh;

  ldp_init_check ();

  if (((domain == AF_INET) || (domain == AF_INET6)) &&
      ((sock_type == SOCK_STREAM) || (sock_type == SOCK_DGRAM)))
    {
      u8 proto;
      if (ldp->transparent_tls)
	{
	  proto = VPPCOM_PROTO_TLS;
	}
      else
	proto = ((sock_type == SOCK_DGRAM) ?
		 VPPCOM_PROTO_UDP : VPPCOM_PROTO_TCP);

      LDBG (0, "calling vls_create: proto %u (%s), is_nonblocking %u",
	    proto, vppcom_proto_str (proto), is_nonblocking);

      vlsh = vls_create (proto, is_nonblocking);
      if (vlsh < 0)
	{
	  errno = -vlsh;
	  rv = -1;
	}
      else
	{
	  if (ldp->transparent_tls)
	    {
	      if (assign_cert_key_pair (vlsh) < 0)
		return -1;
	    }
	  rv = ldp_vlsh_to_fd (vlsh);
	}
    }
  else
    {
      LDBG (0, "calling libc_socket");
      rv = libc_socket (domain, type, protocol);
    }

  return rv;
}

/*
 * Create two new sockets, of type TYPE in domain DOMAIN and using
 * protocol PROTOCOL, which are connected to each other, and put file
 * descriptors for them in FDS[0] and FDS[1].  If PROTOCOL is zero,
 * one will be chosen automatically.
 * Returns 0 on success, -1 for errors.
 * */
int
socketpair (int domain, int type, int protocol, int fds[2])
{
  int rv, sock_type = type & ~(SOCK_CLOEXEC | SOCK_NONBLOCK);

  ldp_init_check ();

  if (((domain == AF_INET) || (domain == AF_INET6)) &&
      ((sock_type == SOCK_STREAM) || (sock_type == SOCK_DGRAM)))
    {
      LDBG (0, "LDP-TBD");
      errno = ENOSYS;
      rv = -1;
    }
  else
    {
      LDBG (1, "calling libc_socketpair");
      rv = libc_socketpair (domain, type, protocol, fds);
    }

  return rv;
}

int
bind (int fd, __CONST_SOCKADDR_ARG addr, socklen_t len)
{
  vls_handle_t vlsh;
  int rv;

  ldp_init_check ();

  vlsh = ldp_fd_to_vlsh (fd);
  if (vlsh != VLS_INVALID_HANDLE)
    {
      vppcom_endpt_t ep;

      switch (addr->sa_family)
	{
	case AF_INET:
	  if (len != sizeof (struct sockaddr_in))
	    {
	      LDBG (0, "ERROR: fd %d: vlsh %u: Invalid AF_INET addr len %u!",
		    fd, vlsh, len);
	      errno = EINVAL;
	      rv = -1;
	      goto done;
	    }
	  ep.is_ip4 = VPPCOM_IS_IP4;
	  ep.ip = (u8 *) & ((const struct sockaddr_in *) addr)->sin_addr;
	  ep.port = (u16) ((const struct sockaddr_in *) addr)->sin_port;
	  break;

	case AF_INET6:
	  if (len != sizeof (struct sockaddr_in6))
	    {
	      LDBG (0, "ERROR: fd %d: vlsh %u: Invalid AF_INET6 addr len %u!",
		    fd, vlsh, len);
	      errno = EINVAL;
	      rv = -1;
	      goto done;
	    }
	  ep.is_ip4 = VPPCOM_IS_IP6;
	  ep.ip = (u8 *) & ((const struct sockaddr_in6 *) addr)->sin6_addr;
	  ep.port = (u16) ((const struct sockaddr_in6 *) addr)->sin6_port;
	  break;

	default:
	  LDBG (0, "ERROR: fd %d: vlsh %u: Unsupported address family %u!",
		fd, vlsh, addr->sa_family);
	  errno = EAFNOSUPPORT;
	  rv = -1;
	  goto done;
	}
      LDBG (0, "fd %d: calling vls_bind: vlsh %u, addr %p, len %u", fd, vlsh,
	    addr, len);

      rv = vls_bind (vlsh, &ep);
      if (rv != VPPCOM_OK)
	{
	  errno = -rv;
	  rv = -1;
	}
    }
  else
    {
      LDBG (0, "fd %d: calling libc_bind: addr %p, len %u", fd, addr, len);
      rv = libc_bind (fd, addr, len);
    }

done:
  LDBG (1, "fd %d: returning %d", fd, rv);

  return rv;
}

static inline int
ldp_copy_ep_to_sockaddr (__SOCKADDR_ARG addr, socklen_t * __restrict len,
			 vppcom_endpt_t * ep)
{
  int rv = 0;
  int sa_len, copy_len;

  ldp_init_check ();

  if (addr && len && ep)
    {
      addr->sa_family = (ep->is_ip4 == VPPCOM_IS_IP4) ? AF_INET : AF_INET6;
      switch (addr->sa_family)
	{
	case AF_INET:
	  ((struct sockaddr_in *) addr)->sin_port = ep->port;
	  if (*len > sizeof (struct sockaddr_in))
	    *len = sizeof (struct sockaddr_in);
	  sa_len = sizeof (struct sockaddr_in) - sizeof (struct in_addr);
	  copy_len = *len - sa_len;
	  if (copy_len > 0)
	    memcpy (&((struct sockaddr_in *) addr)->sin_addr, ep->ip,
		    copy_len);
	  break;

	case AF_INET6:
	  ((struct sockaddr_in6 *) addr)->sin6_port = ep->port;
	  if (*len > sizeof (struct sockaddr_in6))
	    *len = sizeof (struct sockaddr_in6);
	  sa_len = sizeof (struct sockaddr_in6) - sizeof (struct in6_addr);
	  copy_len = *len - sa_len;
	  if (copy_len > 0)
	    memcpy (((struct sockaddr_in6 *) addr)->sin6_addr.
		    __in6_u.__u6_addr8, ep->ip, copy_len);
	  break;

	default:
	  /* Not possible */
	  rv = -EAFNOSUPPORT;
	  break;
	}
    }
  return rv;
}

int
getsockname (int fd, __SOCKADDR_ARG addr, socklen_t * __restrict len)
{
  vls_handle_t vlsh;
  int rv;

  ldp_init_check ();

  vlsh = ldp_fd_to_vlsh (fd);
  if (vlsh != VLS_INVALID_HANDLE)
    {
      vppcom_endpt_t ep;
      u8 addr_buf[sizeof (struct in6_addr)];
      u32 size = sizeof (ep);

      ep.ip = addr_buf;

      rv = vls_attr (vlsh, VPPCOM_ATTR_GET_LCL_ADDR, &ep, &size);
      if (rv != VPPCOM_OK)
	{
	  errno = -rv;
	  rv = -1;
	}
      else
	{
	  rv = ldp_copy_ep_to_sockaddr (addr, len, &ep);
	  if (rv != VPPCOM_OK)
	    {
	      errno = -rv;
	      rv = -1;
	    }
	}
    }
  else
    {
      rv = libc_getsockname (fd, addr, len);
    }

  return rv;
}

int
connect (int fd, __CONST_SOCKADDR_ARG addr, socklen_t len)
{
  vls_handle_t vlsh;
  int rv;

  ldp_init_check ();

  if (!addr)
    {
      LDBG (0, "ERROR: fd %d: NULL addr, len %u", fd, len);
      errno = EINVAL;
      rv = -1;
      goto done;
    }

  vlsh = ldp_fd_to_vlsh (fd);
  if (vlsh != VLS_INVALID_HANDLE)
    {
      vppcom_endpt_t ep;

      switch (addr->sa_family)
	{
	case AF_INET:
	  if (len != sizeof (struct sockaddr_in))
	    {
	      LDBG (0, "fd %d: ERROR vlsh %u: Invalid AF_INET addr len %u!",
		    fd, vlsh, len);
	      errno = EINVAL;
	      rv = -1;
	      goto done;
	    }
	  ep.is_ip4 = VPPCOM_IS_IP4;
	  ep.ip = (u8 *) & ((const struct sockaddr_in *) addr)->sin_addr;
	  ep.port = (u16) ((const struct sockaddr_in *) addr)->sin_port;
	  break;

	case AF_INET6:
	  if (len != sizeof (struct sockaddr_in6))
	    {
	      LDBG (0, "fd %d: ERROR vlsh %u: Invalid AF_INET6 addr len %u!",
		    fd, vlsh, len);
	      errno = EINVAL;
	      rv = -1;
	      goto done;
	    }
	  ep.is_ip4 = VPPCOM_IS_IP6;
	  ep.ip = (u8 *) & ((const struct sockaddr_in6 *) addr)->sin6_addr;
	  ep.port = (u16) ((const struct sockaddr_in6 *) addr)->sin6_port;
	  break;

	default:
	  LDBG (0, "fd %d: ERROR vlsh %u: Unsupported address family %u!",
		fd, vlsh, addr->sa_family);
	  errno = EAFNOSUPPORT;
	  rv = -1;
	  goto done;
	}
      LDBG (0, "fd %d: calling vls_connect(): vlsh %u addr %p len %u", fd,
	    vlsh, addr, len);

      rv = vls_connect (vlsh, &ep);
      if (rv != VPPCOM_OK)
	{
	  errno = -rv;
	  rv = -1;
	}
    }
  else
    {
      LDBG (0, "fd %d: calling libc_connect(): addr %p, len %u",
	    fd, addr, len);

      rv = libc_connect (fd, addr, len);
    }

done:
  LDBG (1, "fd %d: returning %d (0x%x)", fd, rv, rv);
  return rv;
}

int
getpeername (int fd, __SOCKADDR_ARG addr, socklen_t * __restrict len)
{
  vls_handle_t vlsh;
  int rv;

  ldp_init_check ();

  vlsh = ldp_fd_to_vlsh (fd);
  if (vlsh != VLS_INVALID_HANDLE)
    {
      vppcom_endpt_t ep;
      u8 addr_buf[sizeof (struct in6_addr)];
      u32 size = sizeof (ep);

      ep.ip = addr_buf;
      rv = vls_attr (vlsh, VPPCOM_ATTR_GET_PEER_ADDR, &ep, &size);
      if (rv != VPPCOM_OK)
	{
	  errno = -rv;
	  rv = -1;
	}
      else
	{
	  rv = ldp_copy_ep_to_sockaddr (addr, len, &ep);
	  if (rv != VPPCOM_OK)
	    {
	      errno = -rv;
	      rv = -1;
	    }
	}
    }
  else
    {
      rv = libc_getpeername (fd, addr, len);
    }

  return rv;
}

ssize_t
send (int fd, const void *buf, size_t n, int flags)
{
  vls_handle_t vlsh = ldp_fd_to_vlsh (fd);
  ssize_t size;

  ldp_init_check ();

  if (vlsh != VLS_INVALID_HANDLE)
    {
      size = vls_sendto (vlsh, (void *) buf, n, flags, NULL);
      if (size < VPPCOM_OK)
	{
	  errno = -size;
	  size = -1;
	}
    }
  else
    {
      size = libc_send (fd, buf, n, flags);
    }

  return size;
}

ssize_t
sendfile (int out_fd, int in_fd, off_t * offset, size_t len)
{
  ldp_worker_ctx_t *ldpw = ldp_worker_get_current ();
  vls_handle_t vlsh;
  ssize_t size = 0;

  ldp_init_check ();

  vlsh = ldp_fd_to_vlsh (out_fd);
  if (vlsh != VLS_INVALID_HANDLE)
    {
      int rv;
      ssize_t results = 0;
      size_t n_bytes_left = len;
      size_t bytes_to_read;
      int nbytes;
      u8 eagain = 0;
      u32 flags, flags_len = sizeof (flags);

      rv = vls_attr (vlsh, VPPCOM_ATTR_GET_FLAGS, &flags, &flags_len);
      if (PREDICT_FALSE (rv != VPPCOM_OK))
	{
	  LDBG (0, "ERROR: out fd %d: vls_attr: vlsh %u, returned %d (%s)!",
		out_fd, vlsh, rv, vppcom_retval_str (rv));

	  vec_reset_length (ldpw->io_buffer);
	  errno = -rv;
	  size = -1;
	  goto done;
	}

      if (offset)
	{
	  off_t off = lseek (in_fd, *offset, SEEK_SET);
	  if (PREDICT_FALSE (off == -1))
	    {
	      size = -1;
	      goto done;
	    }

	  ASSERT (off == *offset);
	}

      do
	{
	  size = vls_attr (vlsh, VPPCOM_ATTR_GET_NWRITE, 0, 0);
	  if (size < 0)
	    {
	      LDBG (0, "ERROR: fd %d: vls_attr: vlsh %u returned %ld (%s)!",
		    out_fd, vlsh, size, vppcom_retval_str (size));
	      vec_reset_length (ldpw->io_buffer);
	      errno = -size;
	      size = -1;
	      goto done;
	    }

	  bytes_to_read = size;
	  if (bytes_to_read == 0)
	    {
	      if (flags & O_NONBLOCK)
		{
		  if (!results)
		    eagain = 1;
		  goto update_offset;
		}
	      else
		continue;
	    }
	  bytes_to_read = clib_min (n_bytes_left, bytes_to_read);
	  vec_validate (ldpw->io_buffer, bytes_to_read);
	  nbytes = libc_read (in_fd, ldpw->io_buffer, bytes_to_read);
	  if (nbytes < 0)
	    {
	      if (results == 0)
		{
		  vec_reset_length (ldpw->io_buffer);
		  size = -1;
		  goto done;
		}
	      goto update_offset;
	    }

	  size = vls_write (vlsh, ldpw->io_buffer, nbytes);
	  if (size < 0)
	    {
	      if (size == VPPCOM_EAGAIN)
		{
		  if (flags & O_NONBLOCK)
		    {
		      if (!results)
			eagain = 1;
		      goto update_offset;
		    }
		  else
		    continue;
		}
	      if (results == 0)
		{
		  vec_reset_length (ldpw->io_buffer);
		  errno = -size;
		  size = -1;
		  goto done;
		}
	      goto update_offset;
	    }

	  results += nbytes;
	  ASSERT (n_bytes_left >= nbytes);
	  n_bytes_left = n_bytes_left - nbytes;
	}
      while (n_bytes_left > 0);

    update_offset:
      vec_reset_length (ldpw->io_buffer);
      if (offset)
	{
	  off_t off = lseek (in_fd, *offset, SEEK_SET);
	  if (PREDICT_FALSE (off == -1))
	    {
	      size = -1;
	      goto done;
	    }

	  ASSERT (off == *offset);
	  *offset += results + 1;
	}
      if (eagain)
	{
	  errno = EAGAIN;
	  size = -1;
	}
      else
	size = results;
    }
  else
    {
      size = libc_sendfile (out_fd, in_fd, offset, len);
    }

done:
  return size;
}

ssize_t
sendfile64 (int out_fd, int in_fd, off_t * offset, size_t len)
{
  return sendfile (out_fd, in_fd, offset, len);
}

ssize_t
recv (int fd, void *buf, size_t n, int flags)
{
  vls_handle_t vlsh;
  ssize_t size;

  ldp_init_check ();

  vlsh = ldp_fd_to_vlsh (fd);
  if (vlsh != VLS_INVALID_HANDLE)
    {
      size = vls_recvfrom (vlsh, buf, n, flags, NULL);
      if (size < 0)
	{
	  errno = -size;
	  size = -1;
	}
    }
  else
    {
      size = libc_recv (fd, buf, n, flags);
    }

  return size;
}

ssize_t
__recv_chk (int fd, void *buf, size_t n, size_t buflen, int flags)
{
  if (n > buflen)
    return -1;

  return recv (fd, buf, n, flags);
}

static int
ldp_vls_sendo (vls_handle_t vlsh, const void *buf, size_t n, int flags,
	       __CONST_SOCKADDR_ARG addr, socklen_t addr_len)
{
  vppcom_endpt_t *ep = 0;
  vppcom_endpt_t _ep;

  if (addr)
    {
      ep = &_ep;
      switch (addr->sa_family)
	{
	case AF_INET:
	  ep->is_ip4 = VPPCOM_IS_IP4;
	  ep->ip =
	    (uint8_t *) & ((const struct sockaddr_in *) addr)->sin_addr;
	  ep->port = (uint16_t) ((const struct sockaddr_in *) addr)->sin_port;
	  break;

	case AF_INET6:
	  ep->is_ip4 = VPPCOM_IS_IP6;
	  ep->ip =
	    (uint8_t *) & ((const struct sockaddr_in6 *) addr)->sin6_addr;
	  ep->port =
	    (uint16_t) ((const struct sockaddr_in6 *) addr)->sin6_port;
	  break;

	default:
	  return EAFNOSUPPORT;
	}
    }

  return vls_sendto (vlsh, (void *) buf, n, flags, ep);
}

static int
ldp_vls_recvfrom (vls_handle_t vlsh, void *__restrict buf, size_t n,
		  int flags, __SOCKADDR_ARG addr,
		  socklen_t * __restrict addr_len)
{
  u8 src_addr[sizeof (struct sockaddr_in6)];
  vppcom_endpt_t ep;
  ssize_t size;
  int rv;

  if (addr)
    {
      ep.ip = src_addr;
      size = vls_recvfrom (vlsh, buf, n, flags, &ep);

      if (size > 0)
	{
	  rv = ldp_copy_ep_to_sockaddr (addr, addr_len, &ep);
	  if (rv < 0)
	    size = rv;
	}
    }
  else
    size = vls_recvfrom (vlsh, buf, n, flags, NULL);

  return size;
}

ssize_t
sendto (int fd, const void *buf, size_t n, int flags,
	__CONST_SOCKADDR_ARG addr, socklen_t addr_len)
{
  vls_handle_t vlsh;
  ssize_t size;

  ldp_init_check ();

  vlsh = ldp_fd_to_vlsh (fd);
  if (vlsh != VLS_INVALID_HANDLE)
    {
      size = ldp_vls_sendo (vlsh, buf, n, flags, addr, addr_len);
      if (size < 0)
	{
	  errno = -size;
	  size = -1;
	}
    }
  else
    {
      size = libc_sendto (fd, buf, n, flags, addr, addr_len);
    }

  return size;
}

ssize_t
recvfrom (int fd, void *__restrict buf, size_t n, int flags,
	  __SOCKADDR_ARG addr, socklen_t * __restrict addr_len)
{
  vls_handle_t vlsh;
  ssize_t size;

  ldp_init_check ();

  vlsh = ldp_fd_to_vlsh (fd);
  if (vlsh != VLS_INVALID_HANDLE)
    {
      size = ldp_vls_recvfrom (vlsh, buf, n, flags, addr, addr_len);
      if (size < 0)
	{
	  errno = -size;
	  size = -1;
	}
    }
  else
    {
      size = libc_recvfrom (fd, buf, n, flags, addr, addr_len);
    }

  return size;
}

ssize_t
sendmsg (int fd, const struct msghdr * msg, int flags)
{
  vls_handle_t vlsh;
  ssize_t size;

  ldp_init_check ();

  vlsh = ldp_fd_to_vlsh (fd);
  if (vlsh != VLS_INVALID_HANDLE)
    {
      struct iovec *iov = msg->msg_iov;
      ssize_t total = 0;
      int i, rv;

      for (i = 0; i < msg->msg_iovlen; ++i)
	{
	  rv = ldp_vls_sendo (vlsh, iov[i].iov_base, iov[i].iov_len, flags,
			      msg->msg_name, msg->msg_namelen);
	  if (rv < 0)
	    break;
	  else
	    {
	      total += rv;
	      if (rv < iov[i].iov_len)
		break;
	    }
	}

      if (rv < 0 && total == 0)
	{
	  errno = -rv;
	  size = -1;
	}
      else
	size = total;
    }
  else
    {
      size = libc_sendmsg (fd, msg, flags);
    }

  return size;
}

#ifdef USE_GNU
int
sendmmsg (int fd, struct mmsghdr *vmessages, unsigned int vlen, int flags)
{
  ssize_t size;
  const char *func_str;
  u32 sh = ldp_fd_to_vlsh (fd);

  ldp_init_check ();

  if (sh != VLS_INVALID_HANDLE)
    {
      clib_warning ("LDP<%d>: LDP-TBD", getpid ());
      errno = ENOSYS;
      size = -1;
    }
  else
    {
      func_str = "libc_sendmmsg";

      if (LDP_DEBUG > 2)
	clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): "
		      "vmessages %p, vlen %u, flags 0x%x",
		      getpid (), fd, fd, func_str, vmessages, vlen, flags);

      size = libc_sendmmsg (fd, vmessages, vlen, flags);
    }

  if (LDP_DEBUG > 2)
    {
      if (size < 0)
	{
	  int errno_val = errno;
	  perror (func_str);
	  clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! "
			"rv %d, errno = %d", getpid (), fd, fd,
			func_str, size, errno_val);
	  errno = errno_val;
	}
      else
	clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)",
		      getpid (), fd, fd, size, size);
    }
  return size;
}
#endif

ssize_t
recvmsg (int fd, struct msghdr * msg, int flags)
{
  vls_handle_t vlsh;
  ssize_t size;

  ldp_init_check ();

  vlsh = ldp_fd_to_vlsh (fd);
  if (vlsh != VLS_INVALID_HANDLE)
    {
      struct iovec *iov = msg->msg_iov;
      ssize_t max_deq, total = 0;
      int i, rv;

      max_deq = vls_attr (vlsh, VPPCOM_ATTR_GET_NREAD, 0, 0);
      if (!max_deq)
	return 0;

      for (i = 0; i < msg->msg_iovlen; i++)
	{
	  rv = ldp_vls_recvfrom (vlsh, iov[i].iov_base, iov[i].iov_len, flags,
				 (i == 0 ? msg->msg_name : NULL),
				 (i == 0 ? &msg->msg_namelen : NULL));
	  if (rv <= 0)
	    break;
	  else
	    {
	      total += rv;
	      if (rv < iov[i].iov_len)
		break;
	    }
	  if (total >= max_deq)
	    break;
	}

      if (rv < 0 && total == 0)
	{
	  errno = -rv;
	  size = -1;
	}
      else
	size = total;
    }
  else
    {
      size = libc_recvmsg (fd, msg, flags);
    }

  return size;
}

#ifdef USE_GNU
int
recvmmsg (int fd, struct mmsghdr *vmessages,
	  unsigned int vlen, int flags, struct timespec *tmo)
{
  ssize_t size;
  const char *func_str;
  u32 sh = ldp_fd_to_vlsh (fd);

  ldp_init_check ();

  if (sh != VLS_INVALID_HANDLE)
    {
      clib_warning ("LDP<%d>: LDP-TBD", getpid ());
      errno = ENOSYS;
      size = -1;
    }
  else
    {
      func_str = "libc_recvmmsg";

      if (LDP_DEBUG > 2)
	clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): "
		      "vmessages %p, vlen %u, flags 0x%x, tmo %p",
		      getpid (), fd, fd, func_str, vmessages, vlen,
		      flags, tmo);

      size = libc_recvmmsg (fd, vmessages, vlen, flags, tmo);
    }

  if (LDP_DEBUG > 2)
    {
      if (size < 0)
	{
	  int errno_val = errno;
	  perror (func_str);
	  clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! "
			"rv %d, errno = %d", getpid (), fd, fd,
			func_str, size, errno_val);
	  errno = errno_val;
	}
      else
	clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)",
		      getpid (), fd, fd, size, size);
    }
  return size;
}
#endif

int
getsockopt (int fd, int level, int optname,
	    void *__restrict optval, socklen_t * __restrict optlen)
{
  vls_handle_t vlsh;
  int rv;

  ldp_init_check ();

  vlsh = ldp_fd_to_vlsh (fd);
  if (vlsh != VLS_INVALID_HANDLE)
    {
      rv = -EOPNOTSUPP;

      switch (level)
	{
	case SOL_TCP:
	  switch (optname)
	    {
	    case TCP_NODELAY:
	      rv = vls_attr (vlsh, VPPCOM_ATTR_GET_TCP_NODELAY,
			     optval, optlen);
	      break;
	    case TCP_MAXSEG:
	      rv = vls_attr (vlsh, VPPCOM_ATTR_GET_TCP_USER_MSS,
			     optval, optlen);
	      break;
	    case TCP_KEEPIDLE:
	      rv = vls_attr (vlsh, VPPCOM_ATTR_GET_TCP_KEEPIDLE,
			     optval, optlen);
	      break;
	    case TCP_KEEPINTVL:
	      rv = vls_attr (vlsh, VPPCOM_ATTR_GET_TCP_KEEPINTVL,
			     optval, optlen);
	      break;
	    case TCP_INFO:
	      if (optval && optlen && (*optlen == sizeof (struct tcp_info)))
		{
		  LDBG (1, "fd %d: vlsh %u SOL_TCP, TCP_INFO, optval %p, "
			"optlen %d: #LDP-NOP#", fd, vlsh, optval, *optlen);
		  memset (optval, 0, *optlen);
		  rv = VPPCOM_OK;
		}
	      else
		rv = -EFAULT;
	      break;
	    case TCP_CONGESTION:
	      *optlen = strlen ("cubic");
	      strncpy (optval, "cubic", *optlen + 1);
	      rv = 0;
	      break;
	    default:
	      LDBG (0, "ERROR: fd %d: getsockopt SOL_TCP: sid %u, "
		    "optname %d unsupported!", fd, vlsh, optname);
	      break;
	    }
	  break;
	case SOL_IPV6:
	  switch (optname)
	    {
	    case IPV6_V6ONLY:
	      rv = vls_attr (vlsh, VPPCOM_ATTR_GET_V6ONLY, optval, optlen);
	      break;
	    default:
	      LDBG (0, "ERROR: fd %d: getsockopt SOL_IPV6: vlsh %u "
		    "optname %d unsupported!", fd, vlsh, optname);
	      break;
	    }
	  break;
	case SOL_SOCKET:
	  switch (optname)
	    {
	    case SO_ACCEPTCONN:
	      rv = vls_attr (vlsh, VPPCOM_ATTR_GET_LISTEN, optval, optlen);
	      break;
	    case SO_KEEPALIVE:
	      rv = vls_attr (vlsh, VPPCOM_ATTR_GET_KEEPALIVE, optval, optlen);
	      break;
	    case SO_PROTOCOL:
	      rv = vls_attr (vlsh, VPPCOM_ATTR_GET_PROTOCOL, optval, optlen);
	      *(int *) optval = *(int *) optval ? SOCK_DGRAM : SOCK_STREAM;
	      break;
	    case SO_SNDBUF:
	      rv = vls_attr (vlsh, VPPCOM_ATTR_GET_TX_FIFO_LEN,
			     optval, optlen);
	      break;
	    case SO_RCVBUF:
	      rv = vls_attr (vlsh, VPPCOM_ATTR_GET_RX_FIFO_LEN,
			     optval, optlen);
	      break;
	    case SO_REUSEADDR:
	      rv = vls_attr (vlsh, VPPCOM_ATTR_GET_REUSEADDR, optval, optlen);
	      break;
	    case SO_REUSEPORT:
	      rv = vls_attr (vlsh, VPPCOM_ATTR_GET_REUSEPORT, optval, optlen);
	      break;
	    case SO_BROADCAST:
	      rv = vls_attr (vlsh, VPPCOM_ATTR_GET_BROADCAST, optval, optlen);
	      break;
	    case SO_DOMAIN:
	      rv = vls_attr (vlsh, VPPCOM_ATTR_GET_DOMAIN, optval, optlen);
	      break;
	    case SO_ERROR:
	      rv = vls_attr (vlsh, VPPCOM_ATTR_GET_ERROR, optval, optlen);
	      break;
	    case SO_BINDTODEVICE:
	      rv = 0;
	      break;
	    default:
	      LDBG (0, "ERROR: fd %d: getsockopt SOL_SOCKET: vlsh %u "
		    "optname %d unsupported!", fd, vlsh, optname);
	      break;
	    }
	  break;
	default:
	  break;
	}

      if (rv != VPPCOM_OK)
	{
	  errno = -rv;
	  rv = -1;
	}
    }
  else
    {
      rv = libc_getsockopt (fd, level, optname, optval, optlen);
    }

  return rv;
}

int
setsockopt (int fd, int level, int optname,
	    const void *optval, socklen_t optlen)
{
  vls_handle_t vlsh;
  int rv;

  ldp_init_check ();

  vlsh = ldp_fd_to_vlsh (fd);
  if (vlsh != VLS_INVALID_HANDLE)
    {
      rv = -EOPNOTSUPP;

      switch (level)
	{
	case SOL_TCP:
	  switch (optname)
	    {
	    case TCP_NODELAY:
	      rv = vls_attr (vlsh, VPPCOM_ATTR_SET_TCP_NODELAY,
			     (void *) optval, &optlen);
	      break;
	    case TCP_MAXSEG:
	      rv = vls_attr (vlsh, VPPCOM_ATTR_SET_TCP_USER_MSS,
			     (void *) optval, &optlen);
	      break;
	    case TCP_KEEPIDLE:
	      rv = vls_attr (vlsh, VPPCOM_ATTR_SET_TCP_KEEPIDLE,
			     (void *) optval, &optlen);
	      break;
	    case TCP_KEEPINTVL:
	      rv = vls_attr (vlsh, VPPCOM_ATTR_SET_TCP_KEEPINTVL,
			     (void *) optval, &optlen);
	      break;
	    case TCP_CONGESTION:
	    case TCP_CORK:
	      /* Ignore */
	      rv = 0;
	      break;
	    default:
	      LDBG (0, "ERROR: fd %d: setsockopt() SOL_TCP: vlsh %u"
		    "optname %d unsupported!", fd, vlsh, optname);
	      break;
	    }
	  break;
	case SOL_IPV6:
	  switch (optname)
	    {
	    case IPV6_V6ONLY:
	      rv = vls_attr (vlsh, VPPCOM_ATTR_SET_V6ONLY,
			     (void *) optval, &optlen);
	      break;
	    default:
	      LDBG (0, "ERROR: fd %d: setsockopt SOL_IPV6: vlsh %u"
		    "optname %d unsupported!", fd, vlsh, optname);
	      break;
	    }
	  break;
	case SOL_SOCKET:
	  switch (optname)
	    {
	    case SO_KEEPALIVE:
	      rv = vls_attr (vlsh, VPPCOM_ATTR_SET_KEEPALIVE,
			     (void *) optval, &optlen);
	      break;
	    case SO_REUSEADDR:
	      rv = vls_attr (vlsh, VPPCOM_ATTR_SET_REUSEADDR,
			     (void *) optval, &optlen);
	      break;
	    case SO_REUSEPORT:
	      rv = vls_attr (vlsh, VPPCOM_ATTR_SET_REUSEPORT, (void *) optval,
			     &optlen);
	      break;
	    case SO_BROADCAST:
	      rv = vls_attr (vlsh, VPPCOM_ATTR_SET_BROADCAST,
			     (void *) optval, &optlen);
	      break;
	    case SO_LINGER:
	      rv = 0;
	      break;
	    default:
	      LDBG (0, "ERROR: fd %d: setsockopt SOL_SOCKET: vlsh %u "
		    "optname %d unsupported!", fd, vlsh, optname);
	      break;
	    }
	  break;
	default:
	  break;
	}

      if (rv != VPPCOM_OK)
	{
	  errno = -rv;
	  rv = -1;
	}
    }
  else
    {
      rv = libc_setsockopt (fd, level, optname, optval, optlen);
    }

  return rv;
}

int
listen (int fd, int n)
{
  vls_handle_t vlsh;
  int rv;

  ldp_init_check ();

  vlsh = ldp_fd_to_vlsh (fd);
  if (vlsh != VLS_INVALID_HANDLE)
    {
      LDBG (0, "fd %d: calling vls_listen: vlsh %u, n %d", fd, vlsh, n);

      rv = vls_listen (vlsh, n);
      if (rv != VPPCOM_OK)
	{
	  errno = -rv;
	  rv = -1;
	}
    }
  else
    {
      LDBG (0, "fd %d: calling libc_listen(): n %d", fd, n);
      rv = libc_listen (fd, n);
    }

  LDBG (1, "fd %d: returning %d", fd, rv);
  return rv;
}

static inline int
ldp_accept4 (int listen_fd, __SOCKADDR_ARG addr,
	     socklen_t * __restrict addr_len, int flags)
{
  vls_handle_t listen_vlsh, accept_vlsh;
  int rv;

  ldp_init_check ();

  listen_vlsh = ldp_fd_to_vlsh (listen_fd);
  if (listen_vlsh != VLS_INVALID_HANDLE)
    {
      vppcom_endpt_t ep;
      u8 src_addr[sizeof (struct sockaddr_in6)];
      memset (&ep, 0, sizeof (ep));
      ep.ip = src_addr;

      LDBG (0, "listen fd %d: calling vppcom_session_accept: listen sid %u,"
	    " ep %p, flags 0x%x", listen_fd, listen_vlsh, &ep, flags);

      accept_vlsh = vls_accept (listen_vlsh, &ep, flags);
      if (accept_vlsh < 0)
	{
	  errno = -accept_vlsh;
	  rv = -1;
	}
      else
	{
	  rv = ldp_copy_ep_to_sockaddr (addr, addr_len, &ep);
	  if (rv != VPPCOM_OK)
	    {
	      (void) vls_close (accept_vlsh);
	      errno = -rv;
	      rv = -1;
	    }
	  else
	    {
	      rv = ldp_vlsh_to_fd (accept_vlsh);
	    }
	}
    }
  else
    {
      LDBG (0, "listen fd %d: calling libc_accept4(): addr %p, addr_len %p,"
	    " flags 0x%x", listen_fd, addr, addr_len, flags);

      rv = libc_accept4 (listen_fd, addr, addr_len, flags);
    }

  LDBG (1, "listen fd %d: accept returning %d", listen_fd, rv);

  return rv;
}

int
accept4 (int fd, __SOCKADDR_ARG addr, socklen_t * __restrict addr_len,
	 int flags)
{
  return ldp_accept4 (fd, addr, addr_len, flags);
}

int
accept (int fd, __SOCKADDR_ARG addr, socklen_t * __restrict addr_len)
{
  return ldp_accept4 (fd, addr, addr_len, 0);
}

int
shutdown (int fd, int how)
{
  vls_handle_t vlsh;
  int rv = 0;

  ldp_init_check ();

  vlsh = ldp_fd_to_vlsh (fd);
  if (vlsh != VLS_INVALID_HANDLE)
    {
      LDBG (0, "called shutdown: fd %u vlsh %u how %d", fd, vlsh, how);
      rv = vls_shutdown (vlsh, how);
    }
  else
    {
      LDBG (0, "fd %d: calling libc_shutdown: how %d", fd, how);
      rv = libc_shutdown (fd, how);
    }

  return rv;
}

int
epoll_create1 (int flags)
{
  ldp_worker_ctx_t *ldpw = ldp_worker_get_current ();
  vls_handle_t vlsh;
  int rv;

  ldp_init_check ();

  if (ldp->vcl_needs_real_epoll || vls_use_real_epoll ())
    {
      /* Make sure workers have been allocated */
      if (!ldp->workers)
	{
	  ldp_alloc_workers ();
	  ldpw = ldp_worker_get_current ();
	}
      rv = libc_epoll_create1 (flags);
      ldp->vcl_needs_real_epoll = 0;
      ldpw->vcl_mq_epfd = rv;
      LDBG (0, "created vcl epfd %u", rv);
      return rv;
    }

  vlsh = vls_epoll_create ();
  if (PREDICT_FALSE (vlsh == VLS_INVALID_HANDLE))
    {
      errno = -vlsh;
      rv = -1;
    }
  else
    {
      rv = ldp_vlsh_to_fd (vlsh);
    }
  LDBG (0, "epoll_create epfd %u vlsh %u", rv, vlsh);
  return rv;
}

int
epoll_create (int size)
{
  return epoll_create1 (0);
}

int
epoll_ctl (int epfd, int op, int fd, struct epoll_event *event)
{
  vls_handle_t vep_vlsh, vlsh;
  int rv;

  ldp_init_check ();

  vep_vlsh = ldp_fd_to_vlsh (epfd);
  if (PREDICT_FALSE (vep_vlsh == VLS_INVALID_HANDLE))
    {
      /* The LDP epoll_create1 always creates VCL epfd's.
       * The app should never have a kernel base epoll fd unless it
       * was acquired outside of the LD_PRELOAD process context.
       * In any case, if we get one, punt it to libc_epoll_ctl.
       */
      LDBG (1, "epfd %d: calling libc_epoll_ctl: op %d, fd %d"
	    " event %p", epfd, op, fd, event);

      rv = libc_epoll_ctl (epfd, op, fd, event);
      goto done;
    }

  vlsh = ldp_fd_to_vlsh (fd);

  LDBG (0, "epfd %d ep_vlsh %d, fd %u vlsh %d, op %u", epfd, vep_vlsh, fd,
	vlsh, op);

  if (vlsh != VLS_INVALID_HANDLE)
    {
      LDBG (1, "epfd %d: calling vls_epoll_ctl: ep_vlsh %d op %d, vlsh %u,"
	    " event %p", epfd, vep_vlsh, op, vlsh, event);

      rv = vls_epoll_ctl (vep_vlsh, op, vlsh, event);
      if (rv != VPPCOM_OK)
	{
	  errno = -rv;
	  rv = -1;
	}
    }
  else
    {
      int libc_epfd;
      u32 size = sizeof (epfd);

      libc_epfd = vls_attr (vep_vlsh, VPPCOM_ATTR_GET_LIBC_EPFD, 0, 0);
      if (!libc_epfd)
	{
	  LDBG (1, "epfd %d, vep_vlsh %d calling libc_epoll_create1: "
		"EPOLL_CLOEXEC", epfd, vep_vlsh);

	  libc_epfd = libc_epoll_create1 (EPOLL_CLOEXEC);
	  if (libc_epfd < 0)
	    {
	      rv = libc_epfd;
	      goto done;
	    }

	  rv = vls_attr (vep_vlsh, VPPCOM_ATTR_SET_LIBC_EPFD, &libc_epfd,
			 &size);
	  if (rv < 0)
	    {
	      errno = -rv;
	      rv = -1;
	      goto done;
	    }
	}
      else if (PREDICT_FALSE (libc_epfd < 0))
	{
	  errno = -epfd;
	  rv = -1;
	  goto done;
	}

      LDBG (1, "epfd %d: calling libc_epoll_ctl: libc_epfd %d, op %d, fd %d,"
	    " event %p", epfd, libc_epfd, op, fd, event);

      rv = libc_epoll_ctl (libc_epfd, op, fd, event);
    }

done:
  return rv;
}

static inline int
ldp_epoll_pwait (int epfd, struct epoll_event *events, int maxevents,
		 int timeout, const sigset_t * sigmask)
{
  ldp_worker_ctx_t *ldpw = ldp_worker_get_current ();
  double time_to_wait = (double) 0, max_time;
  int libc_epfd, rv = 0;
  vls_handle_t ep_vlsh;

  ldp_init_check ();

  if (PREDICT_FALSE (!events || (timeout < -1)))
    {
      errno = EFAULT;
      return -1;
    }

  if (epfd == ldpw->vcl_mq_epfd)
    return libc_epoll_pwait (epfd, events, maxevents, timeout, sigmask);

  ep_vlsh = ldp_fd_to_vlsh (epfd);
  if (PREDICT_FALSE (ep_vlsh == VLS_INVALID_HANDLE))
    {
      LDBG (0, "epfd %d: bad ep_vlsh %d!", epfd, ep_vlsh);
      errno = EBADFD;
      return -1;
    }

  if (PREDICT_FALSE (ldpw->clib_time.init_cpu_time == 0))
    clib_time_init (&ldpw->clib_time);
  time_to_wait = ((timeout >= 0) ? (double) timeout / 1000 : 0);
  max_time = clib_time_now (&ldpw->clib_time) + time_to_wait;

  libc_epfd = vls_attr (ep_vlsh, VPPCOM_ATTR_GET_LIBC_EPFD, 0, 0);
  if (PREDICT_FALSE (libc_epfd < 0))
    {
      errno = -libc_epfd;
      rv = -1;
      goto done;
    }

  LDBG (2, "epfd %d: vep_idx %d, libc_epfd %d, events %p, maxevents %d, "
	"timeout %d, sigmask %p: time_to_wait %.02f", epfd, ep_vlsh,
	libc_epfd, events, maxevents, timeout, sigmask, time_to_wait);
  do
    {
      if (!ldpw->epoll_wait_vcl)
	{
	  rv = vls_epoll_wait (ep_vlsh, events, maxevents, 0);
	  if (rv > 0)
	    {
	      ldpw->epoll_wait_vcl = 1;
	      goto done;
	    }
	  else if (rv < 0)
	    {
	      errno = -rv;
	      rv = -1;
	      goto done;
	    }
	}
      else
	ldpw->epoll_wait_vcl = 0;

      if (libc_epfd > 0)
	{
	  rv = libc_epoll_pwait (libc_epfd, events, maxevents, 0, sigmask);
	  if (rv != 0)
	    goto done;
	}
    }
  while ((timeout == -1) || (clib_time_now (&ldpw->clib_time) < max_time));

done:
  return rv;
}

static inline int
ldp_epoll_pwait_eventfd (int epfd, struct epoll_event *events,
			 int maxevents, int timeout, const sigset_t * sigmask)
{
  ldp_worker_ctx_t *ldpw;
  int libc_epfd, rv = 0, num_ev;
  vls_handle_t ep_vlsh;

  ldp_init_check ();

  if (PREDICT_FALSE (!events || (timeout < -1)))
    {
      errno = EFAULT;
      return -1;
    }

  /* Make sure the vcl worker is valid. Could be that epoll fd was created on
   * one thread but it is now used on another */
  if (PREDICT_FALSE (vppcom_worker_index () == ~0))
    vls_register_vcl_worker ();

  ldpw = ldp_worker_get_current ();
  if (epfd == ldpw->vcl_mq_epfd)
    return libc_epoll_pwait (epfd, events, maxevents, timeout, sigmask);

  ep_vlsh = ldp_fd_to_vlsh (epfd);
  if (PREDICT_FALSE (ep_vlsh == VLS_INVALID_HANDLE))
    {
      LDBG (0, "epfd %d: bad ep_vlsh %d!", epfd, ep_vlsh);
      errno = EBADFD;
      return -1;
    }

  libc_epfd = vls_attr (ep_vlsh, VPPCOM_ATTR_GET_LIBC_EPFD, 0, 0);
  if (PREDICT_FALSE (!libc_epfd))
    {
      u32 size = sizeof (epfd);

      LDBG (1, "epfd %d, vep_vlsh %d calling libc_epoll_create1: "
	    "EPOLL_CLOEXEC", epfd, ep_vlsh);
      libc_epfd = libc_epoll_create1 (EPOLL_CLOEXEC);
      if (libc_epfd < 0)
	{
	  rv = libc_epfd;
	  goto done;
	}

      rv = vls_attr (ep_vlsh, VPPCOM_ATTR_SET_LIBC_EPFD, &libc_epfd, &size);
      if (rv < 0)
	{
	  errno = -rv;
	  rv = -1;
	  goto done;
	}
    }
  if (PREDICT_FALSE (libc_epfd <= 0))
    {
      errno = -libc_epfd;
      rv = -1;
      goto done;
    }

  if (PREDICT_FALSE (!ldpw->mq_epfd_added))
    {
      struct epoll_event e = { 0 };
      e.events = EPOLLIN;
      e.data.fd = ldpw->vcl_mq_epfd;
      if (libc_epoll_ctl (libc_epfd, EPOLL_CTL_ADD, ldpw->vcl_mq_epfd, &e) <
	  0)
	{
	  LDBG (0, "epfd %d, add libc mq epoll fd %d to libc epoll fd %d",
		epfd, ldpw->vcl_mq_epfd, libc_epfd);
	  rv = -1;
	  goto done;
	}
      ldpw->mq_epfd_added = 1;
    }

  /* Request to only drain unhandled to prevent libc_epoll_wait starved */
  rv = vls_epoll_wait (ep_vlsh, events, maxevents, -2);
  if (rv > 0)
    goto done;
  else if (PREDICT_FALSE (rv < 0))
    {
      errno = -rv;
      rv = -1;
      goto done;
    }

  rv = libc_epoll_pwait (libc_epfd, events, maxevents, timeout, sigmask);
  if (rv <= 0)
    goto done;
  for (int i = 0; i < rv; i++)
    {
      if (events[i].data.fd == ldpw->vcl_mq_epfd)
	{
	  /* We should remove mq epoll fd from events. */
	  rv--;
	  if (i != rv)
	    {
	      events[i].events = events[rv].events;
	      events[i].data.u64 = events[rv].data.u64;
	    }
	  num_ev = vls_epoll_wait (ep_vlsh, &events[rv], maxevents - rv, 0);
	  if (PREDICT_TRUE (num_ev > 0))
	    rv += num_ev;
	  break;
	}
    }

done:
  return rv;
}

int
epoll_pwait (int epfd, struct epoll_event *events,
	     int maxevents, int timeout, const sigset_t * sigmask)
{
  if (vls_use_eventfd ())
    return ldp_epoll_pwait_eventfd (epfd, events, maxevents, timeout,
				    sigmask);
  else
    return ldp_epoll_pwait (epfd, events, maxevents, timeout, sigmask);
}

int
epoll_wait (int epfd, struct epoll_event *events, int maxevents, int timeout)
{
  if (vls_use_eventfd ())
    return ldp_epoll_pwait_eventfd (epfd, events, maxevents, timeout, NULL);
  else
    return ldp_epoll_pwait (epfd, events, maxevents, timeout, NULL);
}

int
poll (struct pollfd *fds, nfds_t nfds, int timeout)
{
  ldp_worker_ctx_t *ldpw = ldp_worker_get_current ();
  int rv, i, n_revents = 0;
  vls_handle_t vlsh;
  vcl_poll_t *vp;
  double max_time;

  LDBG (3, "fds %p, nfds %ld, timeout %d", fds, nfds, timeout);

  if (PREDICT_FALSE (ldpw->clib_time.init_cpu_time == 0))
    clib_time_init (&ldpw->clib_time);

  max_time = (timeout >= 0) ? (f64) timeout / 1000 : 0;
  max_time += clib_time_now (&ldpw->clib_time);

  for (i = 0; i < nfds; i++)
    {
      if (fds[i].fd < 0)
	continue;

      vlsh = ldp_fd_to_vlsh (fds[i].fd);
      if (vlsh != VLS_INVALID_HANDLE)
	{
	  fds[i].fd = -fds[i].fd;
	  vec_add2 (ldpw->vcl_poll, vp, 1);
	  vp->fds_ndx = i;
	  vp->sh = vlsh_to_sh (vlsh);
	  vp->events = fds[i].events;
#ifdef __USE_XOPEN2K
	  if (fds[i].events & POLLRDNORM)
	    vp->events |= POLLIN;
	  if (fds[i].events & POLLWRNORM)
	    vp->events |= POLLOUT;
#endif
	  vp->revents = fds[i].revents;
	}
      else
	{
	  vec_add1 (ldpw->libc_poll, fds[i]);
	  vec_add1 (ldpw->libc_poll_idxs, i);
	}
    }

  do
    {
      if (vec_len (ldpw->vcl_poll))
	{
	  rv = vppcom_poll (ldpw->vcl_poll, vec_len (ldpw->vcl_poll), 0);
	  if (rv < 0)
	    {
	      errno = -rv;
	      rv = -1;
	      goto done;
	    }
	  else
	    n_revents += rv;
	}

      if (vec_len (ldpw->libc_poll))
	{
	  rv = libc_poll (ldpw->libc_poll, vec_len (ldpw->libc_poll), 0);
	  if (rv < 0)
	    goto done;
	  else
	    n_revents += rv;
	}

      if (n_revents)
	{
	  rv = n_revents;
	  goto done;
	}
    }
  while ((timeout < 0) || (clib_time_now (&ldpw->clib_time) < max_time));
  rv = 0;

done:
  vec_foreach (vp, ldpw->vcl_poll)
  {
    fds[vp->fds_ndx].fd = -fds[vp->fds_ndx].fd;
    fds[vp->fds_ndx].revents = vp->revents;
#ifdef __USE_XOPEN2K
    if ((fds[vp->fds_ndx].revents & POLLIN) &&
	(fds[vp->fds_ndx].events & POLLRDNORM))
      fds[vp->fds_ndx].revents |= POLLRDNORM;
    if ((fds[vp->fds_ndx].revents & POLLOUT) &&
	(fds[vp->fds_ndx].events & POLLWRNORM))
      fds[vp->fds_ndx].revents |= POLLWRNORM;
#endif
  }
  vec_reset_length (ldpw->vcl_poll);

  for (i = 0; i < vec_len (ldpw->libc_poll); i++)
    {
      fds[ldpw->libc_poll_idxs[i]].revents = ldpw->libc_poll[i].revents;
    }
  vec_reset_length (ldpw->libc_poll_idxs);
  vec_reset_length (ldpw->libc_poll);

  return rv;
}

#ifdef USE_GNU
int
ppoll (struct pollfd *fds, nfds_t nfds,
       const struct timespec *timeout, const sigset_t * sigmask)
{
  ldp_init_check ();

  clib_warning ("LDP<%d>: LDP-TBD", getpid ());
  errno = ENOSYS;


  return -1;
}
#endif

void CONSTRUCTOR_ATTRIBUTE ldp_constructor (void);

void DESTRUCTOR_ATTRIBUTE ldp_destructor (void);

/*
 * This function is called when the library is loaded
 */
void
ldp_constructor (void)
{
  swrap_constructor ();
  if (ldp_init () != 0)
    {
      fprintf (stderr, "\nLDP<%d>: ERROR: ldp_constructor: failed!\n",
	       getpid ());
      _exit (1);
    }
  else if (LDP_DEBUG > 0)
    clib_warning ("LDP<%d>: LDP constructor: done!\n", getpid ());
}

/*
 * This function is called when the library is unloaded
 */
void
ldp_destructor (void)
{
  /*
     swrap_destructor ();
     if (ldp->init)
     ldp->init = 0;
   */

  /* Don't use clib_warning() here because that calls writev()
   * which will call ldp_init().
   */
  if (LDP_DEBUG > 0)
    fprintf (stderr, "%s:%d: LDP<%d>: LDP destructor: done!\n",
	     __func__, __LINE__, getpid ());
}


/*
 * fd.io coding-style-patch-verification: ON
 *
 * Local Variables:
 * eval: (c-set-style "gnu")
 * End:
 */