summaryrefslogtreecommitdiffstats
path: root/src/plugins/idpf/idpf.h
blob: 0bac575d4b42833ce53b400a764491cfbdaaf58a (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
/*
 *------------------------------------------------------------------
 * Copyright (c) 2023 Intel and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *------------------------------------------------------------------
 */

#ifndef _IDPF_H_
#define _IDPF_H_

#include <vlib/vlib.h>
#include <vppinfra/ring.h>
#include <vlib/unix/unix.h>
#include <vlib/pci/pci.h>
#include <vnet/ethernet/ethernet.h>
#include <vnet/interface/rx_queue_funcs.h>
#include <vnet/interface/tx_queue_funcs.h>

#include <vppinfra/types.h>
#include <vppinfra/error_bootstrap.h>
#include <vppinfra/lock.h>

#include <vlib/log.h>
#include <vlib/pci/pci.h>

#include <vnet/interface.h>

#include <vnet/devices/devices.h>
#include <vnet/flow/flow.h>

#include <idpf/virtchnl2.h>
#include <sys/queue.h>

#define BIT(a) (1UL << (a))

/*
 * LAN PF register
 */
#define MAKEMASK(m, s) ((m) << (s))

/* Receive queues */
#define PF_QRX_BASE		0x00000000
#define PF_QRX_TAIL(_QRX)	(PF_QRX_BASE + (((_QRX) *0x1000)))
#define PF_QRX_BUFFQ_BASE	0x03000000
#define PF_QRX_BUFFQ_TAIL(_QRX) (PF_QRX_BUFFQ_BASE + (((_QRX) *0x1000)))

/* Transmit queues */
#define PF_QTX_BASE		 0x05000000
#define PF_QTX_COMM_DBELL(_DBQM) (PF_QTX_BASE + ((_DBQM) *0x1000))

/* Control(PF Mailbox) Queue */
#define PF_FW_BASE 0x08400000

#define PF_FW_ARQBAL		 (PF_FW_BASE)
#define PF_FW_ARQBAH		 (PF_FW_BASE + 0x4)
#define PF_FW_ARQLEN		 (PF_FW_BASE + 0x8)
#define PF_FW_ARQLEN_ARQLEN_S	 0
#define PF_FW_ARQLEN_ARQLEN_M	 MAKEMASK (0x1FFF, PF_FW_ARQLEN_ARQLEN_S)
#define PF_FW_ARQLEN_ARQVFE_S	 28
#define PF_FW_ARQLEN_ARQVFE_M	 BIT (PF_FW_ARQLEN_ARQVFE_S)
#define PF_FW_ARQLEN_ARQOVFL_S	 29
#define PF_FW_ARQLEN_ARQOVFL_M	 BIT (PF_FW_ARQLEN_ARQOVFL_S)
#define PF_FW_ARQLEN_ARQCRIT_S	 30
#define PF_FW_ARQLEN_ARQCRIT_M	 BIT (PF_FW_ARQLEN_ARQCRIT_S)
#define PF_FW_ARQLEN_ARQENABLE_S 31
#define PF_FW_ARQLEN_ARQENABLE_M BIT (PF_FW_ARQLEN_ARQENABLE_S)
#define PF_FW_ARQH		 (PF_FW_BASE + 0xC)
#define PF_FW_ARQH_ARQH_S	 0
#define PF_FW_ARQH_ARQH_M	 MAKEMASK (0x1FFF, PF_FW_ARQH_ARQH_S)
#define PF_FW_ARQT		 (PF_FW_BASE + 0x10)

#define PF_FW_ATQBAL		 (PF_FW_BASE + 0x14)
#define PF_FW_ATQBAH		 (PF_FW_BASE + 0x18)
#define PF_FW_ATQLEN		 (PF_FW_BASE + 0x1C)
#define PF_FW_ATQLEN_ATQLEN_S	 0
#define PF_FW_ATQLEN_ATQLEN_M	 MAKEMASK (0x3FF, PF_FW_ATQLEN_ATQLEN_S)
#define PF_FW_ATQLEN_ATQVFE_S	 28
#define PF_FW_ATQLEN_ATQVFE_M	 BIT (PF_FW_ATQLEN_ATQVFE_S)
#define PF_FW_ATQLEN_ATQOVFL_S	 29
#define PF_FW_ATQLEN_ATQOVFL_M	 BIT (PF_FW_ATQLEN_ATQOVFL_S)
#define PF_FW_ATQLEN_ATQCRIT_S	 30
#define PF_FW_ATQLEN_ATQCRIT_M	 BIT (PF_FW_ATQLEN_ATQCRIT_S)
#define PF_FW_ATQLEN_ATQENABLE_S 31
#define PF_FW_ATQLEN_ATQENABLE_M BIT (PF_FW_ATQLEN_ATQENABLE_S)
#define PF_FW_ATQH		 (PF_FW_BASE + 0x20)
#define PF_FW_ATQH_ATQH_S	 0
#define PF_FW_ATQH_ATQH_M	 MAKEMASK (0x3FF, PF_FW_ATQH_ATQH_S)
#define PF_FW_ATQT		 (PF_FW_BASE + 0x24)

/* Interrupts */
#define PF_GLINT_BASE		     0x08900000
#define PF_GLINT_DYN_CTL_ITR_INDX_S  3
#define PF_GLINT_DYN_CTL_ITR_INDX_M  MAKEMASK (0x3, PF_GLINT_DYN_CTL_ITR_INDX_S)
#define PF_GLINT_DYN_CTL_INTERVAL_S  5
#define PF_GLINT_DYN_CTL_INTERVAL_M  BIT (PF_GLINT_DYN_CTL_INTERVAL_S)
#define PF_GLINT_DYN_CTL_WB_ON_ITR_S 30
#define PF_GLINT_DYN_CTL_WB_ON_ITR_M BIT (PF_GLINT_DYN_CTL_WB_ON_ITR_S)

/* Generic registers */
#define PFGEN_RSTAT		0x08407008 /* PFR Status */
#define PFGEN_RSTAT_PFR_STATE_S 0
#define PFGEN_RSTAT_PFR_STATE_M MAKEMASK (0x3, PFGEN_RSTAT_PFR_STATE_S)
#define PFGEN_CTRL		0x0840700C
#define PFGEN_CTRL_PFSWR	BIT (0)

#define IDPF_CTLQ_ID	       -1
#define IDPF_CTLQ_LEN	       64
#define IDPF_DFLT_MBX_BUF_SIZE 4096

#define IDPF_MAX_NUM_QUEUES 256
#define IDPF_MIN_BUF_SIZE   1024
#define IDPF_MAX_FRAME_SIZE 9728
#define IDPF_MAX_PKT_TYPE   1024
#define IDPF_QUEUE_SZ_MAX   4096
#define IDPF_QUEUE_SZ_MIN   64

#define IDPF_RESET_SUSPEND_TIME	 20e-3
#define IDPF_RESET_MAX_WAIT_TIME 1

#define IDPF_SEND_TO_PF_SUSPEND_TIME  10e-3
#define IDPF_SEND_TO_PF_MAX_WAIT_TIME 1
#define IDPF_SEND_TO_PF_MAX_TRY_TIMES 200

#define IDPF_RX_MAX_DESC_IN_CHAIN 5

#define IDPF_MAX_VPORT_NUM  8
#define IDPF_DFLT_Q_VEC_NUM 1
#define IDPF_DFLT_INTERVAL  16

#define IDPF_DEFAULT_RXQ_NUM 16
#define IDPF_DEFAULT_TXQ_NUM 16

#define IDPF_ETH_ALEN 6

#define IDPF_INVALID_VPORT_IDX 0xffff
#define IDPF_TXQ_PER_GRP       1
#define IDPF_TX_COMPLQ_PER_GRP 1
#define IDPF_RXQ_PER_GRP       1
#define IDPF_RX_BUFQ_PER_GRP   2
#define IDPF_RX_BUF_STRIDE     64

/* Maximum buffer lengths for all control queue types */
#define IDPF_CTLQ_MAX_RING_SIZE 1024
#define IDPF_CTLQ_MAX_BUF_LEN	4096

#define IDPF_HI_DWORD(x) ((u32) ((((x) >> 16) >> 16) & 0xFFFFFFFF))
#define IDPF_LO_DWORD(x) ((u32) ((x) &0xFFFFFFFF))
#define IDPF_HI_WORD(x)	 ((u16) (((x) >> 16) & 0xFFFF))
#define IDPF_LO_WORD(x)	 ((u16) ((x) &0xFFFF))

#define IDPF_CTLQ_DESC(R, i) (&(((idpf_ctlq_desc_t *) ((R)->desc_ring.va))[i]))

#define IDPF_CTLQ_DESC_UNUSED(R)                                              \
  (u16) ((((R)->next_to_clean > (R)->next_to_use) ? 0 : (R)->ring_size) +     \
	 (R)->next_to_clean - (R)->next_to_use - 1)

#define IDPF_GET_PTYPE_SIZE(p)                                                \
  (sizeof (virtchnl2_ptype_t) +                                               \
   (((p)->proto_id_count ? ((p)->proto_id_count - 1) : 0) *                   \
    sizeof ((p)->proto_id[0])))

/* log configuration */
extern vlib_log_class_registration_t idpf_log;
extern vlib_log_class_registration_t idpf_stats_log;

#define idpf_log_err(dev, f, ...)                                             \
  vlib_log (VLIB_LOG_LEVEL_ERR, idpf_log.class, "%U: " f,                     \
	    format_vlib_pci_addr, &dev->pci_addr, ##__VA_ARGS__)

#define idpf_log_warn(dev, f, ...)                                            \
  vlib_log (VLIB_LOG_LEVEL_WARNING, idpf_log.class, "%U: " f,                 \
	    format_vlib_pci_addr, &dev->pci_addr, ##__VA_ARGS__)

#define idpf_log_debug(dev, f, ...)                                           \
  vlib_log (VLIB_LOG_LEVEL_DEBUG, idpf_log.class, "%U: " f,                   \
	    format_vlib_pci_addr, &dev->pci_addr, ##__VA_ARGS__)

#define idpf_stats_log_debug(dev, f, ...)                                     \
  vlib_log (VLIB_LOG_LEVEL_DEBUG, idpf_stats_log.class, "%U: " f,             \
	    format_vlib_pci_addr, &dev->pci_addr, ##__VA_ARGS__)

/* List handler */
#ifndef LIST_HEAD_TYPE
#define LIST_HEAD_TYPE(list_name, type) LIST_HEAD (list_name, type)
#endif

#ifndef LIST_ENTRY_TYPE
#define LIST_ENTRY_TYPE(type) LIST_ENTRY (type)
#endif

#ifndef LIST_FOR_EACH_ENTRY_SAFE
#define LIST_FOR_EACH_ENTRY_SAFE(pos, temp, head, entry_type, list)           \
  LIST_FOREACH (pos, head, list)
#endif

#ifndef LIST_FOR_EACH_ENTRY
#define LIST_FOR_EACH_ENTRY(pos, head, entry_type, list)                      \
  LIST_FOREACH (pos, head, list)
#endif

#define foreach_idpf_device_flags                                             \
  _ (0, INITIALIZED, "initialized")                                           \
  _ (1, ERROR, "error")                                                       \
  _ (2, ADMIN_UP, "admin-up")                                                 \
  _ (3, VA_DMA, "vaddr-dma")                                                  \
  _ (4, LINK_UP, "link-up")                                                   \
  _ (6, ELOG, "elog")                                                         \
  _ (7, PROMISC, "promisc")                                                   \
  _ (8, RX_INT, "rx-interrupts")                                              \
  _ (9, RX_FLOW_OFFLOAD, "rx-flow-offload")

enum
{
#define _(a, b, c) IDPF_DEVICE_F_##b = (1 << a),
  foreach_idpf_device_flags
#undef _
};

#define IDPF_PTYPE_UNKNOWN		     0x00000000
#define IDPF_PTYPE_L2_ETHER		     0x00000001
#define IDPF_PTYPE_L2_ETHER_TIMESYNC	     0x00000002
#define IDPF_PTYPE_L2_ETHER_ARP		     0x00000003
#define IDPF_PTYPE_L2_ETHER_LLDP	     0x00000004
#define IDPF_PTYPE_L2_ETHER_NSH		     0x00000005
#define IDPF_PTYPE_L2_ETHER_VLAN	     0x00000006
#define IDPF_PTYPE_L2_ETHER_QINQ	     0x00000007
#define IDPF_PTYPE_L2_ETHER_PPPOE	     0x00000008
#define IDPF_PTYPE_L2_ETHER_FCOE	     0x00000009
#define IDPF_PTYPE_L2_ETHER_MPLS	     0x0000000a
#define IDPF_PTYPE_L2_MASK		     0x0000000f
#define IDPF_PTYPE_L3_IPV4		     0x00000010
#define IDPF_PTYPE_L3_IPV4_EXT		     0x00000030
#define IDPF_PTYPE_L3_IPV6		     0x00000040
#define IDPF_PTYPE_L3_IPV4_EXT_UNKNOWN	     0x00000090
#define IDPF_PTYPE_L3_IPV6_EXT		     0x000000c0
#define IDPF_PTYPE_L3_IPV6_EXT_UNKNOWN	     0x000000e0
#define IDPF_PTYPE_L3_MASK		     0x000000f0
#define IDPF_PTYPE_L4_TCP		     0x00000100
#define IDPF_PTYPE_L4_UDP		     0x00000200
#define IDPF_PTYPE_L4_FRAG		     0x00000300
#define IDPF_PTYPE_L4_SCTP		     0x00000400
#define IDPF_PTYPE_L4_ICMP		     0x00000500
#define IDPF_PTYPE_L4_NONFRAG		     0x00000600
#define IDPF_PTYPE_L4_IGMP		     0x00000700
#define IDPF_PTYPE_L4_MASK		     0x00000f00
#define IDPF_PTYPE_TUNNEL_IP		     0x00001000
#define IDPF_PTYPE_TUNNEL_GRE		     0x00002000
#define IDPF_PTYPE_TUNNEL_VXLAN		     0x00003000
#define IDPF_PTYPE_TUNNEL_NVGRE		     0x00004000
#define IDPF_PTYPE_TUNNEL_GENEVE	     0x00005000
#define IDPF_PTYPE_TUNNEL_GRENAT	     0x00006000
#define IDPF_PTYPE_TUNNEL_GTPC		     0x00007000
#define IDPF_PTYPE_TUNNEL_GTPU		     0x00008000
#define IDPF_PTYPE_TUNNEL_ESP		     0x00009000
#define IDPF_PTYPE_TUNNEL_L2TP		     0x0000a000
#define IDPF_PTYPE_TUNNEL_VXLAN_GPE	     0x0000b000
#define IDPF_PTYPE_TUNNEL_MPLS_IN_GRE	     0x0000c000
#define IDPF_PTYPE_TUNNEL_MPLS_IN_UDP	     0x0000d000
#define IDPF_PTYPE_TUNNEL_MASK		     0x0000f000
#define IDPF_PTYPE_INNER_L2_ETHER	     0x00010000
#define IDPF_PTYPE_INNER_L2_ETHER_VLAN	     0x00020000
#define IDPF_PTYPE_INNER_L2_ETHER_QINQ	     0x00030000
#define IDPF_PTYPE_INNER_L2_MASK	     0x000f0000
#define IDPF_PTYPE_INNER_L3_IPV4	     0x00100000
#define IDPF_PTYPE_INNER_L3_IPV4_EXT	     0x00200000
#define IDPF_PTYPE_INNER_L3_IPV6	     0x00300000
#define IDPF_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN 0x00400000
#define IDPF_PTYPE_INNER_L3_IPV6_EXT	     0x00500000
#define IDPF_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN 0x00600000
#define IDPF_PTYPE_INNER_L3_MASK	     0x00f00000
#define IDPF_PTYPE_INNER_L4_TCP		     0x01000000
#define IDPF_PTYPE_INNER_L4_UDP		     0x02000000
#define IDPF_PTYPE_INNER_L4_FRAG	     0x03000000
#define IDPF_PTYPE_INNER_L4_SCTP	     0x04000000
#define IDPF_PTYPE_INNER_L4_ICMP	     0x05000000
#define IDPF_PTYPE_INNER_L4_NONFRAG	     0x06000000
#define IDPF_PTYPE_INNER_L4_MASK	     0x0f000000
#define IDPF_PTYPE_ALL_MASK		     0x0fffffff

/* Flags sub-structure
 * |0  |1  |2  |3  |4  |5  |6  |7  |8  |9  |10 |11 |12 |13 |14 |15 |
 * |DD |CMP|ERR|  * RSV *  |FTYPE  | *RSV* |RD |VFC|BUF|  HOST_ID  |
 */
/* command flags and offsets */
#define IDPF_CTLQ_FLAG_DD_S	 0
#define IDPF_CTLQ_FLAG_CMP_S	 1
#define IDPF_CTLQ_FLAG_ERR_S	 2
#define IDPF_CTLQ_FLAG_FTYPE_S	 6
#define IDPF_CTLQ_FLAG_RD_S	 10
#define IDPF_CTLQ_FLAG_VFC_S	 11
#define IDPF_CTLQ_FLAG_BUF_S	 12
#define IDPF_CTLQ_FLAG_HOST_ID_S 13

#define IDPF_CTLQ_FLAG_DD  BIT (IDPF_CTLQ_FLAG_DD_S)  /* 0x1	  */
#define IDPF_CTLQ_FLAG_CMP BIT (IDPF_CTLQ_FLAG_CMP_S) /* 0x2	  */
#define IDPF_CTLQ_FLAG_ERR BIT (IDPF_CTLQ_FLAG_ERR_S) /* 0x4	  */
#define IDPF_CTLQ_FLAG_FTYPE_VM                                               \
  BIT (IDPF_CTLQ_FLAG_FTYPE_S)					 /* 0x40	  */
#define IDPF_CTLQ_FLAG_FTYPE_PF BIT (IDPF_CTLQ_FLAG_FTYPE_S + 1) /* 0x80   */
#define IDPF_CTLQ_FLAG_RD	BIT (IDPF_CTLQ_FLAG_RD_S)	 /* 0x400  */
#define IDPF_CTLQ_FLAG_VFC	BIT (IDPF_CTLQ_FLAG_VFC_S)	 /* 0x800  */
#define IDPF_CTLQ_FLAG_BUF	BIT (IDPF_CTLQ_FLAG_BUF_S)	 /* 0x1000 */

/* Host ID is a special field that has 3b and not a 1b flag */
#define IDPF_CTLQ_FLAG_HOST_ID_M MAKE_MASK (0x7000UL, IDPF_CTLQ_FLAG_HOST_ID_S)

#define IDPF_FLEX_TXD_QW1_DTYPE_S 0
#define IDPF_FLEX_TXD_QW1_DTYPE_M MAKEMASK (0x1FUL, IDPF_FLEX_TXD_QW1_DTYPE_S)
#define IDPF_FLEX_TXD_QW1_CMD_S	  5
#define IDPF_FLEX_TXD_QW1_CMD_M	  MAKEMASK (0x7FFUL, IDPF_FLEX_TXD_QW1_CMD_S)

typedef struct idpf_vport idpf_vport_t;

typedef volatile struct
{
  u64 buf_addr; /* Packet buffer address */
  struct
  {
    u64 cmd_dtype;
    union
    {
      /* DTYPE = IDPF_TX_DESC_DTYPE_FLEX_DATA_(0x03) */
      u8 raw[4];

      /* DTYPE = IDPF_TX_DESC_DTYPE_FLEX_TSYN_L2TAG1 (0x06) */
      struct
      {
	u16 l2tag1;
	u8 flex;
	u8 tsync;
      } tsync;

      /* DTYPE=IDPF_TX_DESC_DTYPE_FLEX_L2TAG1_L2TAG2 (0x07) */
      struct
      {
	u16 l2tag1;
	u16 l2tag2;
      } l2tags;
    } flex;
    u16 buf_size;
  } qw1;
} idpf_flex_tx_desc_t;

typedef struct
{
  union
  {
    u64 qword[2];
  };
} idpf_tx_desc_t;

STATIC_ASSERT_SIZEOF (idpf_tx_desc_t, 16);

typedef struct idpf_rxq
{
  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
  volatile u32 *qrx_tail;
  u16 next;
  u16 size;
  virtchnl2_rx_desc_t *descs;
  u32 *bufs;
  u16 n_enqueued;
  u8 int_mode;
  u8 buffer_pool_index;
  u32 queue_index;

  struct idpf_rxq *bufq1;
  struct idpf_rxq *bufq2;
} idpf_rxq_t;

typedef struct idpf_txq
{
  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
  volatile u32 *qtx_tail;
  u16 next;
  u16 size;
  u32 *ph_bufs;
  clib_spinlock_t lock;
  idpf_tx_desc_t *descs;
  u32 *bufs;
  u16 n_enqueued;
  u16 *rs_slots;

  idpf_tx_desc_t *tmp_descs;
  u32 *tmp_bufs;
  u32 queue_index;

  struct idpf_txq *complq;
} idpf_txq_t;

typedef struct
{
  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
  u32 flags;
  u32 per_interface_next_index;
  u32 cmd_retval;
  u8 *mbx_resp;
  virtchnl2_op_t pend_cmd;

  u32 dev_instance;
  u32 sw_if_index;
  u32 hw_if_index;
  vlib_pci_dev_handle_t pci_dev_handle;
  u32 numa_node;
  void *bar0;
  u8 *name;

  /* queues */
  u16 n_tx_queues;
  u16 n_rx_queues;
  u32 txq_model;
  u32 rxq_model;

  u16 vsi_id;
  u8 hwaddr[6];
  u16 max_mtu;
  vlib_pci_addr_t pci_addr;

  /* error */
  clib_error_t *error;

  /* hw info */
  u8 *hw_addr;
  u64 hw_addr_len;

  /* control queue - send and receive */
  struct idpf_ctlq_info *asq;
  struct idpf_ctlq_info *arq;

  /* pci info */
  u16 device_id;
  u16 vendor_id;
  u16 subsystem_device_id;
  u16 subsystem_vendor_id;

  /* max config queue number per vc message */
  u32 max_rxq_per_msg;
  u32 max_txq_per_msg;

  /* vport info */
  idpf_vport_t **vports;
  u16 max_vport_nb;
  u16 req_vports[IDPF_MAX_VPORT_NUM];
  u16 req_vport_nb;
  u16 cur_vports;
  u16 cur_vport_nb;
  u16 cur_vport_idx;

  u32 ptype_tbl[IDPF_MAX_PKT_TYPE];

  /* device capability */
  u32 csum_caps;
  u32 seg_caps;
  u32 hsplit_caps;
  u32 rsc_caps;
  u64 rss_caps;
  u64 other_caps;

  u16 max_rx_q;
  u16 max_tx_q;
  u16 max_rx_bufq;
  u16 max_tx_complq;
  u16 max_sriov_vfs;
  u16 max_vports;
  u16 default_num_vports;

  u32 device_type;

  LIST_HEAD_TYPE (list_head, idpf_ctlq_info) cq_list_head;
} idpf_device_t;

/* memory allocation tracking */
typedef struct
{
  void *va;
  u64 pa;
  u32 size;
} idpf_dma_mem_t;

/* Message type read in virtual channel from PF */
typedef enum
{
  IDPF_MSG_ERR = -1, /* Meet error when accessing admin queue */
  IDPF_MSG_NON,	     /* Read nothing from admin queue */
  IDPF_MSG_SYS,	     /* Read system msg from admin queue */
  IDPF_MSG_CMD,	     /* Read async command result */
} idpf_vc_result_t;

typedef struct
{
  u32 tx_start_qid;
  u32 rx_start_qid;
  u32 tx_compl_start_qid;
  u32 rx_buf_start_qid;

  u64 tx_qtail_start;
  u32 tx_qtail_spacing;
  u64 rx_qtail_start;
  u32 rx_qtail_spacing;
  u64 tx_compl_qtail_start;
  u32 tx_compl_qtail_spacing;
  u64 rx_buf_qtail_start;
  u32 rx_buf_qtail_spacing;
} idpf_chunks_info_t;

typedef struct
{
  u32 ops;
  u8 *in_args;	    /* buffer for sending */
  u32 in_args_size; /* buffer size for sending */
  u8 *out_buffer;   /* buffer for response */
  u32 out_size;	    /* buffer size for response */
} idpf_cmd_info_t;

typedef struct
{
  idpf_device_t *id;
  u16 idx;
} idpf_vport_param_t;

struct idpf_vport
{
  idpf_device_t *id;
  virtchnl2_create_vport_t *vport_info;
  u16 idx;
  u16 vport_id;
  u32 txq_model;
  u32 rxq_model;
  u32 num_tx_q;
  idpf_txq_t *txqs;
  u16 num_tx_complq;
  u16 num_rx_q;
  idpf_rxq_t *rxqs;
  u16 num_rx_bufq;

  u16 max_mtu;
  u8 default_mac_addr[VIRTCHNL2_ETH_LENGTH_OF_ADDRESS];

  u16 max_pkt_len; /* Maximum packet length */

  /* MSIX info*/
  virtchnl2_queue_vector_t *qv_map; /* queue vector mapping */
  u16 max_vectors;
  virtchnl2_alloc_vectors_t *recv_vectors;

  /* Chunk info */
  idpf_chunks_info_t chunks_info;

  virtchnl2_vport_stats_t eth_stats_offset;
};

#define IDPF_RX_VECTOR_SZ VLIB_FRAME_SIZE

typedef enum
{
  IDPF_PROCESS_REQ_ADD_DEL_ETH_ADDR = 1,
  IDPF_PROCESS_REQ_CONFIG_PROMISC_MDDE = 2,
  IDPF_PROCESS_REQ_PROGRAM_FLOW = 3,
} idpf_process_req_type_t;

typedef struct
{
  idpf_process_req_type_t type;
  u32 dev_instance;
  u32 calling_process_index;
  u8 eth_addr[6];
  int is_add, is_enable;

  /* below parameters are used for 'program flow' event */
  u8 *rule;
  u32 rule_len;
  u8 *program_status;
  u32 status_len;

  clib_error_t *error;
} idpf_process_req_t;

typedef struct
{
  u64 qw1s[IDPF_RX_MAX_DESC_IN_CHAIN - 1];
  u32 buffers[IDPF_RX_MAX_DESC_IN_CHAIN - 1];
} idpf_rx_tail_t;

typedef struct
{
  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
  vlib_buffer_t *bufs[IDPF_RX_VECTOR_SZ];
  u16 next[IDPF_RX_VECTOR_SZ];
  u64 qw1s[IDPF_RX_VECTOR_SZ];
  u32 flow_ids[IDPF_RX_VECTOR_SZ];
  idpf_rx_tail_t tails[IDPF_RX_VECTOR_SZ];
  vlib_buffer_t buffer_template;
} idpf_per_thread_data_t;

typedef struct
{
  u16 msg_id_base;

  idpf_device_t **devices;
  idpf_per_thread_data_t *per_thread_data;
} idpf_main_t;

extern idpf_main_t idpf_main;

typedef struct
{
  vlib_pci_addr_t addr;
  u8 *name;
  u16 rxq_single;
  u16 txq_single;
  u16 rxq_num;
  u16 txq_num;
  u16 req_vport_nb;
  u16 rxq_size;
  u16 txq_size;
  int rv;
  u32 sw_if_index;
  clib_error_t *error;
} idpf_create_if_args_t;

void idpf_create_if (vlib_main_t *vm, idpf_create_if_args_t *args);

extern vlib_node_registration_t idpf_process_node;
extern vnet_device_class_t idpf_device_class;

/* format.c */
format_function_t format_idpf_device_name;
format_function_t format_idpf_device_flags;

static inline void
clear_cmd (idpf_device_t *id)
{
  /* Return value may be checked in anither thread, need to ensure the
   * coherence. */
  CLIB_MEMORY_BARRIER ();
  id->pend_cmd = VIRTCHNL2_OP_UNKNOWN;
  id->cmd_retval = VIRTCHNL2_STATUS_SUCCESS;
}

static_always_inline idpf_device_t *
idpf_get_device (u32 dev_instance)
{
  return pool_elt_at_index (idpf_main.devices, dev_instance)[0];
}

static inline void
idpf_reg_write (idpf_device_t *id, u32 addr, u32 val)
{
  *(volatile u32 *) ((u8 *) id->bar0 + addr) = val;
}

static inline u32
idpf_reg_read (idpf_device_t *id, u32 addr)
{
  u32 val = *(volatile u32 *) (id->bar0 + addr);
  return val;
}

static inline void
idpf_reg_flush (idpf_device_t *id)
{
  idpf_reg_read (id, PFGEN_RSTAT);
  asm volatile("" ::: "memory");
}

typedef struct
{
  u16 qid;
  u16 next_index;
  u32 hw_if_index;
  u32 flow_id;
  u64 qw1s[IDPF_RX_MAX_DESC_IN_CHAIN];
} idpf_input_trace_t;

/* Error Codes */
/* Linux kernel driver can't directly use these. Instead, they are mapped to
 * linux compatible error codes which get translated in the build script.
 */
#define IDPF_SUCCESS		      0
#define IDPF_ERR_PARAM		      -53  /* -EBADR */
#define IDPF_ERR_NOT_IMPL	      -95  /* -EOPNOTSUPP */
#define IDPF_ERR_NOT_READY	      -16  /* -EBUSY */
#define IDPF_ERR_BAD_PTR	      -14  /* -EFAULT */
#define IDPF_ERR_INVAL_SIZE	      -90  /* -EMSGSIZE */
#define IDPF_ERR_DEVICE_NOT_SUPPORTED -19  /* -ENODEV */
#define IDPF_ERR_FW_API_VER	      -13  /* -EACCESS */
#define IDPF_ERR_NO_MEMORY	      -12  /* -ENOMEM */
#define IDPF_ERR_CFG		      -22  /* -EINVAL */
#define IDPF_ERR_OUT_OF_RANGE	      -34  /* -ERANGE */
#define IDPF_ERR_ALREADY_EXISTS	      -17  /* -EEXIST */
#define IDPF_ERR_DOES_NOT_EXIST	      -6   /* -ENXIO */
#define IDPF_ERR_IN_USE		      -114 /* -EALREADY */
#define IDPF_ERR_MAX_LIMIT	      -109 /* -ETOOMANYREFS */
#define IDPF_ERR_RESET_ONGOING	      -104 /* -ECONNRESET */

/* CRQ/CSQ specific error codes */
#define IDPF_ERR_CTLQ_ERROR   -74  /* -EBADMSG */
#define IDPF_ERR_CTLQ_TIMEOUT -110 /* -ETIMEDOUT */
#define IDPF_ERR_CTLQ_FULL    -28  /* -ENOSPC */
#define IDPF_ERR_CTLQ_NO_WORK -42  /* -ENOMSG */
#define IDPF_ERR_CTLQ_EMPTY   -105 /* -ENOBUFS */

/* Used for queue init, response and events */
typedef enum
{
  IDPF_CTLQ_TYPE_MAILBOX_TX = 0,
  IDPF_CTLQ_TYPE_MAILBOX_RX = 1,
  IDPF_CTLQ_TYPE_CONFIG_TX = 2,
  IDPF_CTLQ_TYPE_CONFIG_RX = 3,
  IDPF_CTLQ_TYPE_EVENT_RX = 4,
  IDPF_CTLQ_TYPE_RDMA_TX = 5,
  IDPF_CTLQ_TYPE_RDMA_RX = 6,
  IDPF_CTLQ_TYPE_RDMA_COMPL = 7
} idpf_ctlq_type_t;

typedef enum
{
  IDPF_PROCESS_EVENT_START = 1,
  IDPF_PROCESS_EVENT_DELETE_IF = 2,
  IDPF_PROCESS_EVENT_AQ_INT = 3,
  IDPF_PROCESS_EVENT_REQ = 4,
} idpf_process_event_t;

/*
 * Generic Control Queue Structures
 */
typedef struct
{
  /* used for queue tracking */
  u32 head;
  u32 tail;
  /* Below applies only to default mb (if present) */
  u32 len;
  u32 bah;
  u32 bal;
  u32 len_mask;
  u32 len_ena_mask;
  u32 head_mask;
} idpf_ctlq_reg_t;

/* Generic queue msg structure */
typedef struct
{
  u8 vmvf_type; /* represents the source of the message on recv */
#define IDPF_VMVF_TYPE_VF 0
#define IDPF_VMVF_TYPE_VM 1
#define IDPF_VMVF_TYPE_PF 2
  u8 host_id;
  /* 3b field used only when sending a message to peer - to be used in
   * combination with target func_id to route the message
   */
#define IDPF_HOST_ID_MASK 0x7

  u16 opcode;
  u16 data_len; /* data_len = 0 when no payload is attached */
  union
  {
    u16 func_id; /* when sending a message */
    u16 status;	 /* when receiving a message */
  };
  union
  {
    struct
    {
      u32 chnl_retval;
      u32 chnl_opcode;
    } mbx;
    u64 cookie;
  } cookie;
  union
  {
#define IDPF_DIRECT_CTX_SIZE   16
#define IDPF_INDIRECT_CTX_SIZE 8
    /* 16 bytes of context can be provided or 8 bytes of context
     * plus the address of a DMA buffer
     */
    u8 direct[IDPF_DIRECT_CTX_SIZE];
    struct
    {
      u8 context[IDPF_INDIRECT_CTX_SIZE];
      idpf_dma_mem_t *payload;
    } indirect;
  } ctx;
} idpf_ctlq_msg_t;

/* Generic queue info structures */
/* MB, CONFIG and EVENT q do not have extended info */
typedef struct
{
  idpf_ctlq_type_t type;
  int id;	       /* absolute queue offset passed as input
			* -1 for default mailbox if present
			*/
  u16 len;	       /* Queue length passed as input */
  u16 buf_size;	       /* buffer size passed as input */
  u64 base_address;    /* output, HPA of the Queue start  */
  idpf_ctlq_reg_t reg; /* registers accessed by ctlqs */

  int ext_info_size;
  void *ext_info; /* Specific to q type */
} idpf_ctlq_create_info_t;

/* Control Queue information */
typedef struct idpf_ctlq_info
{
  LIST_ENTRY_TYPE (idpf_ctlq_info) cq_list;

  idpf_ctlq_type_t cq_type;
  int q_id;
  clib_spinlock_t cq_lock; /* queue lock */

  /* used for interrupt processing */
  u16 next_to_use;
  u16 next_to_clean;
  u16 next_to_post;

  idpf_dma_mem_t desc_ring; /* descriptor ring memory */

  union
  {
    idpf_dma_mem_t **rx_buff;
    idpf_ctlq_msg_t **tx_msg;
  } bi;

  u16 buf_size;	       /* queue buffer size */
  u16 ring_size;       /* Number of descriptors */
  idpf_ctlq_reg_t reg; /* registers accessed by ctlqs */
} idpf_ctlq_info_t;

/* PF/VF mailbox commands */
enum idpf_mbx_opc
{
  /* idpf_mbq_opc_send_msg_to_pf:
   *	usage: used by PF or VF to send a message to its CPF
   *	target: RX queue and function ID of parent PF taken from HW
   */
  idpf_mbq_opc_send_msg_to_pf = 0x0801,

  /* idpf_mbq_opc_send_msg_to_vf:
   *	usage: used by PF to send message to a VF
   *	target: VF control queue ID must be specified in descriptor
   */
  idpf_mbq_opc_send_msg_to_vf = 0x0802,

  /* idpf_mbq_opc_send_msg_to_peer_pf:
   *	usage: used by any function to send message to any peer PF
   *	target: RX queue and host of parent PF taken from HW
   */
  idpf_mbq_opc_send_msg_to_peer_pf = 0x0803,

  /* idpf_mbq_opc_send_msg_to_peer_drv:
   *	usage: used by any function to send message to any peer driver
   *	target: RX queue and target host must be specific in descriptor
   */
  idpf_mbq_opc_send_msg_to_peer_drv = 0x0804,
};

typedef struct
{
  u16 flags;
  u16 opcode;
  u16 datalen; /* 0 for direct commands */
  union
  {
    u16 ret_val;
    u16 pfid_vfid;
  };
  u32 cookie_high;
  u32 cookie_low;
  union
  {
    struct
    {
      u32 param0;
      u32 param1;
      u32 param2;
      u32 param3;
    } direct;
    struct
    {
      u32 param0;
      u32 param1;
      u32 addr_high;
      u32 addr_low;
    } indirect;
    u8 raw[16];
  } params;
} idpf_ctlq_desc_t;

int idpf_ctlq_init (vlib_main_t *vm, idpf_device_t *id, u8 num_q,
		    idpf_ctlq_create_info_t *q_info);
int idpf_ctlq_add (vlib_main_t *vm, idpf_device_t *id,
		   idpf_ctlq_create_info_t *qinfo, struct idpf_ctlq_info **cq);
void idpf_ctlq_remove (idpf_device_t *id, struct idpf_ctlq_info *cq);
int idpf_ctlq_send (idpf_device_t *id, struct idpf_ctlq_info *cq,
		    u16 num_q_msg, idpf_ctlq_msg_t q_msg[]);
int idpf_ctlq_recv (struct idpf_ctlq_info *cq, u16 *num_q_msg,
		    idpf_ctlq_msg_t *q_msg);
int idpf_ctlq_clean_sq (struct idpf_ctlq_info *cq, u16 *clean_count,
			idpf_ctlq_msg_t *msg_status[]);
int idpf_ctlq_post_rx_buffs (idpf_device_t *id, struct idpf_ctlq_info *cq,
			     u16 *buff_count, idpf_dma_mem_t **buffs);
void idpf_ctlq_deinit (idpf_device_t *id);
int idpf_ctlq_alloc_ring_res (vlib_main_t *vm, idpf_device_t *id,
			      struct idpf_ctlq_info *cq);
void idpf_ctlq_dealloc_ring_res (idpf_device_t *id, struct idpf_ctlq_info *cq);
void *idpf_alloc_dma_mem (vlib_main_t *vm, idpf_device_t *id,
			  idpf_dma_mem_t *mem, u64 size);
void idpf_free_dma_mem (idpf_device_t *id, idpf_dma_mem_t *mem);

#endif /* IDPF_H */

/*
 * fd.io coding-style-patch-verification: ON
 *
 * Local Variables:
 * eval: (c-set-style "gnu")
 * End:
 */