BIER: fix support for longer bit-string lengths

Change-Id: I2421197b76be58099e5f8ed5554410adff202109
Signed-off-by: Neale Ranns <neale.ranns@cisco.com>

9 files changed, 153 insertions, 142 deletions

diff --git a/src/vnet/bier/bier.api b/src/vnet/bier/bier.api
index 380d2973705..446863c8ee8 100644
--- a/src/vnet/bier/bier.api
+++ b/src/vnet/bier/bier.api
@@ -109,7 +109,9 @@ define bier_route_details
     @param context - sender context, to match reply w/ request
     @param bi_tbl_id - The BIER table-id used to forward post encap
     @param bi_src - The source Bit-position in the encap.
-    @param bi_n_bytes - The number of bytes in the following bit-string
+    @param bi_n_bytes - The number of bytes in the following bit-string.
+                        VPP only supports BSL of 1024 and less, so this is
+			a u8 field.
     @param bi_bytes - The bit-string represented as a byte array (MSB first)
 */
 define bier_imp_add
diff --git a/src/vnet/bier/bier_api.c b/src/vnet/bier/bier_api.c
index 840c33b7a37..4b1d1c28781 100644
--- a/src/vnet/bier/bier_api.c
+++ b/src/vnet/bier/bier_api.c
@@ -73,34 +73,41 @@ vl_api_bier_table_add_del_t_handler (vl_api_bier_table_add_del_t * mp)
     vnm = vnet_get_main ();
     vnm->api_errno = 0;
 
-    bier_table_id_t bti = {
-        .bti_set = mp->bt_tbl_id.bt_set,
-        .bti_sub_domain = mp->bt_tbl_id.bt_sub_domain,
-        .bti_hdr_len = mp->bt_tbl_id.bt_hdr_len_id,
-        .bti_type = BIER_TABLE_MPLS_SPF,
-        .bti_ecmp = BIER_ECMP_TABLE_ID_MAIN,
-    };
-
-    if (mp->bt_is_add)
+    if (mp->bt_tbl_id.bt_hdr_len_id >= BIER_HDR_LEN_2048)
     {
-        mpls_label_t label = ntohl(mp->bt_label);
-
-        /*
-         * convert acceptable 'don't want a label' values from 
-         * the API to the correct internal INVLID value
-         */
-        if ((0 == label) || (~0 == label))
-        {
-            label = MPLS_LABEL_INVALID;
-        }
-        bier_table_add_or_lock(&bti, label);
+        rv = VNET_API_ERROR_BIER_BSL_UNSUP;
     }
     else
     {
-        bier_table_unlock(&bti);
-    }
+        bier_table_id_t bti = {
+            .bti_set = mp->bt_tbl_id.bt_set,
+            .bti_sub_domain = mp->bt_tbl_id.bt_sub_domain,
+            .bti_hdr_len = mp->bt_tbl_id.bt_hdr_len_id,
+            .bti_type = BIER_TABLE_MPLS_SPF,
+            .bti_ecmp = BIER_ECMP_TABLE_ID_MAIN,
+        };
+
+        if (mp->bt_is_add)
+        {
+            mpls_label_t label = ntohl(mp->bt_label);
 
-    rv = vnm->api_errno;
+            /*
+             * convert acceptable 'don't want a label' values from 
+             * the API to the correct internal INVLID value
+             */
+            if ((0 == label) || (~0 == label))
+            {
+                label = MPLS_LABEL_INVALID;
+            }
+            bier_table_add_or_lock(&bti, label);
+        }
+        else
+        {
+            bier_table_unlock(&bti);
+        }
+
+        rv = vnm->api_errno;
+    }
 
     REPLY_MACRO (VL_API_BIER_TABLE_ADD_DEL_REPLY);
 }
@@ -161,10 +168,14 @@ vl_api_bier_route_add_del_t_handler (vl_api_bier_route_add_del_t * mp)
 
     vnm = vnet_get_main ();
     vnm->api_errno = 0;
-
     bp = ntohl(mp->br_bp);
     brpaths = NULL;
 
+    if (mp->br_tbl_id.bt_hdr_len_id >= BIER_HDR_LEN_2048)
+    {
+        rv = VNET_API_ERROR_BIER_BSL_UNSUP;
+        goto done;
+    }
     if (0 == bp || bp > BIER_BP_MAX)
     {
         rv = -1;
@@ -242,9 +253,9 @@ vl_api_bier_route_add_del_t_handler (vl_api_bier_route_add_del_t * mp)
     {
         bier_table_route_remove(&bti, bp, brpaths);
     }
+    vec_free(brpaths);
 
 done:
-    vec_free(brpaths);
     rv = (rv == 0) ? vnm->api_errno : rv;
 
     REPLY_MACRO (VL_API_BIER_ROUTE_ADD_DEL_REPLY);
@@ -333,26 +344,35 @@ vl_api_bier_imp_add_t_handler (vl_api_bier_imp_add_t * mp)
     vnm = vnet_get_main ();
     vnm->api_errno = 0;
 
-    bier_table_id_t bti = {
-        .bti_set = mp->bi_tbl_id.bt_set,
-        .bti_sub_domain = mp->bi_tbl_id.bt_sub_domain,
-        .bti_hdr_len = mp->bi_tbl_id.bt_hdr_len_id,
-        .bti_type = BIER_TABLE_MPLS_SPF,
-        .bti_ecmp = BIER_ECMP_TABLE_ID_MAIN,
-    };
-    bier_bit_string_t bs = {
-        .bbs_len = mp->bi_n_bytes,
-        .bbs_buckets = mp->bi_bytes,
-    };
-
-    bii = bier_imp_add_or_lock(&bti, ntohs(mp->bi_src), &bs);
+    /*
+     * The BSL support by VPP is limited to the size of the
+     * available space in the vlib_buffer_t
+     */
+    if (mp->bi_tbl_id.bt_hdr_len_id >= BIER_HDR_LEN_2048)
+    {
+        rv = VNET_API_ERROR_BIER_BSL_UNSUP;
+    }
+    else
+    {
+        bier_table_id_t bti = {
+            .bti_set = mp->bi_tbl_id.bt_set,
+            .bti_sub_domain = mp->bi_tbl_id.bt_sub_domain,
+            .bti_hdr_len = mp->bi_tbl_id.bt_hdr_len_id,
+            .bti_type = BIER_TABLE_MPLS_SPF,
+            .bti_ecmp = BIER_ECMP_TABLE_ID_MAIN,
+        };
+        bier_bit_string_t bs = {
+            .bbs_len = mp->bi_n_bytes,
+            .bbs_buckets = mp->bi_bytes,
+        };
+
+        bii = bier_imp_add_or_lock(&bti, ntohs(mp->bi_src), &bs);
+    }
 
-    /* *INDENT-OFF* */
     REPLY_MACRO2 (VL_API_BIER_IMP_ADD_REPLY,
     ({
         rmp->bi_index = ntohl (bii);
     }));
-    /* *INDENT-OM* */
 }
 
 static void
@@ -395,10 +415,9 @@ send_bier_imp_details (vl_api_registration_t * reg,
     mp->bi_tbl_id.bt_sub_domain = bi->bi_tbl.bti_sub_domain;
     mp->bi_tbl_id.bt_hdr_len_id = bi->bi_tbl.bti_hdr_len;
 
-
     mp->bi_src = htons(bier_hdr_get_src_id(&copy));
     mp->bi_n_bytes = n_bytes;
-    memcpy(mp->bi_bytes, bi->bi_bits.bits, n_bytes);
+    memcpy(mp->bi_bytes, bi->bi_bits, n_bytes);
 
     vl_api_send_msg (reg, (u8 *) mp);
 }
diff --git a/src/vnet/bier/bier_disp_table.c b/src/vnet/bier/bier_disp_table.c
index db4a2a82d2b..c663286efcb 100644
--- a/src/vnet/bier/bier_disp_table.c
+++ b/src/vnet/bier/bier_disp_table.c
@@ -210,7 +210,7 @@ bier_disp_table_lookup_hton(index_t bdti,
 {
     bier_hdr_src_id_t src = bp;
 
-    return (bier_disp_table_lookup(bdti, clib_host_to_net_u32(src)));
+    return (bier_disp_table_lookup(bdti, clib_host_to_net_u16(src)));
 }
 
 void
diff --git a/src/vnet/bier/bier_imp.c b/src/vnet/bier/bier_imp.c
index c51dede9c0a..bd934b77f16 100644
--- a/src/vnet/bier/bier_imp.c
+++ b/src/vnet/bier/bier_imp.c
@@ -157,7 +157,7 @@ format_bier_imp (u8* s, va_list *args)
         bier_hdr_ntoh(&copy);
         bier_bit_string_init(&bbs,
                              bier_hdr_get_len_id(&copy),
-                             bi->bi_bits.bits);
+                             bi->bi_bits);
 
         s = format(s, "\n%U%U",
                    format_white_space, indent,
diff --git a/src/vnet/bier/bier_imp.h b/src/vnet/bier/bier_imp.h
index fa53989fc93..5b21b06b12d 100644
--- a/src/vnet/bier/bier_imp.h
+++ b/src/vnet/bier/bier_imp.h
@@ -33,16 +33,6 @@
  */
 typedef struct bier_imp_t_ {
     /**
-     * The BIER table into which to forward the post imposed packet
-     */
-    bier_table_id_t bi_tbl;
-
-    /**
-     * number of locks
-     */
-    u32 bi_locks;
-
-    /**
      * The DPO contirubted from the resolving BIER table.
      * One per-IP protocol. This allows us to share a BIER imposition
      * object for a IPv4 and IPv6 mfib path.
@@ -60,7 +50,18 @@ typedef struct bier_imp_t_ {
      *  largest header type so as the bitstring is on the same
      *  cacheline as the header.
      */
-    bier_bit_mask_4096_t bi_bits;
+    u8 bi_bits[BIER_HDR_BUCKETS_1024];
+
+    /**
+     * The BIER table into which to forward the post imposed packet
+     */
+    bier_table_id_t bi_tbl;
+
+    /**
+     * number of locks
+     */
+    u32 bi_locks;
+
 } bier_imp_t;
 
 extern index_t bier_imp_add_or_lock(const bier_table_id_t *bt,
diff --git a/src/vnet/bier/bier_lookup.c b/src/vnet/bier/bier_lookup.c
index 817dcc6f3e8..4e544a3aca5 100644
--- a/src/vnet/bier/bier_lookup.c
+++ b/src/vnet/bier/bier_lookup.c
@@ -84,6 +84,7 @@ bier_lookup (vlib_main_t * vm,
     u32 n_left_from, next_index, * from, * to_next;
     bier_lookup_main_t *blm = &bier_lookup_main;
     u32 thread_index = vlib_get_thread_index();
+    bier_bit_mask_bucket_t buckets_copy[BIER_HDR_BUCKETS_4096];
 
     from = vlib_frame_vector_args (from_frame);
     n_left_from = from_frame->n_vectors;
@@ -98,7 +99,6 @@ bier_lookup (vlib_main_t * vm,
 
         while (n_left_from > 0 && n_left_to_next > 0)
         {
-            bier_bit_mask_bucket_t buckets_copy[BIER_HDR_BUCKETS_256];
             u32 next0, bi0, n_bytes, bti0, bfmi0;
             const bier_fmask_t *bfm0;
             const bier_table_t *bt0;
@@ -199,6 +199,9 @@ bier_lookup (vlib_main_t * vm,
                         /*
                          * go to the next bit-position set
                          */
+                        vlib_node_increment_counter(
+                            vm, node->node_index,
+                            BIER_LOOKUP_ERROR_FMASK_UNRES, 1);
                         bucket = ((int*)bbs.bbs_buckets)[index];
                         continue;
                     }
@@ -210,19 +213,19 @@ bier_lookup (vlib_main_t * vm,
              * Create the number of clones we need based on the number
              * of fmasks we are sending to.
              */
-            u8 num_cloned, clone;
+            u16 num_cloned, clone;
             u32 n_clones;
 
             n_clones = vec_len(blm->blm_fmasks[thread_index]);
 
             if (PREDICT_TRUE(0 != n_clones))
             {
-                ASSERT(n_clones < 256);
                 num_cloned = vlib_buffer_clone(vm, bi0,
                                                blm->blm_clones[thread_index],
-                                               n_clones, 128);
+                                               n_clones,
+                                               n_bytes + 8);
 
-                if (num_cloned != n_clones)
+                if (num_cloned != vec_len(blm->blm_fmasks[thread_index]))
                 {
                     vlib_node_increment_counter
                         (vm, node->node_index,
@@ -301,12 +304,12 @@ bier_lookup (vlib_main_t * vm,
             }
         }
 
-        vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+        vlib_put_next_frame(vm, node, next_index, n_left_to_next);
     }
 
-    vlib_node_increment_counter (vm, bier_lookup_node.index,
-                                 BIER_LOOKUP_ERROR_NONE,
-                                 from_frame->n_vectors);
+    vlib_node_increment_counter(vm, bier_lookup_node.index,
+                                BIER_LOOKUP_ERROR_NONE,
+                                from_frame->n_vectors);
     return (from_frame->n_vectors);
 }
 
@@ -355,11 +358,11 @@ bier_lookup_module_init (vlib_main_t * vm)
          thread_index++)
     {
         /*
-         *  4096 is the most we will ever need to support
-         * a Bit-Mask length of 4096
+         *  1024 is the most we will ever need to support
+         * a Bit-Mask length of 1024
          */
-        vec_validate(blm->blm_fmasks[thread_index], 4095);
-        vec_validate(blm->blm_clones[thread_index], 4095);
+        vec_validate(blm->blm_fmasks[thread_index], 1023);
+        vec_validate(blm->blm_clones[thread_index], 1023);
     }
 
     return 0;
diff --git a/src/vnet/bier/bier_table.c b/src/vnet/bier/bier_table.c
index 0f0f37677e9..80231fd8a72 100644
--- a/src/vnet/bier/bier_table.c
+++ b/src/vnet/bier/bier_table.c
@@ -562,20 +562,22 @@ bier_table_route_add (const bier_table_id_t *btid,
 }
 
 void
-bier_table_route_remove (const bier_table_id_t *bti,
+bier_table_route_remove (const bier_table_id_t *btid,
                          bier_bp_t bp,
                          fib_route_path_t *brps)
 {
     fib_route_path_t *brp = NULL;
+    index_t bfmi, bti, bei;
     bier_table_t *bt;
-    index_t bei;
+    u32 ii;
 
-    bt = bier_table_find(bti);
+    bt = bier_table_find(btid);
 
     if (NULL == bt) {
         return;
     }
 
+    bti = bier_table_get_index(bt);
     bei = bier_table_lookup(bt, bp);
 
     if (INDEX_INVALID == bei)
@@ -584,9 +586,34 @@ bier_table_route_remove (const bier_table_id_t *bti,
         return;
     }
 
-    vec_foreach(brp, brps)
+    /*
+     * set the FIB index in the path to the BIER table index
+     */
+    vec_foreach_index(ii, brps)
     {
-        brp->frp_bier_fib_index = bier_table_get_index(bt);
+        brp = &brps[ii];
+        bfmi = bier_fmask_db_find(bti, brp);
+
+        if (INDEX_INVALID == bfmi)
+        {
+            /*
+             * no matching fmask, not a path we can remove
+             */
+            vec_del1(brps, ii);
+            continue;
+        }
+
+        /*
+         * then modify the path to resolve via this fmask object
+         * and use it to resolve the BIER entry.
+         */
+        brp->frp_flags = FIB_ROUTE_PATH_BIER_FMASK;
+        brp->frp_bier_fmask = bfmi;
+    }
+
+    if (0 == vec_len(brps))
+    {
+        return;
     }
 
     if (0 == bier_entry_path_remove(bei, brps))
diff --git a/src/vnet/bier/bier_test.c b/src/vnet/bier/bier_test.c
index 0fbca7e7b9b..2c13abc2aea 100644
--- a/src/vnet/bier/bier_test.c
+++ b/src/vnet/bier/bier_test.c
@@ -310,7 +310,7 @@ bier_test_mpls_spf (void)
             .as_u32 = clib_host_to_net_u32(0x01010101),
         },
     };
-    fib_route_path_t *paths_1_1_1_1 = NULL;
+    fib_route_path_t *paths_1_1_1_1 = NULL, *input_paths_1_1_1_1;
     fib_route_path_t path_1_1_1_1 = {
         .frp_addr = nh_1_1_1_1,
         .frp_bier_fib_index = bti,
@@ -325,7 +325,8 @@ bier_test_mpls_spf (void)
     };
     index_t bei_1;
 
-    bier_table_route_add(&bt_0_0_0_256, 1, paths_1_1_1_1);
+    input_paths_1_1_1_1 = vec_dup(paths_1_1_1_1);
+    bier_table_route_add(&bt_0_0_0_256, 1, input_paths_1_1_1_1);
     bei_1 = bier_table_lookup(bier_table_get(bti), 1);
 
     BIER_TEST((INDEX_INVALID != bei_1), "BP:1 present");
@@ -477,7 +478,8 @@ bier_test_mpls_spf (void)
      */
     index_t bei_2;
 
-    bier_table_route_add(&bt_0_0_0_256, 2, paths_1_1_1_1);
+    input_paths_1_1_1_1 = vec_dup(paths_1_1_1_1);
+    bier_table_route_add(&bt_0_0_0_256, 2, input_paths_1_1_1_1);
     bei_2 = bier_table_lookup(bier_table_get(bti), 2);
 
     bier_entry_contribute_forwarding(bei_2, &dpo_bei);
@@ -498,13 +500,14 @@ bier_test_mpls_spf (void)
         .fp_len = 32,
         .fp_proto = FIB_PROTOCOL_IP4,
     };
-    fib_route_path_t *paths_1_1_1_2 = NULL, path_1_1_1_2 = {
+    fib_route_path_t *paths_1_1_1_2 = NULL, *input_paths_1_1_1_2, path_1_1_1_2 = {
         .frp_addr = nh_1_1_1_2,
         .frp_bier_fib_index = bti,
         .frp_sw_if_index = ~0,
     };
     vec_add1(path_1_1_1_2.frp_label_stack, 501);
     vec_add1(paths_1_1_1_2, path_1_1_1_2);
+    input_paths_1_1_1_2 = vec_dup(paths_1_1_1_2);
     index_t bei_3;
 
     mpls_label_t *out_lbl_101 = NULL;
@@ -520,7 +523,7 @@ bier_test_mpls_spf (void)
                                    1,
                                    out_lbl_101,
                                    FIB_ROUTE_PATH_FLAG_NONE);
-    bier_table_route_add(&bt_0_0_0_256, 3, paths_1_1_1_2);
+    bier_table_route_add(&bt_0_0_0_256, 3, input_paths_1_1_1_2);
     bei_3 = bier_table_lookup(bier_table_get(bti), 3);
 
     BIER_TEST((INDEX_INVALID != bei_3), "BP:3 present");
@@ -562,7 +565,8 @@ bier_test_mpls_spf (void)
      * Load-balance BP:3 over both next-hops
      */
     paths_1_1_1_1[0] = path_1_1_1_1;
-    bier_table_route_add(&bt_0_0_0_256, 3, paths_1_1_1_1);
+    input_paths_1_1_1_1 = vec_dup(paths_1_1_1_1);
+    bier_table_route_add(&bt_0_0_0_256, 3, input_paths_1_1_1_1);
 
     BIER_TEST(bier_test_validate_entry(bei_3, 2,
                                        &dpo_o_bfm_1_1_1_1,
@@ -630,7 +634,8 @@ bier_test_mpls_spf (void)
     /*
      * remove the original 1.1.1.2 fmask from BP:3
      */
-    bier_table_route_remove(&bt_0_0_0_256, 3, paths_1_1_1_2);
+    input_paths_1_1_1_2 = vec_dup(paths_1_1_1_2);
+    bier_table_route_remove(&bt_0_0_0_256, 3, input_paths_1_1_1_2);
     bier_entry_contribute_forwarding(bei_3, &dpo_bei);
     BIER_TEST((dpo_bei.dpoi_index == bfmi_1_1_1_1),
               "BP:3 stacks on fmask 1.1.1.1");
@@ -648,10 +653,14 @@ bier_test_mpls_spf (void)
     /*
      * remove the routes added
      */
-    bier_table_route_remove(&bt_0_0_0_256, 2, paths_1_1_1_1);
-    bier_table_route_remove(&bt_0_0_0_256, 3, paths_1_1_1_2);
-    bier_table_route_remove(&bt_0_0_0_256, 3, paths_1_1_1_1);
-    bier_table_route_remove(&bt_0_0_0_256, 1, paths_1_1_1_1);
+    input_paths_1_1_1_1 = vec_dup(paths_1_1_1_1);
+    bier_table_route_remove(&bt_0_0_0_256, 2, input_paths_1_1_1_1);
+    input_paths_1_1_1_2 = vec_dup(paths_1_1_1_2);
+    bier_table_route_remove(&bt_0_0_0_256, 3, input_paths_1_1_1_2);
+    input_paths_1_1_1_1 = vec_dup(paths_1_1_1_1);
+    bier_table_route_remove(&bt_0_0_0_256, 3, input_paths_1_1_1_1);
+    input_paths_1_1_1_1 = vec_dup(paths_1_1_1_1);
+    bier_table_route_remove(&bt_0_0_0_256, 1, input_paths_1_1_1_1);
 
     /*
      * delete the table
@@ -685,6 +694,8 @@ bier_test_mpls_spf (void)
 
     vec_free(paths_1_1_1_1);
     vec_free(paths_1_1_1_2);
+    vec_free(input_paths_1_1_1_1);
+    vec_free(input_paths_1_1_1_2);
 
     return (0);
 }
diff --git a/src/vnet/bier/bier_types.h b/src/vnet/bier/bier_types.h
index d484ba9ed03..ebddbe8d820 100644
--- a/src/vnet/bier/bier_types.h
+++ b/src/vnet/bier/bier_types.h
@@ -56,6 +56,11 @@ typedef enum bier_hdr_len_id_t_ {
     BIER_HDR_LEN_256,
     BIER_HDR_LEN_512,
     BIER_HDR_LEN_1024,
+    /**
+     * Bit-string lengths greater than 1024 are not supported due to the
+     * limited about pf space available in a vlib_buffer_t to prepend a
+     * BIER header at imposition.
+     */
     BIER_HDR_LEN_2048,
     BIER_HDR_LEN_4096,
     BIER_HDR_LEN_INVALID,
@@ -252,56 +257,6 @@ typedef enum bier_hdr_ctrl_sub_code_t_ {
 typedef u8 bier_bit_mask_bucket_t;
 
 /**
- * A BIER Bit-String value of length 64 bits.
- */
-typedef struct bier_bit_mask_64_t_ {
-    bier_bit_mask_bucket_t bits[BIER_HDR_BUCKETS_64];
-} bier_bit_mask_64_t;
-
-/**
- * A BIER Bit-String value of length 128 bits.
- */
-typedef struct bier_bit_mask_128_t_ {
-    bier_bit_mask_bucket_t bits[BIER_HDR_BUCKETS_128];
-} bier_bit_mask_128_t;
-
-/**
- * A BIER Bit-String value of length 256 bits.
- */
-typedef struct bier_bit_mask_256_t_ {
-    bier_bit_mask_bucket_t bits[BIER_HDR_BUCKETS_256];
-} bier_bit_mask_256_t;
-
-/**
- * A BIER Bit-String value of length 512 bits.
- */
-typedef struct bier_bit_mask_512_t_ {
-    bier_bit_mask_bucket_t bits[BIER_HDR_BUCKETS_512];
-} bier_bit_mask_512_t;
-
-/**
- * A BIER Bit-String value of length 1024 bits.
- */
-typedef struct bier_bit_mask_1024_t_ {
-    bier_bit_mask_bucket_t bits[BIER_HDR_BUCKETS_1024];
-} bier_bit_mask_1024_t;
-
-/**
- * A BIER Bit-String value of length 2048 bits.
- */
-typedef struct bier_bit_mask_2048_t_ {
-    bier_bit_mask_bucket_t bits[BIER_HDR_BUCKETS_2048];
-} bier_bit_mask_2048_t;
-
-/**
- * A BIER Bit-String value of length 4096 bits.
- */
-typedef struct bier_bit_mask_4096_t_ {
-    bier_bit_mask_bucket_t bits[BIER_HDR_BUCKETS_4096];
-} bier_bit_mask_4096_t;
-
-
-/**
  * 256 bits = 32 bytes
  */
 #define BIER_BIT_MASK_NUM_BUCKETS 32
@@ -333,13 +288,6 @@ typedef struct bier_bit_string_t_ {
 } bier_bit_string_t;
 
 /**
- * A BIER Bit-mask value
- *
- * The size of this mask represents this platforms BIER capabilities
- */
-typedef bier_bit_mask_256_t bier_bit_mask_t;
-
-/**
  * A bit positon
  *  as assigned to egress PEs
  */