VPP-118: add support for variable length arrays to jvpp

* extends VPP's message definition language with the following syntax:

u32 count:
u8 array[count];

which is traslated to:

u32 count;
u8 array[0];

but now, python API representation generated by vppapigen
contains information about where the array length is stored.

* modifies existing response messages to use the new syntax

Change-Id: I68210bc7a3a755d03d067e9b79a567f40e2d31f3
Signed-off-by: Marek Gradzki <mgradzki@cisco.com>

7 files changed, 54 insertions, 9 deletions

diff --git a/vpp-api/java/jvpp/gen/jvpp_c_gen.py b/vpp-api/java/jvpp/gen/jvpp_c_gen.py
index c006c34103b..0b7fc67d41f 100644
--- a/vpp-api/java/jvpp/gen/jvpp_c_gen.py
+++ b/vpp-api/java/jvpp/gen/jvpp_c_gen.py
@@ -210,7 +210,10 @@ def generate_jni_impl(func_list, inputfile):
             for t in zip(f['c_types'], f['args'], f['lengths']):
                 c_type = t[0]
                 c_name = t[1]
-                field_length = t[2]
+                # variable length arrays do not need special handling in requests,
+                # because the length of Java array is known:
+                field_length = t[2][0]
+
                 java_field_name = util.underscore_to_camelcase(c_name)
 
                 struct_setter_template = struct_setter_templates[c_type]
@@ -241,6 +244,9 @@ default_dto_field_setter_template = Template("""
     (*env)->Set${jni_setter}(env, dto, ${java_name}FieldId, mp->${c_name});
 """)
 
+variable_length_array_value_template = Template("""mp->${length_var_name}""")
+variable_length_array_template = Template("""clib_net_to_host_${length_field_type}(${value})""")
+
 u16_dto_field_setter_template = Template("""
     (*env)->Set${jni_setter}(env, dto, ${java_name}FieldId, clib_net_to_host_u16(mp->${c_name}));
 """)
@@ -267,6 +273,8 @@ u32_array_dto_field_setter_template = Template("""
         for (_i = 0; _i < ${field_length}; _i++) {
             ${java_name}ArrayElements[_i] = clib_net_to_host_u32(mp->${c_name}[_i]);
         }
+
+        (*env)->ReleaseIntArrayElements(env,  ${java_name}, ${java_name}ArrayElements, 0);
         (*env)->SetObjectField(env, dto, ${java_name}FieldId, ${java_name});
     }
 """)
@@ -281,6 +289,8 @@ u64_array_dto_field_setter_template = Template("""
         for (_i = 0; _i < ${field_length}; _i++) {
             ${java_name}ArrayElements[_i] = clib_net_to_host_u64(mp->${c_name}[_i]);
         }
+
+        (*env)->ReleaseLongArrayElements(env,  ${java_name}, ${java_name}ArrayElements, 0);
         (*env)->SetObjectField(env, dto, ${java_name}FieldId, ${java_name});
     }
 """)
@@ -353,7 +363,16 @@ def generate_msg_handlers(func_list, inputfile):
             c_type = t[0]
             jni_type = t[1]
             c_name = t[2]
-            field_length = t[3]
+            field_length = t[3][0]
+
+            # check if we are processing variable length array
+            if t[3][1]:
+                length_var_name = t[3][0]
+                length_field_type = f['c_types'][f['args'].index(length_var_name)]
+                field_length = variable_length_array_value_template.substitute(length_var_name=length_var_name)
+                if length_field_type != 'u8':  # we need net to host conversion:
+                    field_length = variable_length_array_template.substitute(
+                        length_field_type=length_field_type, value=field_length)
 
             # for retval don't generate setters and generate retval check
             if util.is_retval_field(c_name):
diff --git a/vpp-api/java/jvpp/gen/jvpp_gen.py b/vpp-api/java/jvpp/gen/jvpp_gen.py
index c08593e180f..26bcea1523a 100755
--- a/vpp-api/java/jvpp/gen/jvpp_gen.py
+++ b/vpp-api/java/jvpp/gen/jvpp_gen.py
@@ -80,11 +80,15 @@ def get_types(t, filter):
         if len(i) is 3:  # array type
             types_list.append(vpp_2_jni_type_mapping[i[0]] + 'Array')
             c_types_list.append(i[0] + '[]')
-            lengths_list.append(i[2])
+            lengths_list.append((i[2], False))
+        elif len(i) is 4:  # variable length array type
+            types_list.append(vpp_2_jni_type_mapping[i[0]] + 'Array')
+            c_types_list.append(i[0] + '[]')
+            lengths_list.append((i[3], True))
         else:  # primitive type
             types_list.append(vpp_2_jni_type_mapping[i[0]])
             c_types_list.append(i[0])
-            lengths_list.append(0)
+            lengths_list.append((0, False))
     return types_list, c_types_list, lengths_list
 
 
diff --git a/vpp-api/python/pneum/api-gen.py b/vpp-api/python/pneum/api-gen.py
index df92c196fe2..081fb983539 100755
--- a/vpp-api/python/pneum/api-gen.py
+++ b/vpp-api/python/pneum/api-gen.py
@@ -67,7 +67,7 @@ def get_pack(t):
     j = -1
     for i in t:
         j += 1
-        if len(i) is 3:
+        if len(i) is 3 or len(i) is 4:  # TODO: add support for variable length arrays (VPP-162)
             size = type_size[i[0]]
             bytecount += size * int(i[2])
             # Check if we have a zero length array
diff --git a/vpp/api/vpe.api b/vpp/api/vpe.api
index 24bd0ae6865..f91dd8712db 100644
--- a/vpp/api/vpe.api
+++ b/vpp/api/vpe.api
@@ -643,7 +643,7 @@ typeonly manual_print manual_endian define ip4_fib_counter {
 manual_java manual_print manual_endian define vnet_ip4_fib_counters {
     u32 vrf_id;
     u32 count;
-    vl_api_ip4_fib_counter_t c[0];
+    vl_api_ip4_fib_counter_t c[count];
 };
 
 typeonly manual_print manual_endian define ip6_fib_counter {
@@ -656,7 +656,7 @@ typeonly manual_print manual_endian define ip6_fib_counter {
 manual_java manual_print manual_endian define vnet_ip6_fib_counters {
     u32 vrf_id;
     u32 count;
-    vl_api_ip6_fib_counter_t c[0];
+    vl_api_ip6_fib_counter_t c[count];
 };
 
 /** \brief Request for a single block of summary stats
diff --git a/vppapigen/gram.y b/vppapigen/gram.y
index 09e20067128..ccdcc8f2573 100644
--- a/vppapigen/gram.y
+++ b/vppapigen/gram.y
@@ -31,6 +31,7 @@ void generate (YYSTYPE);
  YYSTYPE add_union(YYSTYPE, YYSTYPE);
  YYSTYPE add_scalar_vbl(YYSTYPE);
  YYSTYPE add_vector_vbl(YYSTYPE, YYSTYPE);
+ YYSTYPE add_variable_length_vector_vbl(YYSTYPE, YYSTYPE);
  YYSTYPE set_flags(YYSTYPE, YYSTYPE);
 %}
 
@@ -86,4 +87,5 @@ onedef:   PRIMTYPE vbl SEMI      {$$ = add_primtype($1, $2, 0);}
 
 vbl:      NAME                      {$$ = add_scalar_vbl($1);}
         | NAME LBRACK NUMBER RBRACK {$$ = add_vector_vbl($1, $3);}
+        | NAME LBRACK NAME RBRACK {$$ = add_variable_length_vector_vbl($1, $3);}
           ;
diff --git a/vppapigen/node.c b/vppapigen/node.c
index 2fb65c2d60b..30a3b017190 100644
--- a/vppapigen/node.c
+++ b/vppapigen/node.c
@@ -1161,7 +1161,11 @@ void node_vector_generate (node_t *this, enum passid which, FILE *fp)
         fprintf(fp, "}\n");
         break;
     case PYTHON_PASS:
-        fprintf(fp, "'%s', '%d'),\n", CDATA0, IDATA1);
+        if (CDATA2 != 0) { // variable length vector
+            fprintf(fp, "'%s', '%d', '%s'),\n", CDATA0, IDATA1, CDATA2);
+        } else {
+            fprintf(fp, "'%s', '%d'),\n", CDATA0, IDATA1);
+        }
         break;
 
     default:
@@ -1461,6 +1465,21 @@ YYSTYPE add_vector_vbl (YYSTYPE a1, YYSTYPE a2)
 }
 
 /*
+ * add_vector_vbl (char *vector_name, char *vector_length_var)
+ */
+
+YYSTYPE add_variable_length_vector_vbl (YYSTYPE vector_name, YYSTYPE vector_length_var)
+{
+    node_t *np;
+
+    np = make_node(NODE_VECTOR);
+    np->data[0] = (void *) vector_name;
+    np->data[1] = (void *) 0; // vector size used for vpe.api.h generation (array of length zero)
+    np->data[2] = (void *) vector_length_var; // name of the variable that stores vector length
+    return ((YYSTYPE) np);
+}
+
+/*
  * add_scalar_vbl (char *name)
  */
 YYSTYPE add_scalar_vbl (YYSTYPE a1)
diff --git a/vppapigen/node.h b/vppapigen/node.h
index 1f5a153c24f..fc331507485 100644
--- a/vppapigen/node.h
+++ b/vppapigen/node.h
@@ -73,12 +73,13 @@ typedef struct node_ {
     struct node_ *peer;
     struct node_ *deeper;
     int flags;
-    void *data[2];
+    void *data[3];
 } node_t;
 
 /* To shut up gcc-4.2.x warnings */
 #define CDATA0 ((char *)(this->data[0]))
 #define IDATA1 ((int)(uword)(this->data[1]))
+#define CDATA2 ((char *)(this->data[2]))
 
 #define NODE_FLAG_MANUAL_PRINT (1<<0)
 #define NODE_FLAG_MANUAL_ENDIAN (1<<1)