jvpp: object model for jvpp generator (VPP-1184)

Introduces JSON parser which builds object model of Java API.

Also rewrites JNI translation of typedefs
to use per type translation functions
instead of code inlining.

Not covered:
- integrate with vappigen plugin (VPP-1154) or vapi parser (VPP-1155)
- use better templating engine (VPP-480)
- improvements of generator structure (e.g. VPP-1186)

Change-Id: I9e12d76c2f3c6ee041669f58e8a37917f656aa90
Signed-off-by: Marek Gradzki <mgradzki@cisco.com>

1 files changed, 128 insertions, 264 deletions

diff --git a/src/vpp-api/java/jvpp/gen/jvppgen/jni_gen.py b/src/vpp-api/java/jvpp/gen/jvppgen/jni_gen.py
index 8c8be5b8877..a20a90d4720 100644..100755
--- a/src/vpp-api/java/jvpp/gen/jvppgen/jni_gen.py
+++ b/src/vpp-api/java/jvpp/gen/jvppgen/jni_gen.py
@@ -1,6 +1,6 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 #
-# Copyright (c) 2016 Cisco and/or its affiliates.
+# Copyright (c) 2016,2018 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -12,285 +12,149 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+#
 from string import Template
 
-import util
-
-variable_length_array_value_template = Template("""mp->${length_var_name}""")
-variable_length_array_template = Template("""clib_net_to_host_${length_field_type}(${value})""")
+from jni_impl_gen import generate_jni_impl
+from jni_msg_handlers_gen import generate_jni_handlers
+from jni_type_handlers_gen import generate_type_handlers
+from jvpp_model import is_control_ping, is_dump, is_request, is_control_ping_reply
 
-dto_field_id_template = Template("""
-    jfieldID ${field_reference_name}FieldId = (*env)->GetFieldID(env, ${class_ref_name}Class, "${field_name}", "${jni_signature}");""")
 
-default_dto_field_setter_template = Template("""
-    (*env)->Set${jni_setter}(env, ${object_name}, ${field_reference_name}FieldId, mp->${c_name});
-""")
-
-u16_dto_field_setter_template = Template("""
-    (*env)->Set${jni_setter}(env, ${object_name}, ${field_reference_name}FieldId, clib_net_to_host_u16(mp->${c_name}));
-""")
+def generate_jni(work_dir, model, logger):
+    logger.debug("Generating jvpp C for %s" % model.json_api_files)
+    plugin_name = model.plugin_name
+    messages = model.messages
 
-u32_dto_field_setter_template = Template("""
-    (*env)->Set${jni_setter}(env, ${object_name}, ${field_reference_name}FieldId, clib_net_to_host_u32(mp->${c_name}));
-""")
+    with open("%s/jvpp_%s_gen.h" % (work_dir, plugin_name), "w") as f:
+        f.write(_JVPP_C_TEMPLATE.substitute(
+            json_filename=model.json_api_files,
+            class_cache=_generate_class_cache(plugin_name, messages),
+            api_verification=_generate_api_verification(messages),
+            type_handlers=generate_type_handlers(model),
+            jni_implementations=generate_jni_impl(model),
+            msg_handlers=generate_jni_handlers(model),
+            handler_registration=_generate_handler_registration(messages)))
 
-u64_dto_field_setter_template = Template("""
-    (*env)->Set${jni_setter}(env, ${object_name}, ${field_reference_name}FieldId, clib_net_to_host_u64(mp->${c_name}));
-""")
+_JVPP_C_TEMPLATE = Template("""/**
+ * This file contains JNI bindings for jvpp Java API.
+ * It was generated by jvpp_jni_gen.py based on $json_filename.
+ */
+$class_cache
 
-u8_array_dto_field_setter_template = Template("""
-    jbyteArray ${field_reference_name} = (*env)->NewByteArray(env, ${field_length});
-    (*env)->SetByteArrayRegion(env, ${field_reference_name}, 0, ${field_length}, (const jbyte*)mp->${c_name});
-    (*env)->SetObjectField(env, ${object_name}, ${field_reference_name}FieldId, ${field_reference_name});
-    (*env)->DeleteLocalRef(env, ${field_reference_name});
-""")
+$api_verification
 
-u16_array_dto_field_setter_template = Template("""
-    {
-        jshortArray ${field_reference_name} = (*env)->NewShortArray(env, ${field_length});
-        jshort * ${field_reference_name}ArrayElements = (*env)->GetShortArrayElements(env, ${field_reference_name}, NULL);
-        unsigned int _i;
-        for (_i = 0; _i < ${field_length}; _i++) {
-            ${field_reference_name}ArrayElements[_i] = clib_net_to_host_u16(mp->${c_name}[_i]);
-        }
+// Type handlers
+$type_handlers
 
-        (*env)->ReleaseShortArrayElements(env,  ${field_reference_name}, ${field_reference_name}ArrayElements, 0);
-        (*env)->SetObjectField(env, ${object_name}, ${field_reference_name}FieldId, ${field_reference_name});
-        (*env)->DeleteLocalRef(env, ${field_reference_name});
-    }
-""")
+// JNI bindings
+$jni_implementations
 
-u32_array_dto_field_setter_template = Template("""
-    {
-        jintArray ${field_reference_name} = (*env)->NewIntArray(env, ${field_length});
-        jint * ${field_reference_name}ArrayElements = (*env)->GetIntArrayElements(env, ${field_reference_name}, NULL);
-        unsigned int _i;
-        for (_i = 0; _i < ${field_length}; _i++) {
-            ${field_reference_name}ArrayElements[_i] = clib_net_to_host_u32(mp->${c_name}[_i]);
-        }
+// Message handlers
+$msg_handlers
 
-        (*env)->ReleaseIntArrayElements(env,  ${field_reference_name}, ${field_reference_name}ArrayElements, 0);
-        (*env)->SetObjectField(env, ${object_name}, ${field_reference_name}FieldId, ${field_reference_name});
-        (*env)->DeleteLocalRef(env, ${field_reference_name});
-    }
+$handler_registration
 """)
 
-# For each u64 array we get its elements. Then we convert values to host byte order.
-# All changes to  jlong* buffer are written to jlongArray (isCopy is set to NULL)
-u64_array_dto_field_setter_template = Template("""
-    {
-        jlongArray ${field_reference_name} = (*env)->NewLongArray(env, ${field_length});
-        jlong * ${field_reference_name}ArrayElements = (*env)->GetLongArrayElements(env, ${field_reference_name}, NULL);
-        unsigned int _i;
-        for (_i = 0; _i < ${field_length}; _i++) {
-            ${field_reference_name}ArrayElements[_i] = clib_net_to_host_u64(mp->${c_name}[_i]);
-        }
 
-        (*env)->ReleaseLongArrayElements(env,  ${field_reference_name}, ${field_reference_name}ArrayElements, 0);
-        (*env)->SetObjectField(env, ${object_name}, ${field_reference_name}FieldId, ${field_reference_name});
-        (*env)->DeleteLocalRef(env, ${field_reference_name});
-    }
+def _generate_class_cache(plugin_name, messages):
+    references = []
+    for msg in messages:
+        if is_control_ping(msg) or is_control_ping_reply(msg):
+            # Skip control_ping managed by jvpp registry.
+            continue
+        references.append((
+            msg.java_name_lower,
+            'io/fd/vpp/jvpp/%s/dto/%s' % (plugin_name, msg.java_name_upper)
+        ))
+
+    references.append(('callbackException', 'io/fd/vpp/jvpp/VppCallbackException'))
+
+    return _CLASS_CACHE_TEMPLATE.substitute(
+        class_references=_generate_class_references(references),
+        create_references=_generate_create_references(references),
+        delete_references=_generate_delete_references(references)
+    )
+
+_CLASS_CACHE_TEMPLATE = Template("""
+// JAVA class reference cache
+$class_references
+
+static int cache_class_references(JNIEnv* env) {
+$create_references
+    return 0;
+}
+
+static void delete_class_references(JNIEnv* env) {
+$delete_references
+}""")
+
+
+def _generate_class_references(references):
+    return "\n".join("jclass %sClass;" % r[0] for r in references)
+
+
+def _generate_create_references(references):
+    items = []
+    for r in references:
+        items.append(_CREATE_GLOBAL_REF_TEMPLATE.substitute(
+            ref_name=r[0],
+            fqn_name=r[1]
+        ))
+    return "".join(items)
+
+_CREATE_GLOBAL_REF_TEMPLATE = Template("""
+    ${ref_name}Class = (jclass)(*env)->NewGlobalRef(env, (*env)->FindClass(env, "${fqn_name}"));
+    if ((*env)->ExceptionCheck(env)) {
+        (*env)->ExceptionDescribe(env);
+        return JNI_ERR;
+    }""")
+
+
+def _generate_delete_references(references):
+    items = []
+    for r in references:
+        items.append(_DELETE_CLASS_INVOCATION_TEMPLATE.substitute(ref_name=r[0]))
+    return "".join(items)
+
+_DELETE_CLASS_INVOCATION_TEMPLATE = Template("""
+    if (${ref_name}Class) {
+        (*env)->DeleteGlobalRef(env, ${ref_name}Class);
+    }""")
+
+
+def _generate_api_verification(messages):
+    items = []
+    for msg in messages:
+        items.append("_(%s_%s) \\" % (msg.name, msg.crc))
+    return _API_VERIFICATION_TEMPLATE.substitute(messages="\n".join(items))
+
+_API_VERIFICATION_TEMPLATE = Template("""
+// List of supported API messages used for verification
+#define foreach_supported_api_message \\
+$messages
 """)
 
-dto_field_setter_templates = {'u8': default_dto_field_setter_template,
-                              'u16': u16_dto_field_setter_template,
-                              'u32': u32_dto_field_setter_template,
-                              'i32': u32_dto_field_setter_template,
-                              'u64': u64_dto_field_setter_template,
-                              'f64': default_dto_field_setter_template,  # fixme
-                              'u8[]': u8_array_dto_field_setter_template,
-                              'u16[]': u16_array_dto_field_setter_template,
-                              'u32[]': u32_array_dto_field_setter_template,
-                              'u64[]': u64_array_dto_field_setter_template
-                              }
-
 
-def jni_reply_handler_for_type(handler_name, ref_name, field_type, c_name, field_reference_name,
-                               field_name, field_length, is_variable_len_array, length_field_type,
-                               object_name="dto"):
+def _generate_handler_registration(messages):
     """
-    Generates jni code that initializes a field of java object (dto or custom type).
-    To be used in reply message handlers.
-    :param field_type: type of the field to be initialized (as defined in vpe.api)
-    :param c_name: name of the message struct member that stores initialization value
-    :param field_reference_name: name of the field reference in generated code
-    :param field_name: name of the field (camelcase)
-    :param field_length: integer or name of variable that stores field length
-    :param object_name: name of the object to be initialized
+    Generates msg handler registration for all messages except for dumps and requests.
+    :param messages: collection of VPP API messages.
     """
-
-    # todo move validation to vppapigen
-    if field_type.endswith('[]') and field_length == '0':
-        raise Exception('Variable array \'%s\' defined in \'%s\' '
-                        'should have defined length (e.g. \'%s[%s_length]\''
-                        % (c_name, handler_name, c_name, c_name))
-
-    if is_variable_len_array:
-        length_var_name = field_length
-        field_length = variable_length_array_value_template.substitute(length_var_name=length_var_name)
-        if length_field_type != 'u8':  # we need net to host conversion:
-            field_length = variable_length_array_template.substitute(
-                    length_field_type=length_field_type, value=field_length)
-
-    # for retval don't generate setters
-    if util.is_retval_field(c_name):
-        return ""
-
-    jni_signature = util.jni_2_signature_mapping[field_type]
-    jni_setter = util.jni_field_accessors[field_type]
-
-    result = dto_field_id_template.substitute(
-            field_reference_name=field_reference_name,
-            field_name=field_name,
-            class_ref_name=ref_name,
-            jni_signature=jni_signature)
-
-    dto_setter_template = dto_field_setter_templates[field_type]
-
-    result += dto_setter_template.substitute(
-            jni_signature=jni_signature,
-            object_name=object_name,
-            field_reference_name=field_reference_name,
-            c_name=c_name,
-            jni_setter=jni_setter,
-            field_length=field_length)
-    return result
-
-
-request_field_identifier_template = Template("""
-    jfieldID ${field_reference_name}FieldId = (*env)->GetFieldID(env, ${object_name}Class, "${field_name}", "${jni_signature}");
-    ${jni_type} ${field_reference_name} = (*env)->Get${jni_getter}(env, ${object_name}, ${field_reference_name}FieldId);
-    """)
-
-array_length_enforcement_template = Template("""
-        size_t max_size = ${field_length};
-        if (cnt > max_size) cnt = max_size;""")
-
-u8_struct_setter_template = Template("""
-    mp->${c_name} = ${field_reference_name};""")
-
-u16_struct_setter_template = Template("""
-    mp->${c_name} = clib_host_to_net_u16(${field_reference_name});""")
-
-u32_struct_setter_template = Template("""
-    mp->${c_name} = clib_host_to_net_u32(${field_reference_name});""")
-
-i32_struct_setter_template = Template("""
-    mp->${c_name} = clib_host_to_net_i32(${field_reference_name});!""")
-
-u64_struct_setter_template = Template("""
-    mp->${c_name} = clib_host_to_net_u64(${field_reference_name});""")
-
-u8_array_struct_setter_template = Template("""
-    if (${field_reference_name}) {
-        jsize cnt = (*env)->GetArrayLength (env, ${field_reference_name});
-        ${field_length_check}
-        (*env)->GetByteArrayRegion(env, ${field_reference_name}, 0, cnt, (jbyte *)mp->${c_name});
-    }
+    handlers = []
+    for msg in messages:
+        if is_control_ping(msg) or is_control_ping_reply(msg):
+            # Skip control_ping managed by jvpp registry.
+            continue
+        if is_dump(msg) or is_request(msg):
+            continue
+        name = msg.name
+        crc = msg.crc
+        handlers.append("_(%s_%s, %s) \\" % (name, crc, name))
+    return _HANDLER_REGISTRATION_TEMPLATE.substitute(handlers="\n".join(handlers))
+
+_HANDLER_REGISTRATION_TEMPLATE = Template("""
+// Registration of message handlers in vlib
+#define foreach_api_reply_handler \\
+$handlers
 """)
-
-u16_array_struct_setter_template = Template("""
-    if (${field_reference_name}) {
-        jshort * ${field_reference_name}ArrayElements = (*env)->GetShortArrayElements(env, ${field_reference_name}, NULL);
-        size_t _i;
-        jsize cnt = (*env)->GetArrayLength (env, ${field_reference_name});
-        ${field_length_check}
-        for (_i = 0; _i < cnt; _i++) {
-            mp->${c_name}[_i] = clib_host_to_net_u16(${field_reference_name}ArrayElements[_i]);
-        }
-        (*env)->ReleaseShortArrayElements (env, ${field_reference_name}, ${field_reference_name}ArrayElements, 0);
-    }
-    """)
-
-u32_array_struct_setter_template = Template("""
-    if (${field_reference_name}) {
-        jint * ${field_reference_name}ArrayElements = (*env)->GetIntArrayElements(env, ${field_reference_name}, NULL);
-        size_t _i;
-        jsize cnt = (*env)->GetArrayLength (env, ${field_reference_name});
-        ${field_length_check}
-        for (_i = 0; _i < cnt; _i++) {
-            mp->${c_name}[_i] = clib_host_to_net_u32(${field_reference_name}ArrayElements[_i]);
-        }
-        (*env)->ReleaseIntArrayElements (env, ${field_reference_name}, ${field_reference_name}ArrayElements, 0);
-    }
-    """)
-
-u64_array_struct_setter_template = Template("""
-    if (${field_reference_name}) {
-        jlong * ${field_reference_name}ArrayElements = (*env)->GetLongArrayElements(env, ${field_reference_name}, NULL);
-        size_t _i;
-        jsize cnt = (*env)->GetArrayLength (env, ${field_reference_name});
-        ${field_length_check}
-        for (_i = 0; _i < cnt; _i++) {
-            mp->${c_name}[_i] = clib_host_to_net_u64(${field_reference_name}ArrayElements[_i]);
-        }
-        (*env)->ReleaseLongArrayElements (env, ${field_reference_name}, ${field_reference_name}ArrayElements, 0);
-    }
-    """)
-
-struct_setter_templates = {'u8': u8_struct_setter_template,
-                           'u16': u16_struct_setter_template,
-                           'u32': u32_struct_setter_template,
-                           'i32': u32_struct_setter_template,
-                           'u64': u64_struct_setter_template,
-                           'u8[]': u8_array_struct_setter_template,
-                           'u16[]': u16_array_struct_setter_template,
-                           'u32[]': u32_array_struct_setter_template,
-                           'u64[]': u64_array_struct_setter_template
-                           }
-
-
-def jni_request_identifiers_for_type(field_type, field_reference_name, field_name, object_name="request"):
-    """
-    Generates jni code that defines C variable corresponding to field of java object
-    (dto or custom type). To be used in request message handlers.
-    :param field_type: type of the field to be initialized (as defined in vpe.api)
-    :param field_reference_name: name of the field reference in generated code
-    :param field_name: name of the field (camelcase)
-    :param object_name: name of the object to be initialized
-    """
-    # field identifiers
-    jni_type = util.vpp_2_jni_type_mapping[field_type]
-    jni_signature = util.jni_2_signature_mapping[field_type]
-    jni_getter = util.jni_field_accessors[field_type]
-
-    # field identifier
-    return request_field_identifier_template.substitute(
-            jni_type=jni_type,
-            field_reference_name=field_reference_name,
-            field_name=field_name,
-            jni_signature=jni_signature,
-            jni_getter=jni_getter,
-            object_name=object_name)
-
-
-def jni_request_binding_for_type(field_type, c_name, field_reference_name, field_length, is_variable_len_array):
-    """
-    Generates jni code that initializes C structure that corresponds to a field of java object
-    (dto or custom type). To be used in request message handlers.
-    :param field_type: type of the field to be initialized (as defined in vpe.api)
-    :param c_name: name of the message struct member to be initialized
-    :param field_reference_name: name of the field reference in generated code
-    :param field_length: integer or name of variable that stores field length
-    """
-
-    # field setter
-    field_length_check = ""
-
-    # check if we are processing variable length array:
-    if is_variable_len_array:
-        field_length = util.underscore_to_camelcase(field_length)
-
-    # enforce max length if array has fixed length or uses variable length syntax
-    if str(field_length) != "0":
-        field_length_check = array_length_enforcement_template.substitute(field_length=field_length)
-
-    struct_setter_template = struct_setter_templates[field_type]
-
-    msg_initialization = struct_setter_template.substitute(
-            c_name=c_name,
-            field_reference_name=field_reference_name,
-            field_length_check=field_length_check)
-
-    return msg_initialization