jvpp: object model for jvpp generator (VPP-1184)

Introduces JSON parser which builds object model of Java API.

Also rewrites JNI translation of typedefs
to use per type translation functions
instead of code inlining.

Not covered:
- integrate with vappigen plugin (VPP-1154) or vapi parser (VPP-1155)
- use better templating engine (VPP-480)
- improvements of generator structure (e.g. VPP-1186)

Change-Id: I9e12d76c2f3c6ee041669f58e8a37917f656aa90
Signed-off-by: Marek Gradzki <mgradzki@cisco.com>

1 files changed, 174 insertions, 263 deletions

diff --git a/src/vpp-api/java/jvpp/gen/jvppgen/dto_gen.py b/src/vpp-api/java/jvpp/gen/jvppgen/dto_gen.py
index 035a5b49eb8..ca015a30456 100644..100755
--- a/src/vpp-api/java/jvpp/gen/jvppgen/dto_gen.py
+++ b/src/vpp-api/java/jvpp/gen/jvppgen/dto_gen.py
@@ -1,6 +1,6 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 #
-# Copyright (c) 2016 Cisco and/or its affiliates.
+# Copyright (c) 2016,2018 Cisco and/or its affiliates.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
@@ -13,160 +13,196 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
 from string import Template
 
-import util
+from jvpp_common_gen import generate_hash_code, generate_equals, generate_to_string, generate_fields
+from jvpp_model import is_request, is_reply, is_retval, is_dump, is_details, is_event, is_control_ping, \
+    is_control_ping_reply
 
-dto_template = Template("""
-package $plugin_package.$dto_package;
 
-/**
- * <p>This class represents $description.
- * <br>It was generated by dto_gen.py based on $inputfile preparsed data:
- * <pre>
-$docs
- * </pre>
- */
-public final class $cls_name implements $base_package.$dto_package.$base_type {
+def generate_dtos(work_dir, model, logger):
+    logger.debug("Generating DTOs for %s " % model.json_api_files)
+    _generate_message_dtos(work_dir, model, logger)
+    _generate_dump_reply_wrappers(work_dir, model, logger)
 
-$fields
-$methods
-}
-""")
 
-dto_template_typeless = Template("""
-package $plugin_package.$dto_package;
+def _generate_message_dtos(work_dir, model, logger):
+    for msg in model.messages:
+        logger.debug("Generating DTO for message %s", msg)
+        class_name = msg.java_name_upper
+
+        if is_control_ping(msg) or is_control_ping_reply(msg):
+            # Skip control_ping managed by jvpp registry.
+            continue
+        if is_request(msg):
+            dto = _generate_request_dto(msg, model, base_type="JVppRequest")
+        elif is_dump(msg):
+            dto = _generate_request_dto(msg, model, base_type="JVppDump")
+        elif is_reply(msg) or is_details(msg):
+            dto = _generate_reply_dto(msg, model)
+        elif is_event(msg):
+            dto = _generate_event_dto(msg, model)
+        else:
+            logger.warn("Failed to generate DTO for: %s. Message type is not supported." % msg)
+            continue
+        with open("%s/%s.java" % (work_dir, class_name), "w") as f:
+            f.write(dto)
+
+
+def _generate_request_dto(msg, model, base_type):
+    msg_java_name_upper = msg.java_name_upper
+    fields = msg.fields
+    return _REQUEST_TEMPLATE.substitute(
+        plugin_package=model.plugin_package,
+        json_filename=model.json_api_files,
+        json_definition=msg.doc,
+        class_name=msg_java_name_upper,
+        base_type=base_type,
+        fields=generate_fields(fields),
+        hash_code=generate_hash_code(fields),
+        equals=generate_equals(msg_java_name_upper, fields),
+        to_string=generate_to_string(msg_java_name_upper, fields),
+        send=_generate_send(model, msg))
+
+_REQUEST_TEMPLATE = Template("""
+package $plugin_package.dto;
 
 /**
- * <p>This class represents $description.
- * <br>It was generated by dto_gen.py based on $inputfile preparsed data:
+ * <p>This class represents request DTO.
+ * <br>It was generated by dto_gen.py based on $json_filename:
  * <pre>
-$docs
+$json_definition
  * </pre>
  */
-public final class $cls_name {
-
+public final class $class_name implements io.fd.vpp.jvpp.dto.$base_type {
 $fields
-$methods
+$hash_code
+$equals
+$to_string
+$send
 }
 """)
 
-field_template = Template("""    public $type $name;\n""")
 
-send_template = Template("""    @Override
-    public int send(final $base_package.JVpp jvpp) throws io.fd.vpp.jvpp.VppInvocationException {
+def _generate_send(model, msg):
+    return _SEND_TEMPLATE.substitute(
+        plugin_package=model.plugin_package,
+        plugin_name=model.plugin_java_name,
+        method_name=msg.java_name_lower,
+        args="this" if msg.has_fields else ""
+    )
+
+_SEND_TEMPLATE = Template("""
+    @Override
+    public int send(final io.fd.vpp.jvpp.JVpp jvpp) throws io.fd.vpp.jvpp.VppInvocationException {
         return (($plugin_package.JVpp${plugin_name})jvpp).$method_name($args);
     }""")
 
 
-def generate_dtos(func_list, base_package, plugin_package, plugin_name, dto_package, inputfile, logger):
-    """ Generates dto objects in a dedicated package """
-    logger.debug("Generating DTOs for %s" % inputfile)
-
-    if not os.path.exists(dto_package):
-        os.mkdir(dto_package)
-
-    for func in func_list:
-        camel_case_dto_name = util.underscore_to_camelcase_upper(func['name'])
-        camel_case_method_name = util.underscore_to_camelcase(func['name'])
-        dto_path = os.path.join(dto_package, camel_case_dto_name + ".java")
-
-        if util.is_control_ping(camel_case_dto_name):
-            continue
-
-        fields = generate_dto_fields(camel_case_dto_name, func)
-        methods = generate_dto_base_methods(camel_case_dto_name, func)
-        base_type = ""
-
-        if util.is_reply(camel_case_dto_name):
-            description = "reply DTO"
-            request_dto_name = util.remove_reply_suffix(camel_case_dto_name)
-            if util.is_details(camel_case_dto_name):
-                base_type += "JVppReply<%s.%s.%s>" % (plugin_package, dto_package, request_dto_name + "Dump")
-                generate_dump_reply_dto(request_dto_name, base_package, plugin_package, dto_package,
-                                        camel_case_dto_name, camel_case_method_name, func)
-            else:
-                base_type += "JVppReply<%s.%s.%s>" % (plugin_package, dto_package, request_dto_name)
-        elif util.is_dump(camel_case_dto_name) or util.is_request(func['name'], func_list):
-            args = "" if fields is "" else "this"
-            methods += send_template.substitute(method_name=camel_case_method_name,
-                                                base_package=base_package,
-                                                plugin_package=plugin_package,
-                                                plugin_name=plugin_name,
-                                                args=args)
-            if util.is_dump(camel_case_dto_name):
-                base_type += "JVppDump"
-                description = "dump request DTO"
-            else:
-                base_type += "JVppRequest"
-                description = "request DTO"
-        else:
-            description = "event DTO"
-            dto_path = os.path.join(dto_package, camel_case_dto_name + ".java")
-
-        write_dto_file(base_package, plugin_package, base_type, camel_case_dto_name, description, dto_package,
-                       dto_path, fields, func, inputfile, methods)
-
-    flush_dump_reply_dtos(inputfile)
-
+def _generate_reply_dto(msg, model):
+    msg_java_name_upper = msg.java_name_upper
+    # Negative retval is mapped to java exception, so filter it out:
+    fields = filter(lambda field: not is_retval(field), msg.fields)
+    return _REPLY_TEMPLATE.substitute(
+        plugin_package=model.plugin_package,
+        json_filename=model.json_api_files,
+        json_definition=msg.doc,
+        class_name=msg_java_name_upper,
+        request_name=msg.request_java,
+        fields=generate_fields(fields),
+        hash_code=generate_hash_code(fields),
+        equals=generate_equals(msg_java_name_upper, fields),
+        to_string=generate_to_string(msg_java_name_upper, fields))
+
+_REPLY_TEMPLATE = Template("""
+package $plugin_package.dto;
 
-def generate_dto_base_methods(camel_case_dto_name, func):
-    methods = generate_dto_hash(func)
-    methods += generate_dto_equals(camel_case_dto_name, func)
-    methods += generate_dto_tostring(camel_case_dto_name, func)
-    return methods
+/**
+ * <p>This class represents reply DTO.
+ * <br>It was generated by jvpp_dto_gen.py based on $json_filename:
+ * <pre>
+$json_definition
+ * </pre>
+ */
+public final class $class_name implements io.fd.vpp.jvpp.dto.JVppReply<$plugin_package.dto.$request_name> {
+$fields
+$hash_code
+$equals
+$to_string
+}
+""")
 
 
-def generate_dto_fields(camel_case_dto_name, func):
-    fields = ""
-    for t in zip(func['types'], func['args']):
-        # for retval don't generate dto field in Reply
-        field_name = util.underscore_to_camelcase(t[1])
-        if util.is_reply(camel_case_dto_name) and util.is_retval_field(field_name):
-            continue
-        fields += field_template.substitute(type=util.jni_2_java_type_mapping[t[0]],
-                                            name=field_name)
-    return fields
+def _generate_event_dto(msg, model):
+    msg_java_name_upper = msg.java_name_upper
+    # Negative retval is mapped to java exception, so filter it out:
+    fields = filter(lambda field: not is_retval(field), msg.fields)
+    return _EVENT_TEMPLATE.substitute(
+        plugin_package=model.plugin_package,
+        json_filename=model.json_api_files,
+        json_definition=msg.doc,
+        class_name=msg_java_name_upper,
+        fields=generate_fields(fields),
+        hash_code=generate_hash_code(fields),
+        equals=generate_equals(msg_java_name_upper, fields),
+        to_string=generate_to_string(msg_java_name_upper, fields))
 
+_EVENT_TEMPLATE = Template("""
+package $plugin_package.dto;
 
-tostring_field_template = Template("""                \"$field_name=\" + $field_name + ", " +\n""")
-tostring_array_field_template = Template("""                \"$field_name=\" + java.util.Arrays.toString($field_name) + ", " +\n""")
-tostring_template = Template("""    @Override
-    public String toString() {
-        return "$cls_name{" +
-$fields_tostring "}";
-    }\n\n""")
+/**
+ * <p>This class represents event DTO.
+ * <br>It was generated by jvpp_dto_gen.py based on $json_filename:
+ * <pre>
+$json_definition
+ * </pre>
+ */
+public final class $class_name {
+$fields
+$hash_code
+$equals
+$to_string
+}
+""")
 
 
-def generate_dto_tostring(camel_case_dto_name, func):
-    tostring_fields = ""
-    for t in zip(func['types'], func['args']):
+def _generate_dump_reply_wrappers(work_dir, model, logger):
+    for msg in model.messages:
+        if is_details(msg):
+            logger.debug("Generating ReplyDump DTO for message %s", msg)
+            details_class = msg.java_name_upper
+            dto = _REPLY_DUMP_TEMPLATE.substitute(
+                plugin_package=model.plugin_package,
+                json_filename=model.json_api_files,
+                json_definition=msg.doc,
+                details_class=details_class,
+                details_field=msg.java_name_lower,
+                dump_class=msg.request_java
+            )
+            with open("%s/%sReplyDump.java" % (work_dir, details_class), "w") as f:
+                f.write(dto)
+
+_REPLY_DUMP_TEMPLATE = Template("""
+package $plugin_package.dto;
 
-        field_name = util.underscore_to_camelcase(t[1])
-        # for retval don't generate dto field in Reply
-        if util.is_retval_field(field_name):
-            continue
+/**
+ * <p>This class represents dump reply wrapper.
+ * <br>It was generated by jvpp_dto_gen.py based on $json_filename:
+ * <pre>
+$json_definition
+ * </pre>
+ */
+public final class ${details_class}ReplyDump implements io.fd.vpp.jvpp.dto.JVppReplyDump<${plugin_package}.dto.${dump_class}, ${plugin_package}.dto.${details_class}> {
 
-        # handle array types
-        if util.is_array(util.jni_2_java_type_mapping[t[0]]):
-            tostring_fields += tostring_array_field_template.substitute(field_name=field_name)
-        else:
-            tostring_fields += tostring_field_template.substitute(field_name=field_name)
+    public java.util.List<${details_class}> ${details_field} = new java.util.ArrayList<>();
 
-    return tostring_template.substitute(cls_name=camel_case_dto_name,
-                                        fields_tostring=tostring_fields[:-8])
+    @Override
+    @io.fd.vpp.jvpp.coverity.SuppressFBWarnings("UWF_UNWRITTEN_PUBLIC_OR_PROTECTED_FIELD")
+    public int hashCode() {
+        return java.util.Objects.hash(${details_field});
+    }
 
-equals_other_template = Template("""
-        final $cls_name other = ($cls_name) o;
-\n""")
-equals_field_template = Template("""        if (!java.util.Objects.equals(this.$field_name, other.$field_name)) {
-            return false;
-        }\n""")
-equals_array_field_template = Template("""        if (!java.util.Arrays.equals(this.$field_name, other.$field_name)) {
-            return false;
-        }\n""")
-equals_template = Template("""    @Override
+    @Override
     public boolean equals(final Object o) {
         if (this == o) {
             return true;
@@ -174,147 +210,22 @@ equals_template = Template("""    @Override
         if (o == null || getClass() != o.getClass()) {
             return false;
         }
-$comparisons
-        return true;
-    }\n\n""")
-
-
-def generate_dto_equals(camel_case_dto_name, func):
-    equals_fields = ""
-    for t in zip(func['types'], func['args']):
-        field_name = util.underscore_to_camelcase(t[1])
-        # for retval don't generate dto field in Reply
-        if util.is_retval_field(field_name):
-            continue
-
-        # handle array types
-        if util.is_array(util.jni_2_java_type_mapping[t[0]]):
-            equals_fields += equals_array_field_template.substitute(field_name=field_name)
-        else:
-            equals_fields += equals_field_template.substitute(field_name=field_name)
-
-    if equals_fields != "":
-        equals_fields = equals_other_template.substitute(cls_name=camel_case_dto_name) + equals_fields
-
-    return equals_template.substitute(comparisons=equals_fields)
 
+        final ${details_class}ReplyDump other = (${details_class}ReplyDump) o;
 
-hash_template = Template("""    @Override
-    @io.fd.vpp.jvpp.coverity.SuppressFBWarnings("UWF_UNWRITTEN_PUBLIC_OR_PROTECTED_FIELD")
-    public int hashCode() {
-        return java.util.Objects.hash($fields);
-    }\n\n""")
-hash_single_array_type_template = Template("""    @Override
-    @io.fd.vpp.jvpp.coverity.SuppressFBWarnings("UWF_UNWRITTEN_PUBLIC_OR_PROTECTED_FIELD")
-    public int hashCode() {
-        return java.util.Arrays.hashCode($fields);
-    }\n\n""")
-
+        if (!java.util.Objects.equals(this.${details_field}, other.${details_field})) {
+            return false;
+        }
 
-def generate_dto_hash(func):
-    hash_fields = ""
+        return true;
+    }
 
-    # Special handling for hashCode in case just a single array field is present. Cannot use Objects.equals since the
-    # array is mistaken for a varargs parameter. Instead use Arrays.hashCode in such case.
-    if len(func['args']) == 1:
-        single_type = func['types'][0]
-        single_type_name = func['args'][0]
-        if util.is_array(util.jni_2_java_type_mapping[single_type]):
-            return hash_single_array_type_template.substitute(fields=util.underscore_to_camelcase(single_type_name))
+    @Override
+    public String toString() {
+        return "${details_class}ReplyDump{" +
+                "${details_field}=" + ${details_field} + "}";
+    }
 
-    for t in zip(func['types'], func['args']):
-        field_name = util.underscore_to_camelcase(t[1])
-        # for retval don't generate dto field in Reply
-        if util.is_retval_field(field_name):
-            continue
 
-        hash_fields += field_name + ", "
-
-    return hash_template.substitute(fields=hash_fields[:-2])
-
-
-def write_dto_file(base_package, plugin_package, base_type, camel_case_dto_name, description, dto_package, dto_path,
-                   fields, func, inputfile, methods):
-    dto_file = open(dto_path, 'w')
-    if base_type != "":
-        dto_file.write(dto_template.substitute(inputfile=inputfile,
-                                               description=description,
-                                               docs=util.api_message_to_javadoc(func),
-                                               cls_name=camel_case_dto_name,
-                                               fields=fields,
-                                               methods=methods,
-                                               base_package=base_package,
-                                               plugin_package=plugin_package,
-                                               base_type=base_type,
-                                               dto_package=dto_package))
-    else:
-        dto_file.write(dto_template_typeless.substitute(inputfile=inputfile,
-                                                        description=description,
-                                                        docs=util.api_message_to_javadoc(func),
-                                                        cls_name=camel_case_dto_name,
-                                                        fields=fields,
-                                                        methods=methods,
-                                                        plugin_package=plugin_package,
-                                                        dto_package=dto_package))
-    dto_file.flush()
-    dto_file.close()
-
-
-dump_dto_suffix = "ReplyDump"
-dump_reply_artificial_dtos = {}
-
-
-def flush_dump_reply_dtos(inputfile):
-    for dump_reply_artificial_dto in dump_reply_artificial_dtos.values():
-        dto_path = os.path.join(dump_reply_artificial_dto['dto_package'],
-                                dump_reply_artificial_dto['cls_name'] + ".java")
-        dto_file = open(dto_path, 'w')
-        dto_file.write(dto_template.substitute(inputfile=inputfile,
-                                               description="dump reply wrapper",
-                                               docs=dump_reply_artificial_dto['docs'],
-                                               cls_name=dump_reply_artificial_dto['cls_name'],
-                                               fields=dump_reply_artificial_dto['fields'],
-                                               methods=dump_reply_artificial_dto['methods'],
-                                               plugin_package=dump_reply_artificial_dto['plugin_package'],
-                                               base_package=dump_reply_artificial_dto['base_package'],
-                                               base_type=dump_reply_artificial_dto['base_type'],
-                                               dto_package=dump_reply_artificial_dto['dto_package']))
-        dto_file.flush()
-        dto_file.close()
-
-
-def generate_dump_reply_dto(request_dto_name, base_package, plugin_package, dto_package, camel_case_dto_name,
-                            camel_case_method_name, func):
-    base_type = "JVppReplyDump<%s.%s.%s, %s.%s.%s>" % (
-        plugin_package, dto_package, util.remove_reply_suffix(camel_case_dto_name) + "Dump",
-        plugin_package, dto_package, camel_case_dto_name)
-    fields = "    public java.util.List<%s> %s = new java.util.ArrayList<>();" % (camel_case_dto_name, camel_case_method_name)
-    cls_name = camel_case_dto_name + dump_dto_suffix
-    # using artificial type for fields, just to bypass the is_array check in base methods generators
-    # the type is not really used
-    artificial_type = 'u8'
-
-    # In case of already existing artificial reply dump DTO, just update it
-    # Used for sub-dump dtos
-    if request_dto_name in dump_reply_artificial_dtos.keys():
-        dump_reply_artificial_dtos[request_dto_name]['fields'] += '\n' + fields
-        dump_reply_artificial_dtos[request_dto_name]['field_names'].append(func['name'])
-        dump_reply_artificial_dtos[request_dto_name]['field_types'].append(artificial_type)
-        methods = '\n' + generate_dto_base_methods(dump_reply_artificial_dtos[request_dto_name]['cls_name'],
-                                            {'args': dump_reply_artificial_dtos[request_dto_name]['field_names'],
-                                             'types': dump_reply_artificial_dtos[request_dto_name]['field_types']})
-        dump_reply_artificial_dtos[request_dto_name]['methods'] = methods
-    else:
-        methods = '\n' + generate_dto_base_methods(cls_name, {'args': [func['name']],
-                                                              'types': [artificial_type]})
-        dump_reply_artificial_dtos[request_dto_name] = ({'docs': util.api_message_to_javadoc(func),
-                                                         'cls_name': cls_name,
-                                                         'fields': fields,
-                                                         'field_names': [func['name']],
-                                                         'field_types': [artificial_type],
-                                                         # strip too many newlines at the end of base method block
-                                                         'methods': methods,
-                                                         'plugin_package': plugin_package,
-                                                         'base_package': base_package,
-                                                         'base_type': base_type,
-                                                         'dto_package': dto_package})
+}
+""")