From 413f4a5b2123c1625d615315db293a080078482b Mon Sep 17 00:00:00 2001 From: Ole Troan Date: Wed, 28 Nov 2018 11:36:05 +0100 Subject: API: Use string type instead of u8. The new string type is modelled after string in proto3. It is always variable length. Change-Id: I64884067e28a80072c8dac31b7c7c82d6e306051 Signed-off-by: Ole Troan Signed-off-by: Michal Cmarada Signed-off-by: Ole Troan --- src/tools/vppapigen/vppapigen_c.py | 16 +++++-- src/tools/vppapigen/vppapigen_json.py | 11 ++--- src/vat/api_format.c | 46 ++++++++++++------ src/vlibapi/CMakeLists.txt | 1 + src/vlibapi/api_common.h | 1 + src/vlibapi/api_types.h | 54 ++++++++++++++++++++++ src/vpp-api/python/vpp_papi/vpp_papi.py | 46 ++++++++++++++++-- src/vpp-api/python/vpp_papi/vpp_serializer.py | 48 ++++++++++++++----- .../python/vpp_papi/vpp_transport_socket.py | 3 +- src/vpp/api/api.c | 39 +++++++++++----- src/vpp/api/custom_dump.c | 4 +- src/vpp/api/types.h | 3 ++ src/vpp/api/vpe.api | 14 +++--- 13 files changed, 224 insertions(+), 62 deletions(-) create mode 100644 src/vlibapi/api_types.h (limited to 'src') diff --git a/src/tools/vppapigen/vppapigen_c.py b/src/tools/vppapigen/vppapigen_c.py index f0fb00831c3..e189b024454 100644 --- a/src/tools/vppapigen/vppapigen_c.py +++ b/src/tools/vppapigen/vppapigen_c.py @@ -3,7 +3,8 @@ import datetime import os import time -datestring = datetime.datetime.utcfromtimestamp(int(os.environ.get('SOURCE_DATE_EPOCH', time.time()))) +datestring = datetime.datetime.utcfromtimestamp( + int(os.environ.get('SOURCE_DATE_EPOCH', time.time()))) input_filename = 'inputfil' top_boilerplate = '''\ /* @@ -94,6 +95,13 @@ def duplicate_wrapper_tail(): return '#endif\n\n' +def api2c(fieldtype): + mappingtable = {'string': 'vl_api_string_t', } + if fieldtype in mappingtable: + return mappingtable[fieldtype] + return fieldtype + + def typedefs(objs, aliases, filename): name = filename.replace('.', '_') output = '''\ @@ -130,12 +138,12 @@ def typedefs(objs, aliases, filename): output += "typedef VL_API_PACKED(struct _vl_api_%s {\n" % o.name for b in o.block: if b.type == 'Field': - output += " %s %s;\n" % (b.fieldtype, b.fieldname) + output += " %s %s;\n" % (api2c(b.fieldtype), b.fieldname) elif b.type == 'Array': if b.lengthfield: - output += " %s %s[0];\n" % (b.fieldtype, b.fieldname) + output += " %s %s[0];\n" % (api2c(b.fieldtype), b.fieldname) else: - output += " %s %s[%s];\n" % (b.fieldtype, b.fieldname, + output += " %s %s[%s];\n" % (api2c(b.fieldtype), b.fieldname, b.length) else: raise ValueError("Error in processing array type %s" % b) diff --git a/src/tools/vppapigen/vppapigen_json.py b/src/tools/vppapigen/vppapigen_json.py index b57e16c990d..a4707c0cd60 100644 --- a/src/tools/vppapigen/vppapigen_json.py +++ b/src/tools/vppapigen/vppapigen_json.py @@ -32,19 +32,18 @@ def walk_defs(s): d = [] d.append(t.name) for b in t.block: - f = [] if b.type == 'Field': - f = [b.fieldtype, b.fieldname] + d.append([b.fieldtype, b.fieldname]) elif b.type == 'Array': if b.lengthfield: - f = [b.fieldtype, b.fieldname, b.length, b.lengthfield] + d.append([b.fieldtype, b.fieldname, b.length, b.lengthfield]) else: - f = [b.fieldtype, b.fieldname, b.length] + d.append([b.fieldtype, b.fieldname, b.length]) elif b.type == 'Union': - print('UNION') + pass else: raise ValueError("Error in processing array type %s" % b) - d.append(f) + if t.crc: c = {} c['crc'] = "{0:#0{1}x}".format(t.crc, 10) diff --git a/src/vat/api_format.c b/src/vat/api_format.c index 7f4dabeffa1..25d2dd3112a 100644 --- a/src/vat/api_format.c +++ b/src/vat/api_format.c @@ -1105,7 +1105,7 @@ vl_api_cli_inband_reply_t_handler (vl_api_cli_inband_reply_t * mp) { vat_main_t *vam = &vat_main; i32 retval = ntohl (mp->retval); - u32 length = ntohl (mp->length); + u32 length = vl_api_string_len (&mp->reply); vec_reset_length (vam->cmd_reply); @@ -1113,7 +1113,8 @@ vl_api_cli_inband_reply_t_handler (vl_api_cli_inband_reply_t * mp) if (retval == 0) { vec_validate (vam->cmd_reply, length); - clib_memcpy ((char *) (vam->cmd_reply), mp->reply, length); + clib_memcpy ((char *) (vam->cmd_reply), + vl_api_from_api_string (&mp->reply), length); vam->cmd_reply[length] = 0; } vam->result_ready = 1; @@ -1129,7 +1130,8 @@ vl_api_cli_inband_reply_t_handler_json (vl_api_cli_inband_reply_t * mp) vat_json_init_object (&node); vat_json_object_add_int (&node, "retval", ntohl (mp->retval)); - vat_json_object_add_string_copy (&node, "reply", mp->reply); + vat_json_object_add_string_copy (&node, "reply", + vl_api_from_api_string (&mp->reply)); vat_json_print (vam->ofp, &node); vat_json_free (&node); @@ -1300,10 +1302,18 @@ static void vl_api_show_version_reply_t_handler if (retval >= 0) { - errmsg (" program: %s", mp->program); - errmsg (" version: %s", mp->version); - errmsg (" build date: %s", mp->build_date); - errmsg ("build directory: %s", mp->build_directory); + char *p = (char *) &mp->program; + errmsg (" program: %s\n", + vl_api_from_api_string ((vl_api_string_t *) p)); + p += vl_api_string_len ((vl_api_string_t *) p) + sizeof (u32); + errmsg (" version: %s\n", + vl_api_from_api_string ((vl_api_string_t *) p)); + p += vl_api_string_len ((vl_api_string_t *) p) + sizeof (u32); + errmsg (" build date: %s\n", + vl_api_from_api_string ((vl_api_string_t *) p)); + p += vl_api_string_len ((vl_api_string_t *) p) + sizeof (u32); + errmsg ("build directory: %s\n", + vl_api_from_api_string ((vl_api_string_t *) p)); } vam->retval = retval; vam->result_ready = 1; @@ -1317,11 +1327,22 @@ static void vl_api_show_version_reply_t_handler_json vat_json_init_object (&node); vat_json_object_add_int (&node, "retval", ntohl (mp->retval)); - vat_json_object_add_string_copy (&node, "program", mp->program); - vat_json_object_add_string_copy (&node, "version", mp->version); - vat_json_object_add_string_copy (&node, "build_date", mp->build_date); + char *p = (char *) &mp->program; + vat_json_object_add_string_copy (&node, "program", + vl_api_from_api_string ((vl_api_string_t *) + p)); + p += vl_api_string_len ((vl_api_string_t *) p) + sizeof (u32); + vat_json_object_add_string_copy (&node, "version", + vl_api_from_api_string ((vl_api_string_t *) + p)); + p += vl_api_string_len ((vl_api_string_t *) p) + sizeof (u32); + vat_json_object_add_string_copy (&node, "build_date", + vl_api_from_api_string ((vl_api_string_t *) + p)); + p += vl_api_string_len ((vl_api_string_t *) p) + sizeof (u32); vat_json_object_add_string_copy (&node, "build_directory", - mp->build_directory); + vl_api_from_api_string ((vl_api_string_t *) + p)); vat_json_print (vam->ofp, &node); vat_json_free (&node); @@ -6339,8 +6360,7 @@ exec_inband (vat_main_t * vam) */ u32 len = vec_len (vam->input->buffer); M2 (CLI_INBAND, mp, len); - clib_memcpy (mp->cmd, vam->input->buffer, len); - mp->length = htonl (len); + vl_api_to_api_string (len, (const char *) vam->input->buffer, &mp->cmd); S (mp); W (ret); diff --git a/src/vlibapi/CMakeLists.txt b/src/vlibapi/CMakeLists.txt index 55f87b8f7ac..dfd6b5b5045 100644 --- a/src/vlibapi/CMakeLists.txt +++ b/src/vlibapi/CMakeLists.txt @@ -17,6 +17,7 @@ install( api.h vat_helper_macros.h api_common.h + api_types.h DESTINATION include/vlibapi diff --git a/src/vlibapi/api_common.h b/src/vlibapi/api_common.h index 497a1e8bd16..735921b30f7 100644 --- a/src/vlibapi/api_common.h +++ b/src/vlibapi/api_common.h @@ -25,6 +25,7 @@ */ #include +#include #include #include diff --git a/src/vlibapi/api_types.h b/src/vlibapi/api_types.h new file mode 100644 index 00000000000..759298e735d --- /dev/null +++ b/src/vlibapi/api_types.h @@ -0,0 +1,54 @@ +/* + *------------------------------------------------------------------ + * api_types.h + * + * Copyright (c) 2018 Cisco and/or its affiliates. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------ + */ + +#ifndef included_api_types_h +#define included_api_types_h + +#include + +/* VPP API string type */ +typedef struct +{ + u32 length; + u8 buf[0]; +} __attribute__ ((packed)) vl_api_string_t; + +static inline int +vl_api_to_api_string (u32 len, const char *buf, vl_api_string_t * str) +{ + if (strncpy_s ((char *) str->buf, len, buf, len - 1) != 0) + len = 0; + str->length = clib_host_to_net_u32 (len); + return len + sizeof (u32); +} + +/* Return a C string from API string */ +static inline u8 * +vl_api_from_api_string (vl_api_string_t * astr) +{ + return astr->buf; +} + +static inline u32 +vl_api_string_len (vl_api_string_t * astr) +{ + return clib_net_to_host_u32 (astr->length); +} + +#endif diff --git a/src/vpp-api/python/vpp_papi/vpp_papi.py b/src/vpp-api/python/vpp_papi/vpp_papi.py index 32abe14da94..4de257c6924 100644 --- a/src/vpp-api/python/vpp_papi/vpp_papi.py +++ b/src/vpp-api/python/vpp_papi/vpp_papi.py @@ -29,6 +29,8 @@ import atexit from . vpp_serializer import VPPType, VPPEnumType, VPPUnionType, BaseTypes from . vpp_serializer import VPPMessage, vpp_get_type, VPPTypeAlias +logger = logging.getLogger(__name__) + if sys.version[0] == '2': import Queue as queue else: @@ -63,6 +65,19 @@ else: return d.items() +def call_logger(msgdef, kwargs): + s = 'Calling {}('.format(msgdef.name) + for k, v in kwargs.items(): + s += '{}:{} '.format(k, v) + s += ')' + return s + + +def return_logger(r): + s = 'Return from {}'.format(r) + return s + + class VppApiDynamicMethodHolder(object): pass @@ -493,10 +508,31 @@ class VPP(object): else: raise VPPIOError(2, 'RPC reply message received in event handler') + def has_context(self, msg): + if len(msg) < 10: + return False + + header = VPPType('header_with_context', [['u16', 'msgid'], + ['u32', 'client_index'], + ['u32', 'context']]) + + (i, ci, context), size = header.unpack(msg, 0) + if self.id_names[i] == 'rx_thread_exit': + return + + # + # Decode message and returns a tuple. + # + msgobj = self.id_msgdef[i] + if 'context' in msgobj.field_by_name and context >= 0: + return True + return False + def decode_incoming_msg(self, msg, no_type_conversion=False): if not msg: self.logger.warning('vpp_api.read failed') return + (i, ci), size = self.header.unpack(msg, 0) if self.id_names[i] == 'rx_thread_exit': return @@ -537,7 +573,7 @@ class VPP(object): raise VPPValueError('Invalid argument {} to {}' .format(list(d), msg.name)) - def _call_vpp(self, i, msg, multipart, **kwargs): + def _call_vpp(self, i, msgdef, multipart, **kwargs): """Given a message, send the message and await a reply. msgdef - the message packing definition @@ -567,8 +603,11 @@ class VPP(object): kwargs['client_index'] = self.transport.socket_index except AttributeError: pass - self.validate_args(msg, kwargs) - b = msg.pack(kwargs) + self.validate_args(msgdef, kwargs) + + logging.debug(call_logger(msgdef, kwargs)) + + b = msgdef.pack(kwargs) self.transport.suspend() self.transport.write(b) @@ -601,6 +640,7 @@ class VPP(object): self.transport.resume() + logger.debug(return_logger(rl)) return rl def _call_vpp_async(self, i, msg, **kwargs): diff --git a/src/vpp-api/python/vpp_papi/vpp_serializer.py b/src/vpp-api/python/vpp_papi/vpp_serializer.py index 5dce03b6188..418c0243402 100644 --- a/src/vpp-api/python/vpp_papi/vpp_serializer.py +++ b/src/vpp-api/python/vpp_papi/vpp_serializer.py @@ -59,6 +59,7 @@ def conversion_unpacker(data, field_type): class BaseTypes(object): def __init__(self, type, elements=0): base_types = {'u8': '>B', + 'string': '>s', 'u16': '>H', 'u32': '>I', 'i32': '>i', @@ -67,13 +68,11 @@ class BaseTypes(object): 'bool': '>?', 'header': '>HI'} - if elements > 0 and type == 'u8': + if elements > 0 and (type == 'u8' or type == 'string'): self.packer = struct.Struct('>%ss' % elements) else: self.packer = struct.Struct(base_types[type]) self.size = self.packer.size - logger.debug('Adding {} with format: {}' - .format(type, base_types[type])) def pack(self, data, kwargs=None): if not data: # Default to zero if not specified @@ -84,10 +83,32 @@ class BaseTypes(object): return self.packer.unpack_from(data, offset)[0], self.packer.size +class String(object): + def __init__(self): + self.name = 'string' + self.size = 1 + self.length_field_packer = BaseTypes('u32') + + def pack(self, list, kwargs=None): + if not list: + return self.length_field_packer.pack(0) + b"" + return self.length_field_packer.pack(len(list)) + list.encode('utf8') + + def unpack(self, data, offset=0, result=None, ntc=False): + length, length_field_size = self.length_field_packer.unpack(data, + offset) + if length == 0: + return b'', 0 + p = BaseTypes('u8', length) + x, size = p.unpack(data, offset + length_field_size) + x2 = x.split(b'\0',1)[0] + return (x2.decode('utf8'), size + length_field_size) + + types = {'u8': BaseTypes('u8'), 'u16': BaseTypes('u16'), 'u32': BaseTypes('u32'), 'i32': BaseTypes('i32'), 'u64': BaseTypes('u64'), 'f64': BaseTypes('f64'), - 'bool': BaseTypes('bool')} + 'bool': BaseTypes('bool'), 'string': String()} def vpp_get_type(name): @@ -107,6 +128,7 @@ class FixedList_u8(object): self.num = num self.packer = BaseTypes(field_type, num) self.size = self.packer.size + self.field_type = field_type def pack(self, data, kwargs=None): """Packs a fixed length bytestring. Left-pads with zeros @@ -128,6 +150,10 @@ class FixedList_u8(object): 'Invalid array length for "{}" got {}' ' expected {}' .format(self.name, len(data[offset:]), self.num)) + if self.field_type == 'string': + s = self.packer.unpack(data, offset) + s2 = s[0].split(b'\0', 1)[0] + return (s2.decode('utf-8'), self.num) return self.packer.unpack(data, offset) @@ -164,6 +190,7 @@ class FixedList(object): class VLAList(object): def __init__(self, name, field_type, len_field_name, index): self.name = name + self.field_type = field_type self.index = index self.packer = types[field_type] self.size = self.packer.size @@ -179,6 +206,7 @@ class VLAList(object): b = bytes() # u8 array + if self.packer.size == 1: return bytearray(list) @@ -249,7 +277,6 @@ class VPPEnumType(object): e_hash[ename] = evalue self.enum = IntEnum(name, e_hash) types[name] = self - logger.debug('Adding enum {}'.format(name)) def __getattr__(self, name): return self.enum[name] @@ -290,7 +317,6 @@ class VPPUnionType(object): types[name] = self self.tuple = collections.namedtuple(name, fields, rename=True) - logger.debug('Adding union {}'.format(name)) # Union of variable length? def pack(self, data, kwargs=None): @@ -381,7 +407,7 @@ class VPPType(object): if list_elements == 0: p = VLAList_legacy(f_name, f_type) self.packers.append(p) - elif f_type == 'u8': + elif f_type == 'u8' or f_type == 'string': p = FixedList_u8(f_name, f_type, list_elements) self.packers.append(p) size += p.size @@ -390,10 +416,9 @@ class VPPType(object): self.packers.append(p) size += p.size elif len(f) == 4: # Variable length list - # Find index of length field - length_index = self.fields.index(f[3]) - p = VLAList(f_name, f_type, f[3], length_index) - self.packers.append(p) + length_index = self.fields.index(f[3]) + p = VLAList(f_name, f_type, f[3], length_index) + self.packers.append(p) else: self.packers.append(types[f_type]) size += types[f_type].size @@ -401,7 +426,6 @@ class VPPType(object): self.size = size self.tuple = collections.namedtuple(name, self.fields, rename=True) types[name] = self - logger.debug('Adding type {}'.format(name)) def pack(self, data, kwargs=None): if not kwargs: diff --git a/src/vpp-api/python/vpp_papi/vpp_transport_socket.py b/src/vpp-api/python/vpp_papi/vpp_transport_socket.py index 393e2e9c9c1..4341cad3e90 100644 --- a/src/vpp-api/python/vpp_papi/vpp_transport_socket.py +++ b/src/vpp-api/python/vpp_papi/vpp_transport_socket.py @@ -59,8 +59,7 @@ class VppTransport(object): return # Put either to local queue or if context == 0 # callback queue - r = self.parent.decode_incoming_msg(msg) - if hasattr(r, 'context') and r.context > 0: + if self.parent.has_context(msg): self.q.put(msg) else: self.parent.msg_handler_async(msg) diff --git a/src/vpp/api/api.c b/src/vpp/api/api.c index cf5d252e347..1f376dcc64f 100644 --- a/src/vpp/api/api.c +++ b/src/vpp/api/api.c @@ -217,16 +217,25 @@ vl_api_cli_inband_t_handler (vl_api_cli_inband_t * mp) vlib_main_t *vm = vlib_get_main (); unformat_input_t input; u8 *out_vec = 0; + u32 len = 0; - unformat_init_string (&input, (char *) mp->cmd, ntohl (mp->length)); + if (vl_msg_api_get_msg_length (mp) < vl_api_string_len (&mp->cmd)) + { + rv = -1; + goto error; + } + + unformat_init_string (&input, (char *) vl_api_from_api_string (&mp->cmd), + vl_api_string_len (&mp->cmd)); vlib_cli_input (vm, &input, inband_cli_output, (uword) & out_vec); - u32 len = vec_len (out_vec); + len = vec_len (out_vec); + +error: /* *INDENT-OFF* */ REPLY_MACRO3(VL_API_CLI_INBAND_REPLY, len, ({ - rmp->length = htonl (len); - clib_memcpy (rmp->reply, out_vec, len); + vl_api_to_api_string(len, (const char *)out_vec, &rmp->reply); })); /* *INDENT-ON* */ vec_free (out_vec); @@ -241,16 +250,22 @@ vl_api_show_version_t_handler (vl_api_show_version_t * mp) char *vpe_api_get_version (void); char *vpe_api_get_build_date (void); + u32 program_len = strnlen_s ("vpe", 32) + 1; + u32 version_len = strnlen_s (vpe_api_get_version (), 32) + 1; + u32 build_date_len = strnlen_s (vpe_api_get_build_date (), 32) + 1; + u32 build_directory_len = + strnlen_s (vpe_api_get_build_directory (), 256) + 1; + + u32 n = program_len + version_len + build_date_len + build_directory_len; + /* *INDENT-OFF* */ - REPLY_MACRO2(VL_API_SHOW_VERSION_REPLY, + REPLY_MACRO3(VL_API_SHOW_VERSION_REPLY, n, ({ - strncpy ((char *) rmp->program, "vpe", ARRAY_LEN(rmp->program)-1); - strncpy ((char *) rmp->build_directory, vpe_api_get_build_directory(), - ARRAY_LEN(rmp->build_directory)-1); - strncpy ((char *) rmp->version, vpe_api_get_version(), - ARRAY_LEN(rmp->version)-1); - strncpy ((char *) rmp->build_date, vpe_api_get_build_date(), - ARRAY_LEN(rmp->build_date)-1); + char *p = (char *)&rmp->program; + p += vl_api_to_api_string(program_len, "vpe", (vl_api_string_t *)p); + p += vl_api_to_api_string(version_len, vpe_api_get_version(), (vl_api_string_t *)p); + p += vl_api_to_api_string(build_date_len, vpe_api_get_build_date(), (vl_api_string_t *)p); + vl_api_to_api_string(build_directory_len, vpe_api_get_build_directory(), (vl_api_string_t *)p); })); /* *INDENT-ON* */ } diff --git a/src/vpp/api/custom_dump.c b/src/vpp/api/custom_dump.c index 41a24ef5a34..78d37044955 100644 --- a/src/vpp/api/custom_dump.c +++ b/src/vpp/api/custom_dump.c @@ -2109,10 +2109,10 @@ static void *vl_api_cli_inband_t_print { u8 *s; u8 *cmd = 0; - u32 length = ntohl (mp->length); + u32 length = vl_api_string_len (&mp->cmd); vec_validate (cmd, length); - clib_memcpy (cmd, mp->cmd, length); + clib_memcpy (cmd, vl_api_from_api_string (&mp->cmd), length); s = format (0, "SCRIPT: exec %v ", cmd); diff --git a/src/vpp/api/types.h b/src/vpp/api/types.h index 252caa2d626..9a45639d030 100644 --- a/src/vpp/api/types.h +++ b/src/vpp/api/types.h @@ -16,6 +16,9 @@ #ifndef __API_TYPES_H__ #define __API_TYPES_H__ +#include +#include + #define vl_typedefs /* define message structures */ #include #undef vl_typedefs diff --git a/src/vpp/api/vpe.api b/src/vpp/api/vpe.api index 488af1727bc..24d44bd31e7 100644 --- a/src/vpp/api/vpe.api +++ b/src/vpp/api/vpe.api @@ -93,8 +93,7 @@ define cli_inband { u32 client_index; u32 context; - u32 length; - u8 cmd[length]; + string cmd; }; /** \brief vpe parser cli string response @@ -112,8 +111,7 @@ define cli_inband_reply { u32 context; i32 retval; - u32 length; - u8 reply[length]; + string reply; }; /** \brief Get node index using name request @@ -187,10 +185,10 @@ define show_version_reply { u32 context; i32 retval; - u8 program[32]; - u8 version[32]; - u8 build_date[32]; - u8 build_directory[256]; + string program; + string version; + string build_date; + string build_directory; }; -- cgit 1.2.3-korg