diff options
Diffstat (limited to 'src/tools/vppapigen')
| -rw-r--r-- | src/tools/vppapigen/CMakeLists.txt | 18 | ||||
| -rw-r--r-- | src/tools/vppapigen/VPPAPI.md | 346 | ||||
| -rw-r--r-- | src/tools/vppapigen/VPPAPI.rst | 597 | ||||
| -rwxr-xr-x | src/tools/vppapigen/generate_go.py | 236 | ||||
| -rwxr-xr-x | src/tools/vppapigen/generate_json.py | 118 | ||||
| -rwxr-xr-x | src/tools/vppapigen/test_vppapigen.py | 120 | ||||
| -rwxr-xr-x | src/tools/vppapigen/vppapigen.py | 916 | ||||
| -rwxr-xr-x[-rw-r--r--] | src/tools/vppapigen/vppapigen_c.py | 1973 | ||||
| -rw-r--r-- | src/tools/vppapigen/vppapigen_crc.py | 17 | ||||
| -rw-r--r-- | src/tools/vppapigen/vppapigen_json.py | 89 |
10 files changed, 2594 insertions, 1836 deletions
diff --git a/src/tools/vppapigen/CMakeLists.txt b/src/tools/vppapigen/CMakeLists.txt index bfabc3a670c..97a6d35f9b5 100644 --- a/src/tools/vppapigen/CMakeLists.txt +++ b/src/tools/vppapigen/CMakeLists.txt @@ -11,6 +11,22 @@ # See the License for the specific language governing permissions and # limitations under the License. +find_package( + Python3 + REQUIRED + COMPONENTS Interpreter +) + +execute_process( + COMMAND ${Python3_EXECUTABLE} -c "import ply" + RESULT_VARIABLE _rv + OUTPUT_QUIET +) + +if (NOT ${_rv} EQUAL 0) + message( FATAL_ERROR "The \"ply\" Python3 package is not installed.") +endif() + install( FILES vppapigen.py RENAME vppapigen @@ -27,7 +43,7 @@ install( vppapigen_json.py generate_json.py DESTINATION - share/vpp + ${CMAKE_INSTALL_DATADIR}/vpp COMPONENT vpp-dev ) diff --git a/src/tools/vppapigen/VPPAPI.md b/src/tools/vppapigen/VPPAPI.md deleted file mode 100644 index df211d866a0..00000000000 --- a/src/tools/vppapigen/VPPAPI.md +++ /dev/null @@ -1,346 +0,0 @@ -# VPP API Language {#api_lang_doc} - -The VPP binary API is a message passing API. -The VPP API language is used to define a RPC interface between VPP and its -control plane. The API messages supports shared memory transport and -Unix domain sockets (SOCK_STREAM). - -The wire format is essentially that of a network formatted (big-endian) packed C struct. - -The VPP API compiler is located in *src/tools/vppapigen* and can currently -compile to JSON or C (used by the VPP binary itself). - -## Language definition - -### Defining a messages - -There are 3 types of message exchanges: - -* Request/Reply -The client sends a request message and the server replies with a -single reply message. The convention is that the reply message is -named as method_name + \_reply. - -* Dump/Detail -The client sends a "bulk" request message to the server, and the -server replies with a set of detail messages. These messages may be of -different type. A dump/detail call must be enclosed in a control ping -block (Otherwise the client will not know the end of the bulk -transmission). The method name must end with method + "\_dump", the -reply message should be named method + "\_details". The exception here -is for the methods that return multiple message types -(e.g. sw_interface_dump). The Dump/Detail methods are typically used -for acquiring bulk information, like the complete FIB table. - -* Events -The client can register for getting asynchronous notifications from -the server. This is useful for getting interface state changes, and so -on. The method name for requesting notifications is conventionally -prefixed with "want_". E.g. "want_interface_events". Which -notification types results from an event registration is defined in -the service definition. - -A message from a client must include the 'client_index', an opaque -cookie identifying the sender, and a 'context' field to let the client -match request with reply. - -An example of a message definition. The client sends the show_version request, -the server replies with the show_version_reply. - -The *client_index* and *context* fields are required in all requests. -The *context* is returned by the server and is used by the client to -match up request and reply messages. - -``` -define show_version -{ - u32 client_index; - u32 context; -}; -define show_version_reply -{ - u32 context; - i32 retval; - string program [32]; - string version [32]; - string build_date [32]; - /* The final field can be a variable length argument */ - string build_directory []; -}; - -``` - -The flags are not used by the clients, but have special meaning -for some of the tracing and debugging of the API. -The *autoreply* flag is a shorthand for a reply message with just a -*retval* field. - -``` - define : DEFINE ID '{' block_statements_opt '}' ';' - define : flist DEFINE ID '{' block_statements_opt '}' ';' - flist : flag - | flist flag - flag : MANUAL_PRINT - | MANUAL_ENDIAN - | DONT_TRACE - | AUTOREPLY - - block_statements_opt : block_statements - block_statements : block_statement - | block_statements block_statement - block_statement : declaration - | option - declaration : type_specifier ID ';' - | type_specifier ID '[' ID '=' assignee ']' ';' - declaration : type_specifier ID '[' NUM ']' ';' - | type_specifier ID '[' ID ']' ';' - type_specifier : U8 - | U16 - | U32 - | U64 - | I8 - | I16 - | I32 - | I64 - | F64 - | BOOL - | STRING - type_specifier : ID -``` - - -### Options -The *option* word is used to specify meta information. -The only current use is to specify a semantic version of the .api file itself. - -Example: -``` -option version = "1.0.0"; -``` - -``` - - option : OPTION ID '=' assignee ';' - assignee : NUM - | TRUE - | FALSE - | STRING_LITERAL -``` - -### Defining new types - -New user defined types are defined just like messages. -A typedef has two forms. It can either define an alias for a -different type (or array). - -Example: - -``` -typedef u8 ip4_address[4]; -typedef u8 ip6_address[16]; -``` - -Where the above defines two new types *vl_api_ip4_address_t* and -*vl_api_ip6_address_t*. These are aliases for the underlying -u8 array. - -In the other form, it is used to specify an abstract data type. - -``` -enum address_family { - ADDRESS_IP4 = 0, - ADDRESS_IP6, -}; - -union address_union { - vl_api_ip4_address_t ip4; - vl_api_ip6_address_t ip6; -}; - -typedef address { - vl_api_address_family_t af; - vl_api_address_union_t un; -}; -``` - -Where the new type *vl_api_address_t* - -``` - typedef : TYPEDEF ID '{' block_statements_opt '}' ';' - typedef : TYPEDEF declaration -``` - - -### Importing Definitions -You can use definitions from other .api files by importing them. -To import another .api's definitions, you add an import statement -to the top of your file: - -import "vnet/ip/ip_types.api"; - -By default you can only use definitions from directly imported .api files. - -The API compiler searches for imported files in a set of directories -specified on the API compiler command line using the --includedir flag. -``` -import : IMPORT STRING_LITERAL ';' -``` - -### Comments - -The API language uses C style comments. -``` -/* */ -// -``` - -### Enumerations -Enums are similar to enums in C. - -Every enum definition must contain a constant that maps to zero -as its first element. This is because: - -There must be a zero value, so that we can use 0 as a numeric default value. -The zero value needs to be the first element. - -As in C, enums can be used as flags or just as numbers. -The on-wire, and in memory representation size of an enum can be specified. -Not all language bindings will support that. The default size is 4 (u32). - -Example -``` -enum ip_neighbor_flags -{ - IP_API_NEIGHBOR_FLAG_NONE = 0, - IP_API_NEIGHBOR_FLAG_STATIC = 0x1, - IP_API_NEIGHBOR_FLAG_NO_FIB_ENTRY = 0x2, -}; -``` - -Which generates the vl_api_ip_neighbor_flags_t in the C binding. -In Python that is represented as an IntFlag object -VppEnum.vl_api_ip_neighbor_flags_t. - -``` - enum : ENUM ID '{' enum_statements '}' ';' - enum : ENUM ID ':' enum_size '{' enum_statements '}' ';' - enum_size : U8 - | U16 - | U32 - enum_statements : enum_statement - | enum_statements enum_statement - enum_statement : ID '=' NUM ',' - | ID ',' -``` - -### Services -The service statement defines the relationship between messages. -For request/response and dump/details messages it ties the -request with the reply. For events, it specifies which events -that can be received for a given want_* call. - -Example: -``` -service { - rpc want_interface_events returns want_interface_events_reply - events sw_interface_event; -}; - -``` - -Which states that the request want_interface_events returns a -want_interface_events_reply and if enabled the client will -receive sw_interface_event messages whenever interface states changes. - -``` - service : SERVICE '{' service_statements '}' ';' - service_statements : service_statement - | service_statements service_statement - service_statement : RPC ID RETURNS NULL ';' - | RPC ID RETURNS ID ';' - | RPC ID RETURNS STREAM ID ';' - | RPC ID RETURNS ID EVENTS event_list ';' - event_list : events - | event_list events - events : ID - | ID ',' -``` - - -## Types -### Scalar Value Types - -.api type|size|C type|Python type ----------|----|------|----------- -i8 | 1|i8 |int -u8 | 1|u8 |int -i16 | 2|i16 |int -u16 | 2|u16 |int -i32 | 4|i32 |int -u32 | 4|u32 |int -i64 | 8|i64 |int -u64 | 8|u64 |int -f64 | 8|f64 |float -bool | 1|bool |boolean -string |variable|vl_api_string_t|str - -### User Defined Types -#### vnet/ip/ip_types.api - -.api type|size|C type|Python type ----------|----|------|----------- -vl_api_address_t|20|vl_api_address_t|`<class 'ipaddress.IPv4Address'> or <class 'ipaddress.IPv6Address'>` -vl_api_ip4_address_t|4|vl_api_ip4_address_t|`<class 'ipaddress.IPv4Address'>` -vl_api_ip6_address_t|16|vl_api_ip6_address_t|`<class 'ipaddress.IPv6Address'>` -vl_api_prefix_t|21|vl_api_prefix_t|`<class 'ipaddress.IPv4Network'> or <class 'ipaddress.IPv6Network'>` -vl_api_ip4_prefix_t|5|vl_api_ip4_prefix_t|`<class 'ipaddress.IPv4Network'>` -vl_api_ip6_prefix_t|17|vl_api_ip6_prefix_t|`<class 'ipaddress.IPv6Network'>` -vl_api_ip4_address_with_prefix_t|5|vl_api_ip4_address_with_prefix_t|`<class 'ipaddress.IPv4Interface'>` -vl_api_ip6_address_with_prefix_t|17|vl_api_ip6_address_with_prefix_t|`<class 'ipaddress.IPv6Interface'>` - -#### vnet/ethernet/ethernet_types.api -.api type|size|C type|Python type ----------|----|------|----------- -vl_api_mac_address_t|6|vl_api_mac_address_t|`class 'vpp_papi.MACAddress'>` - -#### vnet/interface_types.api -.api type|size|C type|Python type ----------|----|------|----------- -vl_api_interface_index_t|4|vl_api_interface_index_t|int - -### New explicit types - -#### String versus bytes -A byte string with a maximum length of 64: -``` -u8 name[64]; -``` -Before the "string" type was added, text string were defined like this. -The implications of that was the user would have to know if the field -represented a \0 ended C-string or a fixed length byte string. -The wire format of the 'string' type is a u32 length - -An IPv4 or IPv6 address was previously defined like: -``` -u8 is_ip6; -u8 address[16]; -``` - -Which made it hard for language bindings to represent the -address as anything but a byte string. -The new explicit address types are shown above. - -## Language generators - -The VPP API compiler currently has two output modules. One generating JSON -and one generating C header files that are directly used by the VPP -infrastructure and plugins. - -The C/C++, Python, Go Lua, and Java language bindings are generated based -on the JSON files. - -### Future considerations -- [ ] Generate C/C++ (vapi) client code directly from vppapigen -- [ ] Embed JSON definitions into the API server, so dynamic languages - can download them directly without going via the filesystem and JSON - files. diff --git a/src/tools/vppapigen/VPPAPI.rst b/src/tools/vppapigen/VPPAPI.rst new file mode 100644 index 00000000000..e8144803a87 --- /dev/null +++ b/src/tools/vppapigen/VPPAPI.rst @@ -0,0 +1,597 @@ +VPP API Language +================ + +The VPP binary API is a message passing API. The VPP API language is +used to define a RPC interface between VPP and its control plane. The +API messages supports shared memory transport and Unix domain sockets +(SOCK_STREAM). + +The wire format is essentially that of a network formatted (big-endian) +packed C struct. + +The VPP API compiler is located in *src/tools/vppapigen* and can +currently compile to JSON or C (used by the VPP binary itself). + +Language definition +------------------- + +Defining a messages +~~~~~~~~~~~~~~~~~~~ + +There are 3 types of message exchanges: + +- Request/Reply The client sends a request message and the server + replies with a single reply message. The convention is that the reply + message is named as method_name + \_reply. + +- Dump/Detail The client sends a “bulk” request message to the server, + and the server replies with a set of detail messages. These messages + may be of different type. A dump/detail call must be enclosed in a + control ping block (Otherwise the client will not know the end of the + bulk transmission). The method name must end with method + “\_dump”, + the reply message should be named method + “\_details”. The exception + here is for the methods that return multiple message types + (e.g. sw_interface_dump). The Dump/Detail methods are typically used + for acquiring bulk information, like the complete FIB table. + +- Events The client can register for getting asynchronous notifications + from the server. This is useful for getting interface state changes, + and so on. The method name for requesting notifications is + conventionally prefixed with “want\_”. E.g. “want_interface_events”. + Which notification types results from an event registration is + defined in the service definition. + +A message from a client must include the ‘client_index’, an opaque +cookie identifying the sender, and a ‘context’ field to let the client +match request with reply. + +An example of a message definition. The client sends the show_version +request, the server replies with the show_version_reply. + +The *client_index* and *context* fields are required in all requests. +The *context* is returned by the server and is used by the client to +match up request and reply messages. + +.. code-block:: c + + define show_version + { + u32 client_index; + u32 context; + }; + define show_version_reply + { + u32 context; + i32 retval; + string program [32]; + string version [32]; + string build_date [32]; + /* The final field can be a variable length argument */ + string build_directory []; + }; + +The flags are not used by the clients, but have special meaning for some +of the tracing and debugging of the API. The *autoreply* flag is a +shorthand for a reply message with just a *retval* field. + +.. code-block:: c + + define : DEFINE ID '{' block_statements_opt '}' ';' + define : flist DEFINE ID '{' block_statements_opt '}' ';' + flist : flag + | flist flag + flag : MANUAL_PRINT + | MANUAL_ENDIAN + | DONT_TRACE + | AUTOREPLY + + block_statements_opt : block_statements + block_statements : block_statement + | block_statements block_statement + block_statement : declaration + | option + declaration : type_specifier ID ';' + | type_specifier ID '[' ID '=' assignee ']' ';' + declaration : type_specifier ID '[' NUM ']' ';' + | type_specifier ID '[' ID ']' ';' + type_specifier : U8 + | U16 + | U32 + | U64 + | I8 + | I16 + | I32 + | I64 + | F64 + | BOOL + | STRING + type_specifier : ID + +Options +~~~~~~~ + +The *option* word is used to specify meta information. The only current +use is to specify a semantic version of the .api file itself. + +Example: + +.. code-block:: c + + option version = "1.0.0"; + +.. code-block:: c + + + option : OPTION ID '=' assignee ';' + assignee : NUM + | TRUE + | FALSE + | STRING_LITERAL + +Defining new types +~~~~~~~~~~~~~~~~~~ + +New user defined types are defined just like messages. A typedef has two +forms. It can either define an alias for a different type (or array). + +Example: + +.. code-block:: c + + typedef u8 ip4_address[4]; + typedef u8 ip6_address[16]; + +Where the above defines two new types *vl_api_ip4_address_t* and +*vl_api_ip6_address_t*. These are aliases for the underlying u8 array. + +In the other form, it is used to specify an abstract data type. + +.. code-block:: c + + enum address_family { + ADDRESS_IP4 = 0, + ADDRESS_IP6, + }; + + union address_union { + vl_api_ip4_address_t ip4; + vl_api_ip6_address_t ip6; + }; + + typedef address { + vl_api_address_family_t af; + vl_api_address_union_t un; + }; + +Where the new type *vl_api_address_t* + +.. code-block:: c + + typedef : TYPEDEF ID '{' block_statements_opt '}' ';' + typedef : TYPEDEF declaration + +Importing Definitions +~~~~~~~~~~~~~~~~~~~~~ + +You can use definitions from other .api files by importing them. To +import another .api’s definitions, you add an import statement to the +top of your file: + +import “vnet/ip/ip_types.api”; + +By default you can only use definitions from directly imported .api +files. + +The API compiler searches for imported files in a set of directories +specified on the API compiler command line using the –includedir flag. + +.. code-block:: c + + import : IMPORT STRING_LITERAL ';' + +Comments +~~~~~~~~ + +The API language uses C style comments. + +.. code-block:: c + + /* */ + // + +Enumerations +~~~~~~~~~~~~ + +Enums are similar to enums in C. + +Every enum definition must contain a constant that maps to zero as its +first element. This is because: + +There must be a zero value, so that we can use 0 as a numeric default +value. The zero value needs to be the first element. + +As in C, enums can be used as flags or just as numbers. The on-wire, and +in memory representation size of an enum can be specified. Not all +language bindings will support that. The default size is 4 (u32). + +Example + +.. code-block:: c + + enum ip_neighbor_flags + { + IP_API_NEIGHBOR_FLAG_NONE = 0, + IP_API_NEIGHBOR_FLAG_STATIC = 0x1, + IP_API_NEIGHBOR_FLAG_NO_FIB_ENTRY = 0x2, + }; + +Which generates the vl_api_ip_neighbor_flags_t in the C binding. In +Python that is represented as an IntFlag object +VppEnum.vl_api_ip_neighbor_flags_t. + +.. code-block:: c + + enum : ENUM ID '{' enum_statements '}' ';' + enum : ENUM ID ':' enum_size '{' enum_statements '}' ';' + enum_size : U8 + | U16 + | U32 + enum_statements : enum_statement + | enum_statements enum_statement + enum_statement : ID '=' NUM ',' + | ID ',' + +Services +~~~~~~~~ + +The service statement defines the relationship between messages. For +request/response and dump/details messages it ties the request with the +reply. For events, it specifies which events that can be received for a +given ``want_*`` call. + +Example: + +.. code-block:: c + + service { + rpc want_interface_events returns want_interface_events_reply + events sw_interface_event; + }; + +Which states that the request want_interface_events returns a +want_interface_events_reply and if enabled the client will receive +sw_interface_event messages whenever interface states changes. + +.. code-block:: c + + service : SERVICE '{' service_statements '}' ';' + service_statements : service_statement + | service_statements service_statement + service_statement : RPC ID RETURNS NULL ';' + | RPC ID RETURNS ID ';' + | RPC ID RETURNS STREAM ID ';' + | RPC ID RETURNS ID EVENTS event_list ';' + event_list : events + | event_list events + events : ID + | ID ',' + +Types +----- + +Scalar Value Types +~~~~~~~~~~~~~~~~~~ + +========= ======== =============== =========== +.api type size C type Python type +========= ======== =============== =========== +i8 1 i8 int +u8 1 u8 int +i16 2 i16 int +u16 2 u16 int +i32 4 i32 int +u32 4 u32 int +i64 8 i64 int +u64 8 u64 int +f64 8 f64 float +bool 1 bool boolean +string variable vl_api_string_t str +========= ======== =============== =========== + +User Defined Types +~~~~~~~~~~~~~~~~~~ + +vnet/ip/ip_types.api +^^^^^^^^^^^^^^^^^^^^ + ++--------------------+--------+-------------+-------------------------+ +| .api type | size | C type | Python type | ++====================+========+=============+=========================+ +| vl_api_address_t | 20 | vl_ap | ` | +| | | i_address_t | `<class 'ipaddress.IPv4 | +| | | | Address'> or <class 'ip | +| | | | address.IPv6Address'>`` | ++--------------------+--------+-------------+-------------------------+ +| vl | 4 | vl_api_ip | ``<class 'ip | +| _api_ip4_address_t | | 4_address_t | address.IPv4Address'>`` | ++--------------------+--------+-------------+-------------------------+ +| vl | 16 | vl_api_ip | ``<class 'ip | +| _api_ip6_address_t | | 6_address_t | address.IPv6Address'>`` | ++--------------------+--------+-------------+-------------------------+ +| vl_api_prefix_t | 21 | vl_a | ` | +| | | pi_prefix_t | `<class 'ipaddress.IPv4 | +| | | | Network'> or <class 'ip | +| | | | address.IPv6Network'>`` | ++--------------------+--------+-------------+-------------------------+ +| v | 5 | vl_api_i | ``<class 'ip | +| l_api_ip4_prefix_t | | p4_prefix_t | address.IPv4Network'>`` | ++--------------------+--------+-------------+-------------------------+ +| v | 17 | vl_api_i | ``<class 'ip | +| l_api_ip6_prefix_t | | p6_prefix_t | address.IPv6Network'>`` | ++--------------------+--------+-------------+-------------------------+ +| vl_api_ip4_add | 5 | vl_api_ip4 | ``<class 'ipad | +| ress_with_prefix_t | | _address_wi | dress.IPv4Interface'>`` | +| | | th_prefix_t | | ++--------------------+--------+-------------+-------------------------+ +| vl_api_ip6_add | 17 | vl_api_ip6 | ``<class 'ipad | +| ress_with_prefix_t | | _address_wi | dress.IPv6Interface'>`` | +| | | th_prefix_t | | ++--------------------+--------+-------------+-------------------------+ + +vnet/ethernet/ethernet_types.api +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ++---------------------+------+---------------------+-------------------+ +| .api type | size | C type | Python type | ++=====================+======+=====================+===================+ +| ``vl_ | 6 | ``vl_ | ``class 'vpp_pa | +| api_mac_address_t`` | | api_mac_address_t`` | pi.MACAddress'>`` | ++---------------------+------+---------------------+-------------------+ + +vnet/interface_types.api +^^^^^^^^^^^^^^^^^^^^^^^^ + +======================== ==== ======================== =========== +.api type size C type Python type +======================== ==== ======================== =========== +vl_api_interface_index_t 4 vl_api_interface_index_t int +======================== ==== ======================== =========== + +New explicit types +~~~~~~~~~~~~~~~~~~ + +String versus bytes +^^^^^^^^^^^^^^^^^^^ + +A byte string with a maximum length of 64: + +.. code-block:: c + + u8 name[64]; + +Before the “string” type was added, text string were defined like this. +The implications of that was the user would have to know if the field +represented a \\0 ended C-string or a fixed length byte string. The wire +format of the ‘string’ type is a u32 length + +An IPv4 or IPv6 address was previously defined like: + +.. code-block:: c + + u8 is_ip6; + u8 address[16]; + +Which made it hard for language bindings to represent the address as +anything but a byte string. The new explicit address types are shown +above. + +Language generators +------------------- + +The VPP API compiler currently has two output modules. One generating +JSON and one generating C header files that are directly used by the VPP +infrastructure and plugins. + +The C/C++, Python, Go Lua, and Java language bindings are generated +based on the JSON files. + +Future considerations +~~~~~~~~~~~~~~~~~~~~~ + +- Generate C/C++ (vapi) client code directly from vppapigen +- Embed JSON definitions into the API server, so dynamic languages + can download them directly without going via the filesystem and JSON + files. + +API Change Process +------------------ + +Purpose +~~~~~~~ + +To minimize the disruptions to the consumers of the VPP API, while permitting +the innovation for the VPP itself. + +Historically, API changes in VPP master branch were allowed at any point in time +outside of a small window between the API freeze milestone and RC1 milestone. +The API changes on the throttle branches were not permitted at all. This model +proved workable, however all the production use cases ended up on throttle +branches, with a lot of forklift activity when it is the time to upgrade to the +next branch. + +This formally structured API change process harmonizes the behavior across all +the VPP branches, and allows more flexibility for the consumer, while permitting +the innovation in the VPP itself. + +The Core Promise +~~~~~~~~~~~~~~~~ + +"If a user is running a VPP version N and does not use any deprecated APIs, they +should be able to simply upgrade the VPP to version N+1 and there should be no +API breakage". + +In-Progress, Production and Deprecated APIs +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This proposal adds a classification of stability of an API call: + +- "In-Progress": APIs in the process of the development, experimentation, and + limited testing. + +- "Production": tested as part of the "make test", considered stable for general + usage. + +- "Deprecated": used as a flag on Production APIs which are slated to be + deprecated in the future release. + +The "In-Progress" APIs or the APIs with the semantic version of 0.x.y are not +subject to any stability checks, thus the developers are free to introduce them, +modify their signatures, and as well remove them completely at will. The users +should not use the in-progress APIs without the interactions with its +maintainers, nor base the production code on those APIs. The goal of +"in-progress" APIs to allow rapid iteration and modifications to ensure the API +signature and function is stabilized. These API calls may be used for testing or +experimentation and prototyping. + +When the maintainer is satisfied with the quality of the APIs, and ensures that +they are tested as part of the "Make test" runs, they can transition their +status to "Production". + +The "Production" APIs can *NOT* be changed in any way that modifies their +representation on the wire and the signature (thus CRC). The only change that +they may incur is to be marked as "Deprecated". These are the APIs that the +downstream users can use for production purposes. They exist to fulfill a core +promise of this process: The "Deprecated" APIs are the "Production" APIs that +are about to be deleted. To ensure the above core promise is maintained, if the +API call was marked as deprecated at any point between RC1 of release N and RC1 +of release N+1, it MUST NOT be deleted until the RC1 milestone of the +release N+2. The deprecated API SHOULD specify a replacement API - which MUST +be a Production API, so as not to decrease the level of stability. + + +The time interval between a commit that marks an API as deprecated and a commit +that deletes that API MUST be at least equal the time between the two subsequent +releases (currently 4 months). + + +Doing so allows a for a good heads-up to those who are using the +"one free upgrade" property to proactively catch and test the transition from +the deprecated APIs using the master. + + +Marking an API as deprecated just 1 day before RC1 branch pull and then deleting +that API one day after does *technically* satisfy "one free upgrade" promise, +but is rather hostile to the users that are proactively tracking it. + +Semantic API Versioning +~~~~~~~~~~~~~~~~~~~~~~~ + +VPP APIs use semantic versioning according to semver.org, with the compatibility +logic being applied at the moment the messages are marked as deprecated. + +To discuss: i.e. if message_2 is being introduced which deprecates the +message_1, then that same commit should increase the major version of the API. + +The 0.x.x API versions, by virtue of being in-progress, are exempt from this +treatment. + +Tooling +~~~~~~~ + +See https://gerrit.fd.io/r/c/vpp/+/26881: + +crcchecker.py is a tool to enforce the policy, with a few other bonus uses: + +extras/scripts/crcchecker.py --check-patchset # returns -1 if backwards incompatible extras/scripts/crcchecker.py --dump-manifest extras/scripts/crcchecker.py --git-revision v20.01 <files> extras/scripts/crcchecker.py -- diff <oldfile> <newfile> + +Notice that you can use this tool to get the list of API changes since a given past release. + +The policy: + +.. highlight:: none + +.. code-block:: + + 1. Production APIs should never change. + The definition of a "production API" is if the major version in + the API file is > 0 that is not marked as "in-progress". + 2. APIs that are experimental / not released are not checked. + An API message can be individually marked as in progress, + by adding the following in the API definition: + option in_progress; + 3. An API can be deprecated in three-to-six steps (the steps + with letters can be combined or split, depending on situation): + Step 1a: A new "in-progress" API new_api_2 is added that + is deemed to be a replacement. + Step 1b: The existing API is marked as "replaced_by" this new API: + option replaced_by="new_api_2"; + Step 2a: The new_api_2 is marked as production by deleting its in-progress status, + provided that this API does have sufficient test coverage to deem it well tested. + Step 2b: the existing API is marked as "deprecated": + option deprecated="optional short message to humans reading it"; + Step 3: the deprecated API is deleted. + +There is a time constraint that the minimum interval between the steps 2 and 3 +must be at least 4 months. The proposal is to have step 2 around a couple of +weeks before the F0 milestone for a release, as triggered by the release manager +(and in the future by an automated means). + +Use Cases +~~~~~~~~~ + +Adding A New Field To A Production API +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The simplest way to add a new field to a Production API message *foo_message* is +to create a new In-Progress message *foo_message_v2*, and add the field to that +one. Typically it will be an extension - so the API message handlers are +trivially chained. If there are changes/adjustments that are needed, this new +message can be freely altered without bothering the users of the Production API. + +When the maintainer is happy with the quality of the implementation, and the +foo_message_v2 is tested in "make test" to the same extent as the foo_message, +they can make two commits: one, removing the in-progress status for +foo_message_v2, and the second one - deprecating foo_message and pointing the +foo_message_v2 as the replacement. Technically after the next throttle pull, +they can delete the foo_message - the deprecation and the replacement will be +already in the corresponding branch. + +Rapid Experimentation For A New Feature +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Add a message that is in-progress, and keep iterating with this message. This +message is not subject to the change control process. + +An In-progress API Accidentally Marked As "production" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This is expected to mainly apply during the initial period of 20.05->20.09, the +proposal is to have it active for 4 weeks from Jun 17 till July 15th, with the +following process. + +If a developer finds that a given API or a set of APIs is not ready for +production due to lack of tests and/or the general API stability, then they: + +- Create a new gerrit change with *just* the marking of the API as + in_progress, subject being: "api: <feature> api message downgrade" and + a comment identifying which APIs are being downgraded and why. + +- Add ayourtch@gmail.com or the current Release Manager as a reviewer -- + for help in guiding the process and to ensure that the gerrit change is not + forgotten. + +- Send an email to vpp-dev mailing list with the subject being the same as the + one-liner commit message, reference to the gerrit change, and the reasoning. + +- Wait for the timeout period of two weeks for the feedback. + +- If no feedback received, assume the community agreement and commit the + change to master branch. + +This needs to be highlighted that this process is an *exception* - normally the +transition is always in_progress => production => deprecated. + +API Change Examples +~~~~~~~~~~~~~~~~~~~ + +https://gerrit.fd.io/r/q/+is:merged+message:%2522%255Eapi:.*%2524%2522 diff --git a/src/tools/vppapigen/generate_go.py b/src/tools/vppapigen/generate_go.py index 1e072ef1e1c..4ed507b5d73 100755 --- a/src/tools/vppapigen/generate_go.py +++ b/src/tools/vppapigen/generate_go.py @@ -5,6 +5,7 @@ import os import pathlib import subprocess import tarfile +import shutil import requests import sys @@ -13,58 +14,46 @@ import sys # GoVPP API generator generates Go bindings compatible with the local VPP # -parser = argparse.ArgumentParser() -parser.add_argument("-govpp-commit", help="GoVPP commit or branch (defaults to v0.3.5-45-g671f16c)", - default="671f16c", # fixed GoVPP version - type=str) -parser.add_argument("-output-dir", help="output target directory for generated bindings", type=str) -parser.add_argument("-api-files", help="api files to generate (without commas)", nargs="+", type=str) -parser.add_argument("-import-prefix", help="prefix imports in the generated go code", type=str) -parser.add_argument("-no-source-path-info", help="disable source path info in generated files", nargs='?', const=True, - default=False) -args = parser.parse_args() - - -# Check input arguments -def validate_args(vpp_dir, o, f, c, i): - if o is not None: - if not os.path.exists(o) or os.path.isfile(o): - print(o + " is not a valid output path") - sys.exit(1) - else: - o = vpp_dir - if f is None: - f = [] - if c is None: - c = "671f16c" - if i is None: - i = "" - - return str(o), f, c, i +DefaultGoVppCommit = "16a47ef937b3a5ce6acf45885386062b323c8d25" + + +def version_geq(ver_a, ver_b): + major_a, minor_a, patch_a = ver_a.split(".") + major_b, minor_b, patch_b = ver_b.split(".") + if major_a > major_b: + return True + elif major_a == major_b and minor_a > minor_b: + return True + elif major_a == major_b and minor_a == minor_b and patch_a >= patch_b: + return True + return False + + +def execute(cli, cwd=None): + p = subprocess.Popen( + cli.split(), + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + output, error = p.communicate() + if p.returncode != 0: + print("Command `%s` failed: %d %s" % (cli, p.returncode, error)) + sys.exit(1) + return output, error -# Returns version of the installed Go def get_go_version(go_root): - p = subprocess.Popen(["./go", "version"], - cwd=go_root + "/bin", - stdout=subprocess.PIPE, - universal_newlines=True, ) - output, _ = p.communicate() - output_fmt = output.replace("go version go", "", 1) - - return output_fmt.rstrip("\n") + # Returns version of the installed Go + output, _ = execute("./go version", cwd=go_root + "/bin") + return output.replace("go version go", "", 1).rstrip("\n") # Returns version of the installed binary API generator def get_binapi_gen_version(go_path): - p = subprocess.Popen(["./binapi-generator", "-version"], - cwd=go_path + "/bin", - stdout=subprocess.PIPE, - universal_newlines=True, ) - output, _ = p.communicate() - output_fmt = output.replace("govpp", "", 1) - - return output_fmt.rstrip("\n") + output, _ = execute("./binapi-generator -version", cwd=go_path + "/bin") + return output.replace("govpp", "", 1).rstrip("\n") # Verifies local Go installation and installs the latest @@ -73,20 +62,30 @@ def install_golang(go_root): go_bin = go_root + "/bin/go" if os.path.exists(go_bin) and os.path.isfile(go_bin): - print('Go ' + get_go_version(go_root) + ' is already installed') + print("Go " + get_go_version(go_root) + " is already installed") return + filename = ( + requests.get("https://golang.org/VERSION?m=text").text + ".linux-amd64.tar.gz" + ) + url = "https://dl.google.com/go/" + filename + print("Go binary not found, installing the latest version...") - go_folders = ['src', 'pkg', 'bin'] + print("Download url = %s" % url) + print("Install directory = %s" % go_root) + text = input("[Y/n] ?") + + if text.strip().lower() != "y" and text.strip().lower() != "yes": + print("Aborting...") + exit(1) + + go_folders = ["src", "pkg", "bin"] for f in go_folders: if not os.path.exists(os.path.join(go_root, f)): os.makedirs(os.path.join(go_root, f)) - - filename = requests.get('https://golang.org/VERSION?m=text').text + ".linux-amd64.tar.gz" - url = "https://dl.google.com/go/" + filename r = requests.get(url) - with open("/tmp/" + filename, 'wb') as f: + with open("/tmp/" + filename, "wb") as f: f.write(r.content) go_tf = tarfile.open("/tmp/" + filename) @@ -97,29 +96,29 @@ def install_golang(go_root): go_tf.close() os.remove("/tmp/" + filename) - print('Go ' + get_go_version(go_root) + ' was installed') + print("Go " + get_go_version(go_root) + " was installed") # Installs latest binary API generator def install_binapi_gen(c, go_root, go_path): - os.environ['GO111MODULE'] = "on" - if os.path.exists(go_root + "/bin/go") & os.path.isfile(go_root + "/bin/go"): - p = subprocess.Popen(["./go", "get", "git.fd.io/govpp.git/cmd/binapi-generator@" + c], - cwd=go_root + "/bin", - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, ) - _, error = p.communicate() - if p.returncode != 0: - print("binapi generator installation failed: %d %s" % (p.returncode, error)) - sys.exit(1) + go_version = get_go_version(go_root) + if version_geq(go_version, "1.18.0"): + execute( + "./go install git.fd.io/govpp.git/cmd/binapi-generator@" + c, + cwd=go_root + "/bin", + ) + else: + os.environ["GO111MODULE"] = "on" + execute( + "./go get git.fd.io/govpp.git/cmd/binapi-generator@" + c, + cwd=go_root + "/bin", + ) bg_ver = get_binapi_gen_version(go_path) - print('Installed binary API generator ' + bg_ver) + print("Installed binary API generator " + bg_ver) # Creates generated bindings using GoVPP binapigen to the target folder def generate_api(output_dir, vpp_dir, api_list, import_prefix, no_source, go_path): - output_binapi = output_dir + "binapi" if output_dir[-1] == "/" else output_dir + "/binapi" json_dir = vpp_dir + "/build-root/install-vpp-native/vpp/share/vpp/api" if not os.path.exists(json_dir): @@ -127,25 +126,19 @@ def generate_api(output_dir, vpp_dir, api_list, import_prefix, no_source, go_pat sys.exit(1) print("Generating API") - cmd = ["./binapi-generator", "--output-dir=" + output_binapi, "--input-dir=" + json_dir] + cmd = ["./binapi-generator", "--input-dir=" + json_dir] + if output_dir: + cmd += ["--output-dir=" + output_dir] if len(api_list): print("Following API files were requested by 'GO_API_FILES': " + str(api_list)) print("Note that dependency requirements may generate additional API files") cmd.append(api_list) - if not import_prefix == "": + if import_prefix: cmd.append("-import-prefix=" + import_prefix) if no_source: cmd.append("-no-source-path-info") - p = subprocess.Popen(cmd, cwd=go_path + "/bin", - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, ) - - out = p.communicate()[1] - if p.returncode != 0: - print("go api generate failed: %d %s" % (p.returncode, out)) - sys.exit(1) + _, out = execute(" ".join(cmd), cwd=go_path + "/bin") # Print nice output of the binapi generator for msg in out.split(): if "=" in msg: @@ -153,30 +146,91 @@ def generate_api(output_dir, vpp_dir, api_list, import_prefix, no_source, go_pat print(msg, end=" ") print("\n") - print("Go API bindings were generated to " + output_binapi) + print("Go API bindings were generated to " + output_dir) -def main(): - # project root directory - root = pathlib.Path(os.path.dirname(os.path.abspath(__file__))) - vpp_dir: str = root.parent.parent.parent - - o, f, c, i = validate_args(vpp_dir, args.output_dir, args.api_files, args.govpp_commit, - args.import_prefix) - +def get_go_variables(): # go specific environment variables if "GOROOT" in os.environ: - go_root = os.environ['GOROOT'] + go_root = os.environ["GOROOT"] else: - go_root = os.environ['HOME'] + "/.go" + go_binary = shutil.which("go") + if go_binary != "": + go_binary_dir, _ = os.path.split(go_binary) + go_root = os.path.join(go_binary_dir, "..") + else: + go_root = os.environ["HOME"] + "/.go" if "GOPATH" in os.environ: - go_path = os.environ['GOPATH'] + go_path = os.environ["GOPATH"] else: - go_path = os.environ['HOME'] + "/go" + go_path = os.environ["HOME"] + "/go" + + return go_root, go_path + +def main(): + # project root directory + root = pathlib.Path(os.path.dirname(os.path.abspath(__file__))) + vpp_dir = root.parent.parent.parent + + parser = argparse.ArgumentParser() + parser.add_argument( + "-govpp-commit", + "--govpp-commit", + help="GoVPP commit or branch ", + default=DefaultGoVppCommit, + type=str, + ) + parser.add_argument( + "-output-dir", + "--output-dir", + help="output target directory for generated bindings", + type=str, + default=os.path.join(vpp_dir, "vppbinapi"), + ) + parser.add_argument( + "-api-files", + "--api-files", + help="api files to generate (without commas)", + nargs="+", + type=str, + default=[], + ) + parser.add_argument( + "-import-prefix", + "--import-prefix", + help="prefix imports in the generated go code", + default="", + type=str, + ) + parser.add_argument( + "-no-source-path-info", + "--no-source-path-info", + help="disable source path info in generated files", + nargs="?", + const=True, + default=True, + ) + args = parser.parse_args() + + go_root, go_path = get_go_variables() install_golang(go_root) - install_binapi_gen(c, go_root, go_path) - generate_api(o, str(vpp_dir), f, i, args.no_source_path_info, go_path) + + if not ( + os.path.exists(go_root + "/bin/go") and os.path.isfile(go_root + "/bin/go") + ): + print(go_root + "/bin/go does not exist") + sys.exit(1) + + install_binapi_gen(args.govpp_commit, go_root, go_path) + generate_api( + args.output_dir, + str(vpp_dir), + args.api_files, + args.import_prefix, + args.no_source_path_info, + go_path, + ) if __name__ == "__main__": diff --git a/src/tools/vppapigen/generate_json.py b/src/tools/vppapigen/generate_json.py index 288e519edd2..dc5cf9c1bbf 100755 --- a/src/tools/vppapigen/generate_json.py +++ b/src/tools/vppapigen/generate_json.py @@ -16,30 +16,37 @@ import argparse import pathlib import subprocess -BASE_DIR = subprocess.check_output('git rev-parse --show-toplevel', - shell=True).strip().decode() -vppapigen_bin = pathlib.Path( - '%s/src/tools/vppapigen/vppapigen.py' % BASE_DIR).as_posix() +import vppapigen +import os +from multiprocessing import Pool + +BASE_DIR = ( + subprocess.check_output("git rev-parse --show-toplevel", shell=True) + .strip() + .decode() +) src_dir_depth = 3 output_path = pathlib.Path( - '%s/build-root/install-vpp-native/vpp/share/vpp/api/' % BASE_DIR) + "%s/build-root/install-vpp-native/vpp/share/vpp/api/" % BASE_DIR +) output_path_debug = pathlib.Path( - '%s/build-root/install-vpp_debug-native/vpp/share/vpp/api/' % BASE_DIR) + "%s/build-root/install-vpp_debug-native/vpp/share/vpp/api/" % BASE_DIR +) output_dir_map = { - 'plugins': 'plugins', - 'vlibmemory': 'core', - 'vnet': 'core', - 'vlib': 'core', - 'vpp': 'core', + "plugins": "plugins", + "vlibmemory": "core", + "vnet": "core", + "vlib": "core", + "vpp": "core", } def api_search_globs(src_dir): globs = [] for g in output_dir_map: - globs.extend(list(src_dir.glob('%s/**/*.api' % g))) + globs.extend(list(src_dir.glob("%s/**/*.api" % g))) return globs @@ -48,31 +55,30 @@ def api_files(src_dir): return [x for x in api_search_globs(src_dir)] -def vppapigen(vppapigen_bin, output_path, src_dir, src_file): - try: - subprocess.check_output( - [vppapigen_bin, '--includedir', src_dir.as_posix(), - '--input', src_file.as_posix(), 'JSON', - '--output', '%s/%s/%s.json' % ( - output_path, - output_dir_map[src_file.as_posix().split('/')[ - src_dir_depth + BASE_DIR.count('/') - 1]], - src_file.name)]) - except KeyError: - print('src_file: %s' % src_file) - raise +def get_n_parallel(n_parallel): + if n_parallel == 0: + n_parallel = os.environ.get("MAKE_PARALLEL_JOBS", os.cpu_count()) + try: + n_parallel = int(n_parallel) + except ValueError: + return os.cpu_count() + return n_parallel or os.cpu_count() def main(): - cliparser = argparse.ArgumentParser( - description='VPP API JSON definition generator') - cliparser.add_argument('--srcdir', action='store', - default='%s/src' % BASE_DIR), - cliparser.add_argument('--output', action='store', - help='directory to store files'), - cliparser.add_argument('--debug-target', action='store_true', - default=False, - help="'True' if -debug target"), + cliparser = argparse.ArgumentParser(description="VPP API JSON definition generator") + cliparser.add_argument("--srcdir", action="store", default="%s/src" % BASE_DIR), + cliparser.add_argument("--output", action="store", help="directory to store files"), + cliparser.add_argument( + "--parallel", type=int, default=0, help="Number of parallel processes" + ), + cliparser.add_argument( + "--debug-target", + action="store_true", + default=False, + help="'True' if -debug target", + ), + args = cliparser.parse_args() src_dir = pathlib.Path(args.srcdir) @@ -86,13 +92,43 @@ def main(): for d in output_dir_map.values(): output_dir.joinpath(d).mkdir(exist_ok=True, parents=True) - for f in output_dir.glob('**/*.api.json'): + for f in output_dir.glob("**/*.api.json"): f.unlink() - for f in api_files(src_dir): - vppapigen(vppapigen_bin, output_dir, src_dir, f) - print('json files written to: %s/.' % output_dir) - - -if __name__ == '__main__': + with Pool(get_n_parallel(args.parallel)) as p: + p.map( + vppapigen.run_kw_vppapigen, + [ + { + "output": "%s/%s/%s.json" + % ( + output_path, + output_dir_map[ + f.as_posix().split("/")[ + src_dir_depth + BASE_DIR.count("/") - 1 + ] + ], + f.name, + ), + "outputdir": "%s/%s/" + % ( + output_path, + output_dir_map[ + f.as_posix().split("/")[ + src_dir_depth + BASE_DIR.count("/") - 1 + ] + ], + ), + "input_file": f.as_posix(), + "includedir": [src_dir.as_posix()], + "output_module": "JSON", + } + for f in api_files(src_dir) + ], + ) + + print("json files written to: %s/." % output_dir) + + +if __name__ == "__main__": main() diff --git a/src/tools/vppapigen/test_vppapigen.py b/src/tools/vppapigen/test_vppapigen.py index c454ffc8638..20f6c6da10d 100755 --- a/src/tools/vppapigen/test_vppapigen.py +++ b/src/tools/vppapigen/test_vppapigen.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 import unittest -from vppapigen import VPPAPI, Option, ParseError, Union, foldup_crcs, \ - global_types +from vppapigen import VPPAPI, Option, ParseError, Union, foldup_crcs, global_types import vppapigen @@ -28,17 +27,17 @@ class TestUnion(unittest.TestCase): cls.parser = VPPAPI() def test_union(self): - test_string = ''' + test_string = """ union foo_union { u32 a; u8 b; }; - ''' + """ r = self.parser.parse_string(test_string) self.assertTrue(isinstance(r[0], Union)) def test_union_vla(self): - test_string = ''' + test_string = """ union foo_union_vla { u32 a; u8 b[a]; @@ -46,13 +45,13 @@ class TestUnion(unittest.TestCase): autoreply define foo { vl_api_foo_union_vla_t v; }; - ''' + """ r = self.parser.parse_string(test_string) self.assertTrue(isinstance(r[0], Union)) self.assertTrue(r[0].vla) s = self.parser.process(r) - test_string2 = ''' + test_string2 = """ union foo_union_vla2 { u32 a; u8 b[a]; @@ -61,10 +60,10 @@ class TestUnion(unittest.TestCase): autoreply define foo2 { vl_api_foo_union_vla2_t v; }; - ''' + """ self.assertRaises(ValueError, self.parser.parse_string, test_string2) - test_string3 = ''' + test_string3 = """ union foo_union_vla3 { u32 a; u8 b[a]; @@ -73,7 +72,7 @@ class TestUnion(unittest.TestCase): vl_api_foo_union_vla3_t v; u32 x; }; - ''' + """ self.assertRaises(ValueError, self.parser.parse_string, test_string3) @@ -83,10 +82,10 @@ class TestTypedef(unittest.TestCase): cls.parser = VPPAPI() def test_duplicatetype(self): - test_string = ''' + test_string = """ typedef foo1 { u8 dummy; }; typedef foo1 { u8 dummy; }; - ''' + """ self.assertRaises(KeyError, self.parser.parse_string, test_string) @@ -96,42 +95,42 @@ class TestDefine(unittest.TestCase): cls.parser = VPPAPI() def test_unknowntype(self): - test_string = 'define foo { foobar foo;};' + test_string = "define foo { foobar foo;};" with self.assertRaises(ParseError) as ctx: self.parser.parse_string(test_string) - self.assertIn('Undefined type: foobar', str(ctx.exception)) + self.assertIn("Undefined type: foobar", str(ctx.exception)) - test_string = 'define { u8 foo;};' + test_string = "define { u8 foo;};" with self.assertRaises(ParseError) as ctx: self.parser.parse_string(test_string) def test_flags(self): - test_string = ''' + test_string = """ manual_print dont_trace manual_endian define foo { u8 foo; }; define foo_reply {u32 context; i32 retval; }; - ''' + """ r = self.parser.parse_string(test_string) self.assertIsNotNone(r) s = self.parser.process(r) self.assertIsNotNone(s) - for d in s['Define']: - if d.name == 'foo': + for d in s["Define"]: + if d.name == "foo": self.assertTrue(d.dont_trace) self.assertTrue(d.manual_endian) self.assertTrue(d.manual_print) self.assertFalse(d.autoreply) - test_string = ''' + test_string = """ nonexisting_flag define foo { u8 foo; }; - ''' + """ with self.assertRaises(ParseError): self.parser.parse_string(test_string) def test_options(self): - test_string = ''' + test_string = """ define foo { option deprecated; u8 foo; }; define foo_reply {u32 context; i32 retval; }; - ''' + """ r = self.parser.parse_string(test_string) self.assertIsNotNone(r) s = self.parser.process(r) @@ -144,14 +143,14 @@ class TestService(unittest.TestCase): cls.parser = VPPAPI() def test_service(self): - test_string = ''' + test_string = """ autoreply define show_version { u8 foo;}; service { rpc show_version returns show_version_reply; }; - ''' + """ r = self.parser.parse_string(test_string) s = self.parser.process(r) - self.assertEqual(s['Service'][0].caller, 'show_version') - self.assertEqual(s['Service'][0].reply, 'show_version_reply') + self.assertEqual(s["Service"][0].caller, "show_version") + self.assertEqual(s["Service"][0].reply, "show_version_reply") def get_crc(apistring, name): @@ -159,52 +158,52 @@ def get_crc(apistring, name): parser = vppapigen.VPPAPI() r = parser.parse_string(apistring) s = parser.process(r) - foldup_crcs(s['Define']) - d = [f for f in s['Define'] if f.name == name] + foldup_crcs(s["Define"]) + d = [f for f in s["Define"] if f.name == name] return d[0].crc class TestCRC(unittest.TestCase): def test_crc(self): - test_string = ''' + test_string = """ typedef list { u8 foo; }; autoreply define foo { u8 foo; vl_api_list_t l;}; - ''' - crc = get_crc(test_string, 'foo') + """ + crc = get_crc(test_string, "foo") # modify underlying type - test_string = ''' + test_string = """ typedef list { u8 foo2; }; autoreply define foo { u8 foo; vl_api_list_t l;}; - ''' - crc2 = get_crc(test_string, 'foo') + """ + crc2 = get_crc(test_string, "foo") self.assertNotEqual(crc, crc2) # two user-defined types - test_string = ''' + test_string = """ typedef address { u8 foo2; }; typedef list { u8 foo2; vl_api_address_t add; }; autoreply define foo { u8 foo; vl_api_list_t l;}; - ''' - crc3 = get_crc(test_string, 'foo') + """ + crc3 = get_crc(test_string, "foo") - test_string = ''' + test_string = """ typedef address { u8 foo3; }; typedef list { u8 foo2; vl_api_address_t add; }; autoreply define foo { u8 foo; vl_api_list_t l;}; - ''' - crc4 = get_crc(test_string, 'foo') + """ + crc4 = get_crc(test_string, "foo") self.assertNotEqual(crc3, crc4) - test_string = ''' + test_string = """ typedef address { u8 foo3; }; typedef list { u8 foo2; vl_api_address_t add; u8 foo3; }; autoreply define foo { u8 foo; vl_api_list_t l;}; - ''' - crc5 = get_crc(test_string, 'foo') + """ + crc5 = get_crc(test_string, "foo") self.assertNotEqual(crc4, crc5) - test_string = ''' + test_string = """ typedef ip6_address { u8 foo; @@ -227,11 +226,11 @@ autoreply define sr_policy_add u32 fib_table; vl_api_srv6_sid_list_t sids; }; -''' +""" - crc = get_crc(test_string, 'sr_policy_add') + crc = get_crc(test_string, "sr_policy_add") - test_string = ''' + test_string = """ typedef ip6_address { u8 foo; @@ -253,14 +252,13 @@ autoreply define sr_policy_add u32 fib_table; vl_api_srv6_sid_list_t sids; }; -''' - crc2 = get_crc(test_string, 'sr_policy_add') +""" + crc2 = get_crc(test_string, "sr_policy_add") self.assertNotEqual(crc, crc2) class TestEnum(unittest.TestCase): - @classmethod def setUpClass(cls): cls.parser = VPPAPI() @@ -278,8 +276,8 @@ enum tunnel_mode : u8 r = self.parser.parse_string(test_string) self.assertIsNotNone(r) s = self.parser.process(r) - for o in s['types']: - if o.type == 'Enum': + for o in s["types"]: + if o.type == "Enum": self.assertEqual(o.name, "tunnel_mode") break else: @@ -298,8 +296,8 @@ enum virtio_flags { r = self.parser.parse_string(test_string) self.assertIsNotNone(r) s = self.parser.process(r) - for o in s['types']: - if o.type == 'Enum': + for o in s["types"]: + if o.type == "Enum": self.assertEqual(o.name, "virtio_flags") break else: @@ -307,7 +305,6 @@ enum virtio_flags { class TestEnumFlag(unittest.TestCase): - @classmethod def setUpClass(cls): cls.parser = VPPAPI() @@ -326,8 +323,9 @@ enumflag tunnel_mode_ef : u8 with self.assertRaises(TypeError) as ctx: r = self.parser.parse_string(test_string) - self.assertTrue(str(ctx.exception).startswith( - 'tunnel_mode_ef is not a flag enum.')) + self.assertTrue( + str(ctx.exception).startswith("tunnel_mode_ef is not a flag enum.") + ) def test_enumflag_as_enumflag(self): test_string = """\ @@ -342,13 +340,13 @@ enumflag virtio_flags_ef { r = self.parser.parse_string(test_string) self.assertIsNotNone(r) s = self.parser.process(r) - for o in s['types']: - if o.type == 'EnumFlag': + for o in s["types"]: + if o.type == "EnumFlag": self.assertEqual(o.name, "virtio_flags_ef") break else: self.fail() -if __name__ == '__main__': +if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/src/tools/vppapigen/vppapigen.py b/src/tools/vppapigen/vppapigen.py index 8415c28fb7b..2b0ce9999d7 100755 --- a/src/tools/vppapigen/vppapigen.py +++ b/src/tools/vppapigen/vppapigen.py @@ -10,9 +10,10 @@ from subprocess import Popen, PIPE import ply.lex as lex import ply.yacc as yacc -assert sys.version_info >= (3, 5), \ - "Not supported Python version: {}".format(sys.version) -log = logging.getLogger('vppapigen') +assert sys.version_info >= (3, 5), "Not supported Python version: {}".format( + sys.version +) +log = logging.getLogger("vppapigen") # Ensure we don't leave temporary files around sys.dont_write_bytecode = True @@ -28,11 +29,10 @@ seen_imports = {} def global_type_add(name, obj): - '''Add new type to the dictionary of types ''' - type_name = 'vl_api_' + name + '_t' + """Add new type to the dictionary of types""" + type_name = "vl_api_" + name + "_t" if type_name in global_types: - raise KeyError("Attempted redefinition of {!r} with {!r}.".format( - name, obj)) + raise KeyError("Attempted redefinition of {!r} with {!r}.".format(name, obj)) global_types[type_name] = obj @@ -49,104 +49,105 @@ class VPPAPILexer: self.filename = filename reserved = { - 'service': 'SERVICE', - 'rpc': 'RPC', - 'returns': 'RETURNS', - 'null': 'NULL', - 'stream': 'STREAM', - 'events': 'EVENTS', - 'define': 'DEFINE', - 'typedef': 'TYPEDEF', - 'enum': 'ENUM', - 'enumflag': 'ENUMFLAG', - 'typeonly': 'TYPEONLY', - 'manual_print': 'MANUAL_PRINT', - 'manual_endian': 'MANUAL_ENDIAN', - 'dont_trace': 'DONT_TRACE', - 'autoreply': 'AUTOREPLY', - 'autoendian': 'AUTOENDIAN', - 'option': 'OPTION', - 'u8': 'U8', - 'u16': 'U16', - 'u32': 'U32', - 'u64': 'U64', - 'i8': 'I8', - 'i16': 'I16', - 'i32': 'I32', - 'i64': 'I64', - 'f64': 'F64', - 'bool': 'BOOL', - 'string': 'STRING', - 'import': 'IMPORT', - 'true': 'TRUE', - 'false': 'FALSE', - 'union': 'UNION', - 'counters': 'COUNTERS', - 'paths': 'PATHS', - 'units': 'UNITS', - 'severity': 'SEVERITY', - 'type': 'TYPE', - 'description': 'DESCRIPTION', + "service": "SERVICE", + "rpc": "RPC", + "returns": "RETURNS", + "null": "NULL", + "stream": "STREAM", + "events": "EVENTS", + "define": "DEFINE", + "typedef": "TYPEDEF", + "enum": "ENUM", + "enumflag": "ENUMFLAG", + "typeonly": "TYPEONLY", + "manual_print": "MANUAL_PRINT", + "manual_endian": "MANUAL_ENDIAN", + "dont_trace": "DONT_TRACE", + "autoreply": "AUTOREPLY", + "autoendian": "AUTOENDIAN", + "option": "OPTION", + "u8": "U8", + "u16": "U16", + "u32": "U32", + "u64": "U64", + "i8": "I8", + "i16": "I16", + "i32": "I32", + "i64": "I64", + "f64": "F64", + "bool": "BOOL", + "string": "STRING", + "import": "IMPORT", + "true": "TRUE", + "false": "FALSE", + "union": "UNION", + "counters": "COUNTERS", + "paths": "PATHS", + "units": "UNITS", + "severity": "SEVERITY", + "type": "TYPE", + "description": "DESCRIPTION", } - tokens = ['STRING_LITERAL', - 'ID', 'NUM'] + list(reserved.values()) + tokens = ["STRING_LITERAL", "COMMENT", "ID", "NUM"] + list(reserved.values()) - t_ignore_LINE_COMMENT = '//.*' + t_ignore_LINE_COMMENT = "//.*" def t_FALSE(self, t): - r'false' + r"false" t.value = False return t def t_TRUE(self, t): - r'false' + r"false" t.value = True return t def t_NUM(self, t): - r'0[xX][0-9a-fA-F]+|-?\d+\.?\d*' - base = 16 if t.value.startswith('0x') else 10 - if '.' in t.value: + r"0[xX][0-9a-fA-F]+|-?\d+\.?\d*" + base = 16 if t.value.startswith("0x") else 10 + if "." in t.value: t.value = float(t.value) else: t.value = int(t.value, base) return t def t_ID(self, t): - r'[a-zA-Z_][a-zA-Z_0-9]*' + r"[a-zA-Z_][a-zA-Z_0-9]*" # Check for reserved words - t.type = VPPAPILexer.reserved.get(t.value, 'ID') + t.type = VPPAPILexer.reserved.get(t.value, "ID") return t # C string def t_STRING_LITERAL(self, t): - r'\"([^\\\n]|(\\.))*?\"' - t.value = str(t.value).replace("\"", "") + r"\"([^\\\n]|(\\.))*?\" " + t.value = str(t.value).replace('"', "") return t # C or C++ comment (ignore) - def t_comment(self, t): - r'(/\*(.|\n)*?\*/)|(//.*)' - t.lexer.lineno += t.value.count('\n') + def t_COMMENT(self, t): + r"(/\*(.|\n)*?\*/)|(//.*)" + t.lexer.lineno += t.value.count("\n") + return t # Error handling rule def t_error(self, t): - raise ParseError("Illegal character '{}' ({})" - "in {}: line {}".format(t.value[0], - hex(ord(t.value[0])), - self.filename, - t.lexer.lineno)) + raise ParseError( + "Illegal character '{}' ({})" + "in {}: line {}".format( + t.value[0], hex(ord(t.value[0])), self.filename, t.lexer.lineno + ) + ) # Define a rule so we can track line numbers def t_newline(self, t): - r'\n+' + r"\n+" t.lexer.lineno += len(t.value) literals = ":{}[];=.," # A string containing ignored characters (spaces and tabs) - t_ignore = ' \t' + t_ignore = " \t" def vla_mark_length_field(block): @@ -164,23 +165,25 @@ def vla_is_last_check(name, block): vla = True if i + 1 < len(block): raise ValueError( - 'VLA field "{}" must be the last field in message "{}"' - .format(b.fieldname, name)) - elif b.fieldtype.startswith('vl_api_'): + 'VLA field "{}" must be the last field in message "{}"'.format( + b.fieldname, name + ) + ) + elif b.fieldtype.startswith("vl_api_"): if global_types[b.fieldtype].vla: vla = True if i + 1 < len(block): raise ValueError( 'VLA field "{}" must be the last ' - 'field in message "{}"' - .format(b.fieldname, name)) - elif b.fieldtype == 'string' and b.length == 0: + 'field in message "{}"'.format(b.fieldname, name) + ) + elif b.fieldtype == "string" and b.length == 0: vla = True if i + 1 < len(block): raise ValueError( 'VLA field "{}" must be the last ' - 'field in message "{}"' - .format(b.fieldname, name)) + 'field in message "{}"'.format(b.fieldname, name) + ) return vla @@ -192,10 +195,9 @@ class Processable: class Service(Processable): - type = 'Service' + type = "Service" - def __init__(self, caller, reply, events=None, stream_message=None, - stream=False): + def __init__(self, caller, reply, events=None, stream_message=None, stream=False): self.caller = caller self.reply = reply self.stream = stream @@ -204,7 +206,7 @@ class Service(Processable): class Typedef(Processable): - type = 'Typedef' + type = "Typedef" def __init__(self, name, flags, block): self.name = name @@ -214,9 +216,9 @@ class Typedef(Processable): self.manual_print = False self.manual_endian = False for f in flags: - if f == 'manual_print': + if f == "manual_print": self.manual_print = True - elif f == 'manual_endian': + elif f == "manual_endian": self.manual_endian = True global_type_add(name, self) @@ -224,14 +226,14 @@ class Typedef(Processable): vla_mark_length_field(self.block) def process(self, result): - result['types'].append(self) + result["types"].append(self) def __repr__(self): return self.name + str(self.flags) + str(self.block) class Using(Processable): - type = 'Using' + type = "Using" def __init__(self, name, flags, alias): self.name = name @@ -243,16 +245,15 @@ class Using(Processable): self.manual_print = False self.manual_endian = False for f in flags: - if f == 'manual_print': + if f == "manual_print": self.manual_print = True - elif f == 'manual_endian': + elif f == "manual_endian": self.manual_endian = True if isinstance(alias, Array): - a = {'type': alias.fieldtype, - 'length': alias.length} + a = {"type": alias.fieldtype, "length": alias.length} else: - a = {'type': alias.fieldtype} + a = {"type": alias.fieldtype} self.alias = a self.using = alias @@ -265,14 +266,14 @@ class Using(Processable): global_type_add(name, self) def process(self, result): # -> Dict - result['types'].append(self) + result["types"].append(self) def __repr__(self): return self.name + str(self.alias) class Union(Processable): - type = 'Union' + type = "Union" def __init__(self, name, flags, block): self.manual_print = False @@ -280,9 +281,9 @@ class Union(Processable): self.name = name for f in flags: - if f == 'manual_print': + if f == "manual_print": self.manual_print = True - elif f == 'manual_endian': + elif f == "manual_endian": self.manual_endian = True self.block = block @@ -292,16 +293,16 @@ class Union(Processable): global_type_add(name, self) def process(self, result): - result['types'].append(self) + result["types"].append(self) def __repr__(self): return str(self.block) class Define(Processable): - type = 'Define' + type = "Define" - def __init__(self, name, flags, block): + def __init__(self, name, flags, block, comment=None): self.name = name self.flags = flags self.block = block @@ -311,16 +312,17 @@ class Define(Processable): self.autoreply = False self.autoendian = 0 self.options = {} + self.comment = comment for f in flags: - if f == 'dont_trace': + if f == "dont_trace": self.dont_trace = True - elif f == 'manual_print': + elif f == "manual_print": self.manual_print = True - elif f == 'manual_endian': + elif f == "manual_endian": self.manual_endian = True - elif f == 'autoreply': + elif f == "autoreply": self.autoreply = True - elif f == 'autoendian': + elif f == "autoendian": self.autoendian = 1 remove = [] @@ -337,12 +339,11 @@ class Define(Processable): self.crc = str(block).encode() def autoreply_block(self, name, parent): - block = [Field('u32', 'context'), - Field('i32', 'retval')] + block = [Field("u32", "context"), Field("i32", "retval")] # inherit the parent's options for k, v in parent.options.items(): block.append(Option(k, v)) - return Define(name + '_reply', [], block) + return Define(name + "_reply", [], block) def process(self, result): # -> Dict tname = self.__class__.__name__ @@ -355,61 +356,64 @@ class Define(Processable): class Enum(Processable): - type = 'Enum' + type = "Enum" - def __init__(self, name, block, enumtype='u32'): + def __init__(self, name, block, enumtype="u32"): self.name = name self.enumtype = enumtype self.vla = False self.manual_print = False - count = 0 + count = -1 block2 = [] block3 = [] bc_set = False for b in block: - if 'value' in b: - count = b['value'] + if "value" in b: + count = b["value"] else: count += 1 - block2.append([b['id'], count]) + block2.append([b["id"], count]) try: - if b['option']['backwards_compatible']: + if b["option"]["backwards_compatible"]: pass bc_set = True except KeyError: - block3.append([b['id'], count]) + block3.append([b["id"], count]) if bc_set: - raise ValueError("Backward compatible enum must " - "be last {!r} {!r}" - .format(name, b['id'])) + raise ValueError( + "Backward compatible enum must " + "be last {!r} {!r}".format(name, b["id"]) + ) self.block = block2 self.crc = str(block3).encode() global_type_add(name, self) def process(self, result): - result['types'].append(self) + result["types"].append(self) def __repr__(self): return self.name + str(self.block) class EnumFlag(Enum): - type = 'EnumFlag' + type = "EnumFlag" - def __init__(self, name, block, enumtype='u32'): + def __init__(self, name, block, enumtype="u32"): super(EnumFlag, self).__init__(name, block, enumtype) for b in self.block: if bin(b[1])[2:].count("1") > 1: - raise TypeError("%s is not a flag enum. No element in a " - "flag enum may have more than a " - "single bit set." % self.name) + raise TypeError( + "%s is not a flag enum. No element in a " + "flag enum may have more than a " + "single bit set." % self.name + ) class Import(Processable): - type = 'Import' + type = "Import" _initialized = False def __new__(cls, *args, **kwargs): @@ -440,7 +444,7 @@ class Import(Processable): class Option(Processable): - type = 'Option' + type = "Option" def __init__(self, option, value=None): self.option = option @@ -458,7 +462,7 @@ class Option(Processable): class Array(Processable): - type = 'Array' + type = "Array" def __init__(self, fieldtype, name, length, modern_vla=False): self.fieldtype = fieldtype @@ -474,12 +478,11 @@ class Array(Processable): self.vla = False def __repr__(self): - return str([self.fieldtype, self.fieldname, self.length, - self.lengthfield]) + return str([self.fieldtype, self.fieldname, self.length, self.lengthfield]) class Field(Processable): - type = 'Field' + type = "Field" def __init__(self, fieldtype, name, limit=None): # limit field has been expanded to an options dict. @@ -487,13 +490,14 @@ class Field(Processable): self.fieldtype = fieldtype self.is_lengthfield = False - if self.fieldtype == 'string': - raise ValueError("The string type {!r} is an " - "array type ".format(name)) + if self.fieldtype == "string": + raise ValueError("The string type {!r} is an " "array type ".format(name)) if name in keyword.kwlist: - raise ValueError("Fieldname {!r} is a python keyword and is not " - "accessible via the python API. ".format(name)) + raise ValueError( + "Fieldname {!r} is a python keyword and is not " + "accessible via the python API. ".format(name) + ) self.fieldname = name self.limit = limit @@ -502,35 +506,34 @@ class Field(Processable): class Counter(Processable): - type = 'Counter' + type = "Counter" def __init__(self, path, counter): self.name = path self.block = counter def process(self, result): # -> Dict - result['Counters'].append(self) + result["Counters"].append(self) class Paths(Processable): - type = 'Paths' + type = "Paths" def __init__(self, pathset): self.paths = pathset def __repr__(self): - return "%s(paths=%s)" % ( - self.__class__.__name__, self.paths - ) + return "%s(paths=%s)" % (self.__class__.__name__, self.paths) class Coord: - """ Coordinates of a syntactic element. Consists of: - - File name - - Line number - - (optional) column number, for the Lexer + """Coordinates of a syntactic element. Consists of: + - File name + - Line number + - (optional) column number, for the Lexer """ - __slots__ = ('file', 'line', 'column', '__weakref__') + + __slots__ = ("file", "line", "column", "__weakref__") def __init__(self, file, line, column=None): self.file = file @@ -559,6 +562,7 @@ class VPPAPIParser: self.logger = logger self.fields = [] self.revision = revision + self.last_comment = None def _parse_error(self, msg, coord): raise ParseError("%s: %s" % (coord, msg)) @@ -568,49 +572,48 @@ class VPPAPIParser: self.logger.warning("%s: %s" % (coord, msg)) def _coord(self, lineno, column=None): - return Coord( - file=self.filename, - line=lineno, column=column) + return Coord(file=self.filename, line=lineno, column=column) def _token_coord(self, p, token_idx): - """ Returns the coordinates for the YaccProduction object 'p' indexed - with 'token_idx'. The coordinate includes the 'lineno' and - 'column'. Both follow the lex semantic, starting from 1. + """Returns the coordinates for the YaccProduction object 'p' indexed + with 'token_idx'. The coordinate includes the 'lineno' and + 'column'. Both follow the lex semantic, starting from 1. """ - last_cr = p.lexer.lexdata.rfind('\n', 0, p.lexpos(token_idx)) + last_cr = p.lexer.lexdata.rfind("\n", 0, p.lexpos(token_idx)) if last_cr < 0: last_cr = -1 - column = (p.lexpos(token_idx) - (last_cr)) + column = p.lexpos(token_idx) - (last_cr) return self._coord(p.lineno(token_idx), column) def p_slist(self, p): - '''slist : stmt - | slist stmt''' + """slist : stmt + | slist stmt""" if len(p) == 2: p[0] = [p[1]] else: p[0] = p[1] + [p[2]] def p_stmt(self, p): - '''stmt : define - | typedef - | option - | import - | enum - | enumflag - | union - | service - | paths - | counters''' + """stmt : define + | typedef + | option + | import + | enum + | enumflag + | union + | service + | paths + | comment + | counters""" p[0] = p[1] def p_import(self, p): - '''import : IMPORT STRING_LITERAL ';' ''' + """import : IMPORT STRING_LITERAL ';'""" p[0] = Import(p[2], revision=self.revision) def p_path_elements(self, p): - '''path_elements : path_element - | path_elements path_element''' + """path_elements : path_element + | path_elements path_element""" if len(p) == 2: p[0] = p[1] else: @@ -620,20 +623,20 @@ class VPPAPIParser: p[0] = p[1] + [p[2]] def p_path_element(self, p): - '''path_element : STRING_LITERAL STRING_LITERAL ';' ''' - p[0] = {'path': p[1], 'counter': p[2]} + """path_element : STRING_LITERAL STRING_LITERAL ';'""" + p[0] = {"path": p[1], "counter": p[2]} def p_paths(self, p): - '''paths : PATHS '{' path_elements '}' ';' ''' + """paths : PATHS '{' path_elements '}' ';'""" p[0] = Paths(p[3]) def p_counters(self, p): - '''counters : COUNTERS ID '{' counter_elements '}' ';' ''' + """counters : COUNTERS ID '{' counter_elements '}' ';'""" p[0] = Counter(p[2], p[4]) def p_counter_elements(self, p): - '''counter_elements : counter_element - | counter_elements counter_element''' + """counter_elements : counter_element + | counter_elements counter_element""" if len(p) == 2: p[0] = [p[1]] else: @@ -643,46 +646,47 @@ class VPPAPIParser: p[0] = p[1] + [p[2]] def p_counter_element(self, p): - '''counter_element : ID '{' counter_statements '}' ';' ''' - p[0] = {**{'name': p[1]}, **p[3]} + """counter_element : ID '{' counter_statements '}' ';'""" + p[0] = {**{"name": p[1]}, **p[3]} def p_counter_statements(self, p): - '''counter_statements : counter_statement - | counter_statements counter_statement''' + """counter_statements : counter_statement + | counter_statements counter_statement""" if len(p) == 2: p[0] = p[1] else: p[0] = {**p[1], **p[2]} def p_counter_statement(self, p): - '''counter_statement : SEVERITY ID ';' - | UNITS STRING_LITERAL ';' - | DESCRIPTION STRING_LITERAL ';' - | TYPE ID ';' ''' + """counter_statement : SEVERITY ID ';' + | UNITS STRING_LITERAL ';' + | DESCRIPTION STRING_LITERAL ';' + | TYPE ID ';'""" p[0] = {p[1]: p[2]} def p_service(self, p): - '''service : SERVICE '{' service_statements '}' ';' ''' + """service : SERVICE '{' service_statements '}' ';'""" p[0] = p[3] def p_service_statements(self, p): - '''service_statements : service_statement - | service_statements service_statement''' + """service_statements : service_statement + | service_statements service_statement""" if len(p) == 2: p[0] = [p[1]] else: p[0] = p[1] + [p[2]] def p_service_statement(self, p): - '''service_statement : RPC ID RETURNS NULL ';' - | RPC ID RETURNS ID ';' - | RPC ID RETURNS STREAM ID ';' - | RPC ID RETURNS ID EVENTS event_list ';' ''' + """service_statement : RPC ID RETURNS NULL ';' + | RPC ID RETURNS ID ';' + | RPC ID RETURNS STREAM ID ';' + | RPC ID RETURNS ID EVENTS event_list ';'""" if p[2] == p[4]: # Verify that caller and reply differ self._parse_error( - 'Reply ID ({}) should not be equal to Caller ID'.format(p[2]), - self._token_coord(p, 1)) + "Reply ID ({}) should not be equal to Caller ID".format(p[2]), + self._token_coord(p, 1), + ) if len(p) == 8: p[0] = Service(p[2], p[4], p[6]) elif len(p) == 7: @@ -691,287 +695,300 @@ class VPPAPIParser: p[0] = Service(p[2], p[4]) def p_service_statement2(self, p): - '''service_statement : RPC ID RETURNS ID STREAM ID ';' ''' + """service_statement : RPC ID RETURNS ID STREAM ID ';'""" p[0] = Service(p[2], p[4], stream_message=p[6], stream=True) def p_event_list(self, p): - '''event_list : events - | event_list events ''' + """event_list : events + | event_list events""" if len(p) == 2: p[0] = [p[1]] else: p[0] = p[1] + [p[2]] def p_event(self, p): - '''events : ID - | ID ',' ''' + """events : ID + | ID ','""" p[0] = p[1] def p_enum(self, p): - '''enum : ENUM ID '{' enum_statements '}' ';' ''' + """enum : ENUM ID '{' enum_statements '}' ';'""" p[0] = Enum(p[2], p[4]) def p_enum_type(self, p): - ''' enum : ENUM ID ':' enum_size '{' enum_statements '}' ';' ''' + """enum : ENUM ID ':' enum_size '{' enum_statements '}' ';'""" if len(p) == 9: p[0] = Enum(p[2], p[6], enumtype=p[4]) else: p[0] = Enum(p[2], p[4]) def p_enumflag(self, p): - '''enumflag : ENUMFLAG ID '{' enum_statements '}' ';' ''' + """enumflag : ENUMFLAG ID '{' enum_statements '}' ';'""" p[0] = EnumFlag(p[2], p[4]) def p_enumflag_type(self, p): - ''' enumflag : ENUMFLAG ID ':' enumflag_size '{' enum_statements '}' ';' ''' # noqa : E502 + """enumflag : ENUMFLAG ID ':' enumflag_size '{' enum_statements '}' ';'""" # noqa : E502 if len(p) == 9: p[0] = EnumFlag(p[2], p[6], enumtype=p[4]) else: p[0] = EnumFlag(p[2], p[4]) def p_enum_size(self, p): - ''' enum_size : U8 - | U16 - | U32 - | I8 - | I16 - | I32 ''' + """enum_size : U8 + | U16 + | U32 + | I8 + | I16 + | I32""" p[0] = p[1] def p_enumflag_size(self, p): - ''' enumflag_size : U8 - | U16 - | U32 ''' + """enumflag_size : U8 + | U16 + | U32""" p[0] = p[1] def p_define(self, p): - '''define : DEFINE ID '{' block_statements_opt '}' ';' ''' + """define : DEFINE ID '{' block_statements_opt '}' ';'""" self.fields = [] - p[0] = Define(p[2], [], p[4]) + p[0] = Define(p[2], [], p[4], self.last_comment) + self.last_comment = None def p_define_flist(self, p): - '''define : flist DEFINE ID '{' block_statements_opt '}' ';' ''' + """define : flist DEFINE ID '{' block_statements_opt '}' ';'""" # Legacy typedef - if 'typeonly' in p[1]: - self._parse_error('legacy typedef. use typedef: {} {}[{}];' - .format(p[1], p[2], p[4]), - self._token_coord(p, 1)) + if "typeonly" in p[1]: + self._parse_error( + "legacy typedef. use typedef: {} {}[{}];".format(p[1], p[2], p[4]), + self._token_coord(p, 1), + ) else: - p[0] = Define(p[3], p[1], p[5]) + p[0] = Define(p[3], p[1], p[5], self.last_comment) + self.last_comment = None def p_flist(self, p): - '''flist : flag - | flist flag''' + """flist : flag + | flist flag""" if len(p) == 2: p[0] = [p[1]] else: p[0] = p[1] + [p[2]] def p_flag(self, p): - '''flag : MANUAL_PRINT - | MANUAL_ENDIAN - | DONT_TRACE - | TYPEONLY - | AUTOENDIAN - | AUTOREPLY''' + """flag : MANUAL_PRINT + | MANUAL_ENDIAN + | DONT_TRACE + | TYPEONLY + | AUTOENDIAN + | AUTOREPLY""" if len(p) == 1: return p[0] = p[1] def p_typedef(self, p): - '''typedef : TYPEDEF ID '{' block_statements_opt '}' ';' ''' + """typedef : TYPEDEF ID '{' block_statements_opt '}' ';'""" p[0] = Typedef(p[2], [], p[4]) def p_typedef_flist(self, p): - '''typedef : flist TYPEDEF ID '{' block_statements_opt '}' ';' ''' + """typedef : flist TYPEDEF ID '{' block_statements_opt '}' ';'""" p[0] = Typedef(p[3], p[1], p[5]) def p_typedef_alias(self, p): - '''typedef : TYPEDEF declaration ''' + """typedef : TYPEDEF declaration""" p[0] = Using(p[2].fieldname, [], p[2]) def p_typedef_alias_flist(self, p): - '''typedef : flist TYPEDEF declaration ''' + """typedef : flist TYPEDEF declaration""" p[0] = Using(p[3].fieldname, p[1], p[3]) def p_block_statements_opt(self, p): - '''block_statements_opt : block_statements ''' + """block_statements_opt : block_statements""" p[0] = p[1] def p_block_statements(self, p): - '''block_statements : block_statement - | block_statements block_statement''' + """block_statements : block_statement + | block_statements block_statement""" if len(p) == 2: p[0] = [p[1]] else: p[0] = p[1] + [p[2]] def p_block_statement(self, p): - '''block_statement : declaration - | option ''' + """block_statement : declaration + | option""" p[0] = p[1] def p_enum_statements(self, p): - '''enum_statements : enum_statement - | enum_statements enum_statement''' + """enum_statements : enum_statement + | enum_statements enum_statement""" if len(p) == 2: p[0] = [p[1]] else: p[0] = p[1] + [p[2]] def p_enum_statement(self, p): - '''enum_statement : ID '=' NUM ',' - | ID ',' - | ID '[' field_options ']' ',' - | ID '=' NUM '[' field_options ']' ',' ''' + """enum_statement : ID '=' NUM ',' + | ID ',' + | ID '[' field_options ']' ',' + | ID '=' NUM '[' field_options ']' ','""" if len(p) == 3: - p[0] = {'id': p[1]} + p[0] = {"id": p[1]} elif len(p) == 5: - p[0] = {'id': p[1], 'value': p[3]} + p[0] = {"id": p[1], "value": p[3]} elif len(p) == 6: - p[0] = {'id': p[1], 'option': p[3]} + p[0] = {"id": p[1], "option": p[3]} elif len(p) == 8: - p[0] = {'id': p[1], 'value': p[3], 'option': p[5]} + p[0] = {"id": p[1], "value": p[3], "option": p[5]} else: - self._parse_error('ERROR', self._token_coord(p, 1)) + self._parse_error("ERROR", self._token_coord(p, 1)) def p_field_options(self, p): - '''field_options : field_option - | field_options field_option''' + """field_options : field_option + | field_options field_option""" if len(p) == 2: p[0] = p[1] else: p[0] = {**p[1], **p[2]} def p_field_option(self, p): - '''field_option : ID - | ID '=' assignee ',' - | ID '=' assignee + """field_option : ID + | ID '=' assignee ',' + | ID '=' assignee - ''' + """ if len(p) == 2: p[0] = {p[1]: None} else: p[0] = {p[1]: p[3]} def p_variable_name(self, p): - '''variable_name : ID - | TYPE - | SEVERITY - | DESCRIPTION - | COUNTERS - | PATHS - ''' + """variable_name : ID + | TYPE + | SEVERITY + | DESCRIPTION + | COUNTERS + | PATHS + """ p[0] = p[1] + def p_comment(self, p): + """comment : COMMENT""" + self.last_comment = p[1] + p[0] = [] + def p_declaration(self, p): - '''declaration : type_specifier variable_name ';' - | type_specifier variable_name '[' field_options ']' ';' - ''' + """declaration : type_specifier variable_name ';' + | type_specifier variable_name '[' field_options ']' ';' + """ if len(p) == 7: p[0] = Field(p[1], p[2], p[4]) elif len(p) == 4: p[0] = Field(p[1], p[2]) else: - self._parse_error('ERROR', self._token_coord(p, 1)) + self._parse_error("ERROR", self._token_coord(p, 1)) self.fields.append(p[2]) def p_declaration_array_vla(self, p): - '''declaration : type_specifier variable_name '[' ']' ';' ''' + """declaration : type_specifier variable_name '[' ']' ';'""" p[0] = Array(p[1], p[2], 0, modern_vla=True) def p_declaration_array(self, p): - '''declaration : type_specifier variable_name '[' NUM ']' ';' - | type_specifier variable_name '[' ID ']' ';' ''' + """declaration : type_specifier variable_name '[' NUM ']' ';' + | type_specifier variable_name '[' ID ']' ';'""" if len(p) != 7: return self._parse_error( - 'array: %s' % p.value, - self._coord(lineno=p.lineno)) + "array: %s" % p.value, self._coord(lineno=p.lineno) + ) # Make this error later if type(p[4]) is int and p[4] == 0: # XXX: Line number is wrong - self._parse_warning('Old Style VLA: {} {}[{}];' - .format(p[1], p[2], p[4]), - self._token_coord(p, 1)) + self._parse_warning( + "Old Style VLA: {} {}[{}];".format(p[1], p[2], p[4]), + self._token_coord(p, 1), + ) if type(p[4]) is str and p[4] not in self.fields: # Verify that length field exists - self._parse_error('Missing length field: {} {}[{}];' - .format(p[1], p[2], p[4]), - self._token_coord(p, 1)) + self._parse_error( + "Missing length field: {} {}[{}];".format(p[1], p[2], p[4]), + self._token_coord(p, 1), + ) p[0] = Array(p[1], p[2], p[4]) def p_option(self, p): - '''option : OPTION ID '=' assignee ';' - | OPTION ID ';' ''' + """option : OPTION ID '=' assignee ';' + | OPTION ID ';'""" if len(p) == 4: p[0] = Option(p[2]) else: p[0] = Option(p[2], p[4]) def p_assignee(self, p): - '''assignee : NUM - | TRUE - | FALSE - | STRING_LITERAL ''' + """assignee : NUM + | TRUE + | FALSE + | STRING_LITERAL""" p[0] = p[1] def p_type_specifier(self, p): - '''type_specifier : U8 - | U16 - | U32 - | U64 - | I8 - | I16 - | I32 - | I64 - | F64 - | BOOL - | STRING''' + """type_specifier : U8 + | U16 + | U32 + | U64 + | I8 + | I16 + | I32 + | I64 + | F64 + | BOOL + | STRING""" p[0] = p[1] # Do a second pass later to verify that user defined types are defined def p_typedef_specifier(self, p): - '''type_specifier : ID ''' + """type_specifier : ID""" if p[1] not in global_types: - self._parse_error('Undefined type: {}'.format(p[1]), - self._token_coord(p, 1)) + self._parse_error( + "Undefined type: {}".format(p[1]), self._token_coord(p, 1) + ) p[0] = p[1] def p_union(self, p): - '''union : UNION ID '{' block_statements_opt '}' ';' ''' + """union : UNION ID '{' block_statements_opt '}' ';'""" p[0] = Union(p[2], [], p[4]) def p_union_flist(self, p): - '''union : flist UNION ID '{' block_statements_opt '}' ';' ''' + """union : flist UNION ID '{' block_statements_opt '}' ';'""" p[0] = Union(p[3], p[1], p[5]) # Error rule for syntax errors def p_error(self, p): if p: - self._parse_error( - 'before: %s' % p.value, - self._coord(lineno=p.lineno)) + if p.type == "COMMENT": + self.parser.errok() + return + self._parse_error("before: %s" % p.value, self._coord(lineno=p.lineno)) else: - self._parse_error('At end of input', self.filename) + self._parse_error("At end of input", self.filename) + def build(self, **kwargs): + self.parser = yacc.yacc(module=self, **kwargs) -class VPPAPI(): - def __init__(self, debug=False, filename='', logger=None, revision=None): +class VPPAPI: + def __init__(self, debug=False, filename="", logger=None, revision=None): self.lexer = lex.lex(module=VPPAPILexer(filename), debug=debug) - self.parser = yacc.yacc(module=VPPAPIParser(filename, logger, - revision=revision), - write_tables=False, debug=debug) + self.parser = VPPAPIParser(filename, logger, revision=revision) + self.parser.build(write_tables=False, debug=debug) self.logger = logger self.revision = revision self.filename = filename def parse_string(self, code, debug=0, lineno=1): self.lexer.lineno = lineno - return self.parser.parse(code, lexer=self.lexer, debug=debug) + return self.parser.parser.parse(code, lexer=self.lexer, debug=debug) def parse_fd(self, fd, debug=0): data = fd.read() @@ -979,38 +996,40 @@ class VPPAPI(): def parse_filename(self, filename, debug=0): if self.revision: - git_show = 'git show {}:{}'.format(self.revision, filename) - proc = Popen(git_show.split(), stdout=PIPE, encoding='utf-8') + git_show = "git show {}:{}".format(self.revision, filename) + proc = Popen(git_show.split(), stdout=PIPE, encoding="utf-8") try: data, errs = proc.communicate() if proc.returncode != 0: - print('File not found: {}:{}' - .format(self.revision, filename), file=sys.stderr) + print( + "File not found: {}:{}".format(self.revision, filename), + file=sys.stderr, + ) sys.exit(2) return self.parse_string(data, debug=debug) except Exception: sys.exit(3) else: try: - with open(filename, encoding='utf-8') as fd: + with open(filename, encoding="utf-8") as fd: return self.parse_fd(fd, None) except FileNotFoundError: - print('File not found: {}'.format(filename), file=sys.stderr) + print("File not found: {}".format(filename), file=sys.stderr) sys.exit(2) def process(self, objs): s = {} - s['Option'] = {} - s['Define'] = [] - s['Service'] = [] - s['types'] = [] - s['Import'] = [] - s['Counters'] = [] - s['Paths'] = [] + s["Option"] = {} + s["Define"] = [] + s["Service"] = [] + s["types"] = [] + s["Import"] = [] + s["Counters"] = [] + s["Paths"] = [] crc = 0 for o in objs: try: - crc = binascii.crc32(o.crc, crc) & 0xffffffff + crc = binascii.crc32(o.crc, crc) & 0xFFFFFFFF except AttributeError: pass @@ -1021,82 +1040,84 @@ class VPPAPI(): else: o.process(s) - msgs = {d.name: d for d in s['Define']} - svcs = {s.caller: s for s in s['Service']} - replies = {s.reply: s for s in s['Service']} + msgs = {d.name: d for d in s["Define"]} + svcs = {s.caller: s for s in s["Service"]} + replies = {s.reply: s for s in s["Service"]} seen_services = {} - s['file_crc'] = crc + s["file_crc"] = crc for service in svcs: if service not in msgs: raise ValueError( - 'Service definition refers to unknown message' - ' definition: {}'.format(service)) - if svcs[service].reply != 'null' and \ - svcs[service].reply not in msgs: - raise ValueError('Service definition refers to unknown message' - ' definition in reply: {}' - .format(svcs[service].reply)) + "Service definition refers to unknown message" + " definition: {}".format(service) + ) + if svcs[service].reply != "null" and svcs[service].reply not in msgs: + raise ValueError( + "Service definition refers to unknown message" + " definition in reply: {}".format(svcs[service].reply) + ) if service in replies: - raise ValueError('Service definition refers to message' - ' marked as reply: {}'.format(service)) + raise ValueError( + "Service definition refers to message" + " marked as reply: {}".format(service) + ) for event in svcs[service].events: if event not in msgs: - raise ValueError('Service definition refers to unknown ' - 'event: {} in message: {}' - .format(event, service)) + raise ValueError( + "Service definition refers to unknown " + "event: {} in message: {}".format(event, service) + ) seen_services[event] = True # Create services implicitly for d in msgs: if d in seen_services: continue - if d.endswith('_reply'): + if d.endswith("_reply"): if d[:-6] in svcs: continue if d[:-6] not in msgs: - raise ValueError('{} missing calling message' - .format(d)) + raise ValueError("{} missing calling message".format(d)) continue - if d.endswith('_dump'): + if d.endswith("_dump"): if d in svcs: continue - if d[:-5]+'_details' in msgs: - s['Service'].append(Service(d, d[:-5]+'_details', - stream=True)) + if d[:-5] + "_details" in msgs: + s["Service"].append(Service(d, d[:-5] + "_details", stream=True)) else: - raise ValueError('{} missing details message' - .format(d)) + raise ValueError("{} missing details message".format(d)) continue - if d.endswith('_details'): - if d[:-8]+'_get' in msgs: - if d[:-8]+'_get' in svcs: + if d.endswith("_details"): + if d[:-8] + "_get" in msgs: + if d[:-8] + "_get" in svcs: continue - raise ValueError('{} should be in a stream service' - .format(d[:-8]+'_get')) - if d[:-8]+'_dump' in msgs: + raise ValueError( + "{} should be in a stream service".format(d[:-8] + "_get") + ) + if d[:-8] + "_dump" in msgs: continue - raise ValueError('{} missing dump or get message' - .format(d)) + raise ValueError("{} missing dump or get message".format(d)) if d in svcs: continue - if d+'_reply' in msgs: - s['Service'].append(Service(d, d+'_reply')) + if d + "_reply" in msgs: + s["Service"].append(Service(d, d + "_reply")) else: raise ValueError( - '{} missing reply message ({}) or service definition' - .format(d, d+'_reply')) + "{} missing reply message ({}) or service definition".format( + d, d + "_reply" + ) + ) return s def process_imports(self, objs, in_import, result): # -> List for o in objs: # Only allow the following object types from imported file - if in_import and not isinstance(o, (Enum, Import, Typedef, - Union, Using)): + if in_import and not isinstance(o, (Enum, Import, Typedef, Union, Using)): continue if isinstance(o, Import): result.append(o) @@ -1109,7 +1130,7 @@ class VPPAPI(): # Add message ids to each message. def add_msg_id(s): for o in s: - o.block.insert(0, Field('u16', '_vl_msg_id')) + o.block.insert(0, Field("u16", "_vl_msg_id")) return s @@ -1129,11 +1150,11 @@ def dirlist_get(): def foldup_blocks(block, crc): for b in block: # Look up CRC in user defined types - if b.fieldtype.startswith('vl_api_'): + if b.fieldtype.startswith("vl_api_"): # Recursively t = global_types[b.fieldtype] try: - crc = binascii.crc32(t.crc, crc) & 0xffffffff + crc = binascii.crc32(t.crc, crc) & 0xFFFFFFFF crc = foldup_blocks(t.block, crc) except AttributeError: pass @@ -1142,49 +1163,38 @@ def foldup_blocks(block, crc): def foldup_crcs(s): for f in s: - f.crc = foldup_blocks(f.block, - binascii.crc32(f.crc) & 0xffffffff) - - -# -# Main -# -def main(): - if sys.version_info < (3, 5,): - log.exception('vppapigen requires a supported version of python. ' - 'Please use version 3.5 or greater. ' - 'Using %s', sys.version) - return 1 - - cliparser = argparse.ArgumentParser(description='VPP API generator') - cliparser.add_argument('--pluginpath', default="") - cliparser.add_argument('--includedir', action='append') - cliparser.add_argument('--outputdir', action='store') - cliparser.add_argument('--input') - cliparser.add_argument('--output', nargs='?', - type=argparse.FileType('w', encoding='UTF-8'), - default=sys.stdout) - - cliparser.add_argument('output_module', nargs='?', default='C') - cliparser.add_argument('--debug', action='store_true') - cliparser.add_argument('--show-name', nargs=1) - cliparser.add_argument('--git-revision', - help="Git revision to use for opening files") - args = cliparser.parse_args() - - dirlist_add(args.includedir) - if not args.debug: + f.crc = foldup_blocks(f.block, binascii.crc32(f.crc) & 0xFFFFFFFF) + + +def run_vppapigen( + input_file=None, + output=sys.stdout, + includedir=None, + debug=False, + show_name=None, + output_module="C", + outputdir=None, + pluginpath="", + git_revision=None, +): + # reset globals + dirlist.clear() + global_types.clear() + seen_imports.clear() + + dirlist_add(includedir) + if not debug: sys.excepthook = exception_handler # Filename - if args.show_name: - filename = args.show_name[0] - elif args.input: - filename = args.input + if show_name: + filename = show_name[0] + elif input_file: + filename = input_file else: - filename = '' + filename = "" - if args.debug: + if debug: logging.basicConfig(stream=sys.stdout, level=logging.WARNING) else: logging.basicConfig() @@ -1195,44 +1205,38 @@ def main(): from importlib.machinery import SourceFileLoader # Default path - pluginpath = '' - if not args.pluginpath: + pluginpath = "" + if not pluginpath: cand = [] cand.append(os.path.dirname(os.path.realpath(__file__))) - cand.append(os.path.dirname(os.path.realpath(__file__)) + - '/../share/vpp/') + cand.append(os.path.dirname(os.path.realpath(__file__)) + "/../share/vpp/") for c in cand: - c += '/' - if os.path.isfile('{}vppapigen_{}.py' - .format(c, args.output_module.lower())): + c += "/" + if os.path.isfile("{}vppapigen_{}.py".format(c, output_module.lower())): pluginpath = c break else: - pluginpath = args.pluginpath + '/' - if pluginpath == '': - log.exception('Output plugin not found') + pluginpath = pluginpath + "/" + if pluginpath == "": + log.exception("Output plugin not found") return 1 - module_path = '{}vppapigen_{}.py'.format(pluginpath, - args.output_module.lower()) + module_path = "{}vppapigen_{}.py".format(pluginpath, output_module.lower()) try: - plugin = SourceFileLoader(args.output_module, - module_path).load_module() + plugin = SourceFileLoader(output_module, module_path).load_module() except Exception as err: - log.exception('Error importing output plugin: %s, %s', - module_path, err) + log.exception("Error importing output plugin: %s, %s", module_path, err) return 1 - parser = VPPAPI(debug=args.debug, filename=filename, logger=log, - revision=args.git_revision) + parser = VPPAPI(debug=debug, filename=filename, logger=log, revision=git_revision) try: - if not args.input: + if not input_file: parsed_objects = parser.parse_fd(sys.stdin, log) else: - parsed_objects = parser.parse_filename(args.input, log) + parsed_objects = parser.parse_filename(input_file, log) except ParseError as e: - print('Parse error: ', e, file=sys.stderr) + print("Parse error: ", e, file=sys.stderr) sys.exit(1) # Build a list of objects. Hash of lists. @@ -1250,32 +1254,90 @@ def main(): else: s = parser.process(parsed_objects) imports = parser.process_imports(parsed_objects, False, result) - s['imported'] = parser.process(imports) + s["imported"] = parser.process(imports) # Add msg_id field - s['Define'] = add_msg_id(s['Define']) + s["Define"] = add_msg_id(s["Define"]) # Fold up CRCs - foldup_crcs(s['Define']) + foldup_crcs(s["Define"]) # # Debug - if args.debug: + if debug: import pprint + pp = pprint.PrettyPrinter(indent=4, stream=sys.stderr) - for t in s['Define']: + for t in s["Define"]: pp.pprint([t.name, t.flags, t.block]) - for t in s['types']: + for t in s["types"]: pp.pprint([t.name, t.block]) - result = plugin.run(args, filename, s) + result = plugin.run(outputdir, filename, s) if result: - print(result, file=args.output) + if isinstance(output, str): + with open(output, "w", encoding="UTF-8") as f: + print(result, file=f) + else: + print(result, file=output) else: - log.exception('Running plugin failed: %s %s', filename, result) + log.exception("Running plugin failed: %s %s", filename, result) return 1 return 0 -if __name__ == '__main__': +def run_kw_vppapigen(kwargs): + return run_vppapigen(**kwargs) + + +# +# Main +# +def main(): + if sys.version_info < ( + 3, + 5, + ): + log.exception( + "vppapigen requires a supported version of python. " + "Please use version 3.5 or greater. " + "Using %s", + sys.version, + ) + return 1 + + cliparser = argparse.ArgumentParser(description="VPP API generator") + cliparser.add_argument("--pluginpath", default="") + cliparser.add_argument("--includedir", action="append") + cliparser.add_argument("--outputdir", action="store") + cliparser.add_argument("--input") + cliparser.add_argument( + "--output", + nargs="?", + type=argparse.FileType("w", encoding="UTF-8"), + default=sys.stdout, + ) + + cliparser.add_argument("output_module", nargs="?", default="C") + cliparser.add_argument("--debug", action="store_true") + cliparser.add_argument("--show-name", nargs=1) + cliparser.add_argument( + "--git-revision", help="Git revision to use for opening files" + ) + args = cliparser.parse_args() + + return run_vppapigen( + includedir=args.includedir, + debug=args.debug, + outputdir=args.outputdir, + show_name=args.show_name, + input_file=args.input, + output_module=args.output_module, + pluginpath=args.pluginpath, + git_revision=args.git_revision, + output=args.output, + ) + + +if __name__ == "__main__": sys.exit(main()) diff --git a/src/tools/vppapigen/vppapigen_c.py b/src/tools/vppapigen/vppapigen_c.py index 8564ade1080..c2e1e7da7b7 100644..100755 --- a/src/tools/vppapigen/vppapigen_c.py +++ b/src/tools/vppapigen/vppapigen_c.py @@ -18,12 +18,13 @@ # binary API format # -''' +""" This module creates C code for core VPP, VPP plugins and client side VAT and VAT2 tests. -''' +""" import datetime +import itertools import os import time import sys @@ -34,95 +35,102 @@ process_imports = False ############################################################################### -class ToJSON(): - '''Class to generate functions converting from VPP binary API to JSON.''' +class ToJSON: + """Class to generate functions converting from VPP binary API to JSON.""" + _dispatch = {} - noprint_fields = {'_vl_msg_id': None, - 'client_index': None, - 'context': None} - is_number = {'u8': None, - 'i8': None, - 'u16': None, - 'i16': None, - 'u32': None, - 'i32': None, - 'u64': None, - 'i64': None, - 'f64': None, - } + noprint_fields = {"_vl_msg_id": None, "client_index": None, "context": None} + is_number = { + "u8": None, + "i8": None, + "u16": None, + "i16": None, + "u32": None, + "i32": None, + "u64": None, + "i64": None, + "f64": None, + } def __init__(self, module, types, defines, imported_types, stream): self.stream = stream self.module = module self.defines = defines self.types = types - self.types_hash = {'vl_api_'+d.name+'_t': - d for d in types + imported_types} + self.types_hash = {"vl_api_" + d.name + "_t": d for d in types + imported_types} self.defines_hash = {d.name: d for d in defines} def header(self): - '''Output the top boilerplate.''' + """Output the top boilerplate.""" write = self.stream.write - write('#ifndef included_{}_api_tojson_h\n'.format(self.module)) - write('#define included_{}_api_tojson_h\n'.format(self.module)) - write('#include <vppinfra/cJSON.h>\n\n') - write('#include <vat2/jsonconvert.h>\n\n') + write("#ifndef included_{}_api_tojson_h\n".format(self.module)) + write("#define included_{}_api_tojson_h\n".format(self.module)) + write("#include <vppinfra/cJSON.h>\n\n") + write("#include <vppinfra/jsonformat.h>\n\n") + if self.module == "interface_types": + write("#define vl_printfun\n") + write("#include <vnet/interface_types.api.h>\n\n") def footer(self): - '''Output the bottom boilerplate.''' + """Output the bottom boilerplate.""" write = self.stream.write - write('#endif\n') + write("#endif\n") def get_base_type(self, t): vt_type = None try: vt = self.types_hash[t] - if vt.type == 'Using' and 'length' not in vt.alias: - vt_type = vt.alias['type'] + if vt.type == "Using" and "length" not in vt.alias: + vt_type = vt.alias["type"] except KeyError: vt = t return vt, vt_type def get_json_func(self, t): - '''Given the type, returns the function to use to create a - cJSON object''' + """Given the type, returns the function to use to create a + cJSON object""" vt, vt_type = self.get_base_type(t) if t in self.is_number or vt_type in self.is_number: - return 'cJSON_AddNumberToObject', '', False - if t == 'bool': - return 'cJSON_AddBoolToObject', '', False + return "cJSON_AddNumberToObject", "", False + if t == "bool": + return "cJSON_AddBoolToObject", "", False # Lookup type name check if it's enum - if vt.type == 'Enum' or vt.type == 'EnumFlag': - return '{t}_tojson'.format(t=t), '', True - return '{t}_tojson'.format(t=t), '&', True + if vt.type == "Enum" or vt.type == "EnumFlag": + return "{t}_tojson".format(t=t), "", True + return "{t}_tojson".format(t=t), "&", True def get_json_array_func(self, t): - '''Given a type returns the function to create a cJSON object - for arrays.''' + """Given a type returns the function to create a cJSON object + for arrays.""" if t in self.is_number: - return 'cJSON_CreateNumber', '' - if t == 'bool': - return 'cJSON_CreateBool', '' + return "cJSON_CreateNumber", "" + if t == "bool": + return "cJSON_CreateBool", "" vt, vt_type = self.get_base_type(t) - if vt.type == 'Enum' or vt.type == 'EnumFlag': - return '{t}_tojson'.format(t=t), '' - return '{t}_tojson'.format(t=t), '&' + if vt.type == "Enum" or vt.type == "EnumFlag": + return "{t}_tojson".format(t=t), "" + return "{t}_tojson".format(t=t), "&" def print_string(self, o): - '''Create cJSON object from vl_api_string_t''' + """Create cJSON object from vl_api_string_t""" write = self.stream.write if o.modern_vla: - write(' vl_api_string_cJSON_AddToObject(o, "{n}", &a->{n});\n' - .format(n=o.fieldname)) + write( + ' vl_api_string_cJSON_AddToObject(o, "{n}", &a->{n});\n'.format( + n=o.fieldname + ) + ) else: - - write(' cJSON_AddStringToObject(o, "{n}", (char *)a->{n});\n' - .format(n=o.fieldname)) + write( + ' cJSON_AddStringToObject(o, "{n}", (char *)a->{n});\n'.format( + n=o.fieldname + ) + ) def print_field(self, o): - '''Called for every field in a typedef or define.''' + """Called for every field in a typedef or define.""" write = self.stream.write if o.fieldname in self.noprint_fields: return @@ -130,19 +138,21 @@ class ToJSON(): f, p, newobj = self.get_json_func(o.fieldtype) if newobj: - write(' cJSON_AddItemToObject(o, "{n}", {f}({p}a->{n}));\n' - .format(f=f, p=p, n=o.fieldname)) + write( + ' cJSON_AddItemToObject(o, "{n}", {f}({p}a->{n}));\n'.format( + f=f, p=p, n=o.fieldname + ) + ) else: - write(' {f}(o, "{n}", {p}a->{n});\n' - .format(f=f, p=p, n=o.fieldname)) + write(' {f}(o, "{n}", {p}a->{n});\n'.format(f=f, p=p, n=o.fieldname)) - _dispatch['Field'] = print_field + _dispatch["Field"] = print_field def print_array(self, o): - '''Converts a VPP API array to cJSON array.''' + """Converts a VPP API array to cJSON array.""" write = self.stream.write - forloop = '''\ + forloop = """\ {{ int i; cJSON *array = cJSON_AddArrayToObject(o, "{n}"); @@ -150,232 +160,263 @@ class ToJSON(): cJSON_AddItemToArray(array, {f}({p}a->{n}[i])); }} }} -''' +""" - if o.fieldtype == 'string': + if o.fieldtype == "string": self.print_string(o) return - lfield = 'a->' + o.lengthfield if o.lengthfield else o.length - if o.fieldtype == 'u8': - write(' {\n') + lfield = "a->" + o.lengthfield if o.lengthfield else o.length + if o.fieldtype == "u8": + write(" {\n") # What is length field doing here? - write(' u8 *s = format(0, "0x%U", format_hex_bytes, ' - '&a->{n}, {lfield});\n' - .format(n=o.fieldname, lfield=lfield)) - write(' cJSON_AddStringToObject(o, "{n}", (char *)s);\n' - .format(n=o.fieldname)) - write(' vec_free(s);\n') - write(' }\n') + write( + ' u8 *s = format(0, "0x%U", format_hex_bytes, ' + "&a->{n}, {lfield});\n".format(n=o.fieldname, lfield=lfield) + ) + write( + ' cJSON_AddStringToObject(o, "{n}", (char *)s);\n'.format( + n=o.fieldname + ) + ) + write(" vec_free(s);\n") + write(" }\n") return f, p = self.get_json_array_func(o.fieldtype) - write(forloop.format(lfield=lfield, - t=o.fieldtype, - n=o.fieldname, - f=f, - p=p)) + write(forloop.format(lfield=lfield, t=o.fieldtype, n=o.fieldname, f=f, p=p)) - _dispatch['Array'] = print_array + _dispatch["Array"] = print_array def print_enum(self, o): - '''Create cJSON object (string) for VPP API enum''' + """Create cJSON object (string) for VPP API enum""" write = self.stream.write - write('static inline cJSON *vl_api_{name}_t_tojson ' - '(vl_api_{name}_t a) {{\n'.format(name=o.name)) + write( + "static inline cJSON *vl_api_{name}_t_tojson " + "(vl_api_{name}_t a) {{\n".format(name=o.name) + ) write(" switch(a) {\n") for b in o.block: write(" case %s:\n" % b[1]) write(' return cJSON_CreateString("{}");\n'.format(b[0])) write(' default: return cJSON_CreateString("Invalid ENUM");\n') - write(' }\n') - write(' return 0;\n') - write('}\n') + write(" }\n") + write(" return 0;\n") + write("}\n") - _dispatch['Enum'] = print_enum + _dispatch["Enum"] = print_enum def print_enum_flag(self, o): - '''Create cJSON object (string) for VPP API enum''' + """Create cJSON object (string) for VPP API enum""" write = self.stream.write - write('static inline cJSON *vl_api_{name}_t_tojson ' - '(vl_api_{name}_t a) {{\n'.format(name=o.name)) - write(' cJSON *array = cJSON_CreateArray();\n') + write( + "static inline cJSON *vl_api_{name}_t_tojson " + "(vl_api_{name}_t a) {{\n".format(name=o.name) + ) + write(" cJSON *array = cJSON_CreateArray();\n") for b in o.block: - write(' if (a & {})\n'.format(b[0])) - write(' cJSON_AddItemToArray(array, cJSON_CreateString("{}"));\n'.format(b[0])) - write(' return array;\n') - write('}\n') + if b[1] == 0: + continue + write(" if (a & {})\n".format(b[0])) + write( + ' cJSON_AddItemToArray(array, cJSON_CreateString("{}"));\n'.format( + b[0] + ) + ) + write(" return array;\n") + write("}\n") - _dispatch['EnumFlag'] = print_enum_flag + _dispatch["EnumFlag"] = print_enum_flag def print_typedef(self, o): - '''Create cJSON (dictionary) object from VPP API typedef''' + """Create cJSON (dictionary) object from VPP API typedef""" write = self.stream.write - write('static inline cJSON *vl_api_{name}_t_tojson ' - '(vl_api_{name}_t *a) {{\n'.format(name=o.name)) - write(' cJSON *o = cJSON_CreateObject();\n') + write( + "static inline cJSON *vl_api_{name}_t_tojson " + "(vl_api_{name}_t *a) {{\n".format(name=o.name) + ) + write(" cJSON *o = cJSON_CreateObject();\n") for t in o.block: self._dispatch[t.type](self, t) - write(' return o;\n') - write('}\n') + write(" return o;\n") + write("}\n") def print_define(self, o): - '''Create cJSON (dictionary) object from VPP API define''' + """Create cJSON (dictionary) object from VPP API define""" write = self.stream.write - write('static inline cJSON *vl_api_{name}_t_tojson ' - '(vl_api_{name}_t *a) {{\n'.format(name=o.name)) - write(' cJSON *o = cJSON_CreateObject();\n') - write(' cJSON_AddStringToObject(o, "_msgname", "{}");\n' - .format(o.name)) + write( + "static inline cJSON *vl_api_{name}_t_tojson " + "(vl_api_{name}_t *a) {{\n".format(name=o.name) + ) + write(" cJSON *o = cJSON_CreateObject();\n") + write(' cJSON_AddStringToObject(o, "_msgname", "{}");\n'.format(o.name)) + write( + ' cJSON_AddStringToObject(o, "_crc", "{crc:08x}");\n'.format(crc=o.crc) + ) for t in o.block: self._dispatch[t.type](self, t) - write(' return o;\n') - write('}\n') + write(" return o;\n") + write("}\n") def print_using(self, o): - '''Create cJSON (dictionary) object from VPP API aliased type''' + """Create cJSON (dictionary) object from VPP API aliased type""" if o.manual_print: return write = self.stream.write - write('static inline cJSON *vl_api_{name}_t_tojson ' - '(vl_api_{name}_t *a) {{\n'.format(name=o.name)) - - write(' u8 *s = format(0, "%U", format_vl_api_{}_t, a);\n' - .format(o.name)) - write(' cJSON *o = cJSON_CreateString((char *)s);\n') - write(' vec_free(s);\n') - write(' return o;\n') - write('}\n') - - _dispatch['Typedef'] = print_typedef - _dispatch['Define'] = print_define - _dispatch['Using'] = print_using - _dispatch['Union'] = print_typedef + write( + "static inline cJSON *vl_api_{name}_t_tojson " + "(vl_api_{name}_t *a) {{\n".format(name=o.name) + ) + + write(' u8 *s = format(0, "%U", format_vl_api_{}_t, a);\n'.format(o.name)) + write(" cJSON *o = cJSON_CreateString((char *)s);\n") + write(" vec_free(s);\n") + write(" return o;\n") + write("}\n") + + _dispatch["Typedef"] = print_typedef + _dispatch["Define"] = print_define + _dispatch["Using"] = print_using + _dispatch["Union"] = print_typedef def generate_function(self, t): - '''Main entry point''' + """Main entry point""" write = self.stream.write if t.manual_print: - write('/* Manual print {} */\n'.format(t.name)) + write("/* Manual print {} */\n".format(t.name)) return self._dispatch[t.type](self, t) def generate_types(self): - '''Main entry point''' + """Main entry point""" for t in self.types: self.generate_function(t) def generate_defines(self): - '''Main entry point''' + """Main entry point""" for t in self.defines: self.generate_function(t) -class FromJSON(): - ''' +class FromJSON: + """ Parse JSON objects into VPP API binary message structures. - ''' + """ + _dispatch = {} - noprint_fields = {'_vl_msg_id': None, - 'client_index': None, - 'context': None} - is_number = {'u8': None, - 'i8': None, - 'u16': None, - 'i16': None, - 'u32': None, - 'i32': None, - 'u64': None, - 'i64': None, - 'f64': None, - } + noprint_fields = {"_vl_msg_id": None, "client_index": None, "context": None} + is_number = { + "u8": None, + "i8": None, + "u16": None, + "i16": None, + "u32": None, + "i32": None, + "u64": None, + "i64": None, + "f64": None, + } def __init__(self, module, types, defines, imported_types, stream): self.stream = stream self.module = module self.defines = defines self.types = types - self.types_hash = {'vl_api_'+d.name+'_t': - d for d in types + imported_types} + self.types_hash = {"vl_api_" + d.name + "_t": d for d in types + imported_types} self.defines_hash = {d.name: d for d in defines} def header(self): - '''Output the top boilerplate.''' + """Output the top boilerplate.""" write = self.stream.write - write('#ifndef included_{}_api_fromjson_h\n'.format(self.module)) - write('#define included_{}_api_fromjson_h\n'.format(self.module)) - write('#include <vppinfra/cJSON.h>\n\n') - write('#include <vat2/jsonconvert.h>\n\n') + write("#ifndef included_{}_api_fromjson_h\n".format(self.module)) + write("#define included_{}_api_fromjson_h\n".format(self.module)) + write("#include <vppinfra/cJSON.h>\n\n") + write("#include <vppinfra/jsonformat.h>\n\n") write('#pragma GCC diagnostic ignored "-Wunused-label"\n') def is_base_type(self, t): - '''Check if a type is one of the VPP API base types''' + """Check if a type is one of the VPP API base types""" if t in self.is_number: return True - if t == 'bool': + if t == "bool": return True return False def footer(self): - '''Output the bottom boilerplate.''' + """Output the bottom boilerplate.""" write = self.stream.write - write('#endif\n') + write("#endif\n") def print_string(self, o, toplevel=False): - '''Convert JSON string to vl_api_string_t''' + """Convert JSON string to vl_api_string_t""" write = self.stream.write msgvar = "a" if toplevel else "*mp" msgsize = "l" if toplevel else "*len" if o.modern_vla: - write(' char *p = cJSON_GetStringValue(item);\n') - write(' size_t plen = strlen(p);\n') - write(' {msgvar} = realloc({msgvar}, {msgsize} + plen);\n' - .format(msgvar=msgvar, msgsize=msgsize)) - write(' if ({msgvar} == 0) goto error;\n'.format(msgvar=msgvar)) - write(' vl_api_c_string_to_api_string(p, (void *){msgvar} + ' - '{msgsize} - sizeof(vl_api_string_t));\n' - .format(msgvar=msgvar, msgsize=msgsize)) - write(' {msgsize} += plen;\n'.format(msgsize=msgsize)) + write(" char *p = cJSON_GetStringValue(item);\n") + write(" size_t plen = strlen(p);\n") + write( + " {msgvar} = cJSON_realloc({msgvar}, {msgsize} + plen);\n".format( + msgvar=msgvar, msgsize=msgsize + ) + ) + write(" if ({msgvar} == 0) goto error;\n".format(msgvar=msgvar)) + write( + " vl_api_c_string_to_api_string(p, (void *){msgvar} + " + "{msgsize} - sizeof(vl_api_string_t));\n".format( + msgvar=msgvar, msgsize=msgsize + ) + ) + write(" {msgsize} += plen;\n".format(msgsize=msgsize)) else: - write(' strncpy_s((char *)a->{n}, sizeof(a->{n}), ' - 'cJSON_GetStringValue(item), sizeof(a->{n}) - 1);\n' - .format(n=o.fieldname)) + write( + " strncpy_s((char *)a->{n}, sizeof(a->{n}), " + "cJSON_GetStringValue(item), sizeof(a->{n}) - 1);\n".format( + n=o.fieldname + ) + ) def print_field(self, o, toplevel=False): - '''Called for every field in a typedef or define.''' + """Called for every field in a typedef or define.""" write = self.stream.write if o.fieldname in self.noprint_fields: return is_bt = self.is_base_type(o.fieldtype) - t = 'vl_api_{}'.format(o.fieldtype) if is_bt else o.fieldtype + t = "vl_api_{}".format(o.fieldtype) if is_bt else o.fieldtype msgvar = "(void **)&a" if toplevel else "mp" msgsize = "&l" if toplevel else "len" if is_bt: - write(' vl_api_{t}_fromjson(item, &a->{n});\n' - .format(t=o.fieldtype, n=o.fieldname)) + write( + " vl_api_{t}_fromjson(item, &a->{n});\n".format( + t=o.fieldtype, n=o.fieldname + ) + ) else: - write(' if ({t}_fromjson({msgvar}, ' - '{msgsize}, item, &a->{n}) < 0) goto error;\n' - .format(t=t, n=o.fieldname, msgvar=msgvar, msgsize=msgsize)) + write( + " if ({t}_fromjson({msgvar}, " + "{msgsize}, item, &a->{n}) < 0) goto error;\n".format( + t=t, n=o.fieldname, msgvar=msgvar, msgsize=msgsize + ) + ) - _dispatch['Field'] = print_field + _dispatch["Field"] = print_field def print_array(self, o, toplevel=False): - '''Convert JSON array to VPP API array''' + """Convert JSON array to VPP API array""" write = self.stream.write - forloop = '''\ + forloop = """\ {{ int i; cJSON *array = cJSON_GetObjectItem(o, "{n}"); @@ -386,14 +427,14 @@ class FromJSON(): {call} }} }} -''' - forloop_vla = '''\ +""" + forloop_vla = """\ {{ int i; cJSON *array = cJSON_GetObjectItem(o, "{n}"); int size = cJSON_GetArraySize(array); {lfield} = size; - {realloc} = realloc({realloc}, {msgsize} + sizeof({t}) * size); + {realloc} = cJSON_realloc({realloc}, {msgsize} + sizeof({t}) * size); {t} *d = (void *){realloc} + {msgsize}; {msgsize} += sizeof({t}) * size; for (i = 0; i < size; i++) {{ @@ -401,271 +442,296 @@ class FromJSON(): {call} }} }} -''' +""" t = o.fieldtype - if o.fieldtype == 'string': + if o.fieldtype == "string": self.print_string(o, toplevel) return - lfield = 'a->' + o.lengthfield if o.lengthfield else o.length + lfield = "a->" + o.lengthfield if o.lengthfield else o.length msgvar = "(void **)&a" if toplevel else "mp" realloc = "a" if toplevel else "*mp" msgsize = "l" if toplevel else "*len" - if o.fieldtype == 'u8': + if o.fieldtype == "u8": if o.lengthfield: - write(' s = u8string_fromjson(o, "{}");\n' - .format(o.fieldname)) - write(' if (!s) goto error;\n') - write(' {} = vec_len(s);\n'.format(lfield)) - - write(' {realloc} = realloc({realloc}, {msgsize} + ' - 'vec_len(s));\n'.format(msgvar=msgvar, msgsize=msgsize, realloc=realloc)) - write(' memcpy((void *){realloc} + {msgsize}, s, ' - 'vec_len(s));\n'.format(realloc=realloc, msgsize=msgsize)) - write(' {msgsize} += vec_len(s);\n'.format(msgsize=msgsize)) - - write(' vec_free(s);\n') + write(' s = u8string_fromjson(o, "{}");\n'.format(o.fieldname)) + write(" if (!s) goto error;\n") + write(" {} = vec_len(s);\n".format(lfield)) + + write( + " {realloc} = cJSON_realloc({realloc}, {msgsize} + " + "vec_len(s));\n".format( + msgvar=msgvar, msgsize=msgsize, realloc=realloc + ) + ) + write( + " clib_memcpy((void *){realloc} + {msgsize}, s, " + "vec_len(s));\n".format(realloc=realloc, msgsize=msgsize) + ) + write(" {msgsize} += vec_len(s);\n".format(msgsize=msgsize)) + + write(" vec_free(s);\n") else: - write(' if (u8string_fromjson2(o, "{n}", a->{n}) < 0) goto error;\n' - .format(n=o.fieldname)) + write( + ' if (u8string_fromjson2(o, "{n}", a->{n}) < 0) goto error;\n'.format( + n=o.fieldname + ) + ) return is_bt = self.is_base_type(o.fieldtype) if o.lengthfield: if is_bt: - call = ('vl_api_{t}_fromjson(e, &d[i]);' - .format(t=o.fieldtype)) + call = "vl_api_{t}_fromjson(e, &d[i]);".format(t=o.fieldtype) else: - call = ('if ({t}_fromjson({msgvar}, len, e, &d[i]) < 0) goto error; ' - .format(t=o.fieldtype, msgvar=msgvar)) - write(forloop_vla.format(lfield=lfield, - t=o.fieldtype, - n=o.fieldname, - call=call, - realloc=realloc, - msgsize=msgsize)) + call = "if ({t}_fromjson({msgvar}, len, e, &d[i]) < 0) goto error; ".format( + t=o.fieldtype, msgvar=msgvar + ) + write( + forloop_vla.format( + lfield=lfield, + t=o.fieldtype, + n=o.fieldname, + call=call, + realloc=realloc, + msgsize=msgsize, + ) + ) else: if is_bt: - call = ('vl_api_{t}_fromjson(e, &a->{n}[i]);' - .format(t=t, n=o.fieldname)) + call = "vl_api_{t}_fromjson(e, &a->{n}[i]);".format(t=t, n=o.fieldname) else: - call = ('if ({}_fromjson({}, len, e, &a->{}[i]) < 0) goto error;' - .format(t, msgvar, o.fieldname)) - write(forloop.format(lfield=lfield, - t=t, - n=o.fieldname, - call=call, - msgvar=msgvar, - realloc=realloc, - msgsize=msgsize)) - - _dispatch['Array'] = print_array + call = "if ({}_fromjson({}, len, e, &a->{}[i]) < 0) goto error;".format( + t, msgvar, o.fieldname + ) + write( + forloop.format( + lfield=lfield, + t=t, + n=o.fieldname, + call=call, + msgvar=msgvar, + realloc=realloc, + msgsize=msgsize, + ) + ) + + _dispatch["Array"] = print_array def print_enum(self, o): - '''Convert to JSON enum(string) to VPP API enum (int)''' + """Convert to JSON enum(string) to VPP API enum (int)""" write = self.stream.write - write('static inline int vl_api_{n}_t_fromjson' - '(void **mp, int *len, cJSON *o, vl_api_{n}_t *a) {{\n' - .format(n=o.name)) - write(' char *p = cJSON_GetStringValue(o);\n') + write( + "static inline int vl_api_{n}_t_fromjson" + "(void **mp, int *len, cJSON *o, vl_api_{n}_t *a) {{\n".format(n=o.name) + ) + write(" char *p = cJSON_GetStringValue(o);\n") for b in o.block: - write(' if (strcmp(p, "{}") == 0) {{*a = {}; return 0;}}\n' - .format(b[0], b[1])) - write(' *a = 0;\n') - write(' return -1;\n') - write('}\n') + write( + ' if (strcmp(p, "{}") == 0) {{*a = {}; return 0;}}\n'.format( + b[0], b[1] + ) + ) + write(" *a = 0;\n") + write(" return -1;\n") + write("}\n") - _dispatch['Enum'] = print_enum + _dispatch["Enum"] = print_enum def print_enum_flag(self, o): - '''Convert to JSON enum(string) to VPP API enum (int)''' + """Convert to JSON enum(string) to VPP API enum (int)""" write = self.stream.write - write('static inline int vl_api_{n}_t_fromjson ' - '(void **mp, int *len, cJSON *o, vl_api_{n}_t *a) {{\n' - .format(n=o.name)) - write(' int i;\n') - write(' *a = 0;\n') - write(' for (i = 0; i < cJSON_GetArraySize(o); i++) {\n') - write(' cJSON *e = cJSON_GetArrayItem(o, i);\n') - write(' char *p = cJSON_GetStringValue(e);\n') - write(' if (!p) return -1;\n') + write( + "static inline int vl_api_{n}_t_fromjson " + "(void **mp, int *len, cJSON *o, vl_api_{n}_t *a) {{\n".format(n=o.name) + ) + write(" int i;\n") + write(" *a = 0;\n") + write(" for (i = 0; i < cJSON_GetArraySize(o); i++) {\n") + write(" cJSON *e = cJSON_GetArrayItem(o, i);\n") + write(" char *p = cJSON_GetStringValue(e);\n") + write(" if (!p) return -1;\n") for b in o.block: - write(' if (strcmp(p, "{}") == 0) *a |= {};\n' - .format(b[0], b[1])) - write(' }\n') - write(' return 0;\n') - write('}\n') + write(' if (strcmp(p, "{}") == 0) *a |= {};\n'.format(b[0], b[1])) + write(" }\n") + write(" return 0;\n") + write("}\n") - _dispatch['EnumFlag'] = print_enum_flag + _dispatch["EnumFlag"] = print_enum_flag def print_typedef(self, o): - '''Convert from JSON object to VPP API binary representation''' + """Convert from JSON object to VPP API binary representation""" write = self.stream.write - write('static inline int vl_api_{name}_t_fromjson (void **mp, ' - 'int *len, cJSON *o, vl_api_{name}_t *a) {{\n' - .format(name=o.name)) - write(' cJSON *item __attribute__ ((unused));\n') - write(' u8 *s __attribute__ ((unused));\n') + write( + "static inline int vl_api_{name}_t_fromjson (void **mp, " + "int *len, cJSON *o, vl_api_{name}_t *a) {{\n".format(name=o.name) + ) + write(" cJSON *item __attribute__ ((unused));\n") + write(" u8 *s __attribute__ ((unused));\n") for t in o.block: - if t.type == 'Field' and t.is_lengthfield: + if t.type == "Field" and t.is_lengthfield: continue - write('\n item = cJSON_GetObjectItem(o, "{}");\n' - .format(t.fieldname)) - write(' if (!item) goto error;\n') + write('\n item = cJSON_GetObjectItem(o, "{}");\n'.format(t.fieldname)) + write(" if (!item) goto error;\n") self._dispatch[t.type](self, t) - write('\n return 0;\n') - write('\n error:\n') - write(' return -1;\n') - write('}\n') + write("\n return 0;\n") + write("\n error:\n") + write(" return -1;\n") + write("}\n") def print_union(self, o): - '''Convert JSON object to VPP API binary union''' + """Convert JSON object to VPP API binary union""" write = self.stream.write - write('static inline int vl_api_{name}_t_fromjson (void **mp, ' - 'int *len, cJSON *o, vl_api_{name}_t *a) {{\n' - .format(name=o.name)) - write(' cJSON *item __attribute__ ((unused));\n') - write(' u8 *s __attribute__ ((unused));\n') + write( + "static inline int vl_api_{name}_t_fromjson (void **mp, " + "int *len, cJSON *o, vl_api_{name}_t *a) {{\n".format(name=o.name) + ) + write(" cJSON *item __attribute__ ((unused));\n") + write(" u8 *s __attribute__ ((unused));\n") for t in o.block: - if t.type == 'Field' and t.is_lengthfield: + if t.type == "Field" and t.is_lengthfield: continue - write(' item = cJSON_GetObjectItem(o, "{}");\n' - .format(t.fieldname)) - write(' if (item) {\n') + write(' item = cJSON_GetObjectItem(o, "{}");\n'.format(t.fieldname)) + write(" if (item) {\n") self._dispatch[t.type](self, t) - write(' };\n') - write('\n return 0;\n') - write('\n error:\n') - write(' return -1;\n') - write('}\n') + write(" };\n") + write("\n return 0;\n") + write("\n error:\n") + write(" return -1;\n") + write("}\n") def print_define(self, o): - '''Convert JSON object to VPP API message''' + """Convert JSON object to VPP API message""" write = self.stream.write error = 0 - write('static inline vl_api_{name}_t *vl_api_{name}_t_fromjson ' - '(cJSON *o, int *len) {{\n'.format(name=o.name)) - write(' cJSON *item __attribute__ ((unused));\n') - write(' u8 *s __attribute__ ((unused));\n') - write(' int l = sizeof(vl_api_{}_t);\n'.format(o.name)) - write(' vl_api_{}_t *a = malloc(l);\n'.format(o.name)) - write('\n') + write( + "static inline vl_api_{name}_t *vl_api_{name}_t_fromjson " + "(cJSON *o, int *len) {{\n".format(name=o.name) + ) + write(" cJSON *item __attribute__ ((unused));\n") + write(" u8 *s __attribute__ ((unused));\n") + write(" int l = sizeof(vl_api_{}_t);\n".format(o.name)) + write(" vl_api_{}_t *a = cJSON_malloc(l);\n".format(o.name)) + write("\n") for t in o.block: if t.fieldname in self.noprint_fields: continue - if t.type == 'Field' and t.is_lengthfield: + if t.type == "Field" and t.is_lengthfield: continue - write(' item = cJSON_GetObjectItem(o, "{}");\n' - .format(t.fieldname)) - write(' if (!item) goto error;\n') + write(' item = cJSON_GetObjectItem(o, "{}");\n'.format(t.fieldname)) + write(" if (!item) goto error;\n") error += 1 self._dispatch[t.type](self, t, toplevel=True) - write('\n') + write("\n") - write(' *len = l;\n') - write(' return a;\n') + write(" *len = l;\n") + write(" return a;\n") if error: - write('\n error:\n') - write(' free(a);\n') - write(' return 0;\n') - write('}\n') + write("\n error:\n") + write(" cJSON_free(a);\n") + write(" return 0;\n") + write("}\n") def print_using(self, o): - '''Convert JSON field to VPP type alias''' + """Convert JSON field to VPP type alias""" write = self.stream.write if o.manual_print: return t = o.using - write('static inline int vl_api_{name}_t_fromjson (void **mp, ' - 'int *len, cJSON *o, vl_api_{name}_t *a) {{\n' - .format(name=o.name)) - if 'length' in o.alias: - if t.fieldtype != 'u8': - raise ValueError("Error in processing type {} for {}" - .format(t.fieldtype, o.name)) - write(' vl_api_u8_string_fromjson(o, (u8 *)a, {});\n' - .format(o.alias['length'])) + write( + "static inline int vl_api_{name}_t_fromjson (void **mp, " + "int *len, cJSON *o, vl_api_{name}_t *a) {{\n".format(name=o.name) + ) + if "length" in o.alias: + if t.fieldtype != "u8": + raise ValueError( + "Error in processing type {} for {}".format(t.fieldtype, o.name) + ) + write( + " vl_api_u8_string_fromjson(o, (u8 *)a, {});\n".format( + o.alias["length"] + ) + ) else: - write(' vl_api_{t}_fromjson(o, ({t} *)a);\n' - .format(t=t.fieldtype)) + write(" vl_api_{t}_fromjson(o, ({t} *)a);\n".format(t=t.fieldtype)) - write(' return 0;\n') - write('}\n') + write(" return 0;\n") + write("}\n") - _dispatch['Typedef'] = print_typedef - _dispatch['Define'] = print_define - _dispatch['Using'] = print_using - _dispatch['Union'] = print_union + _dispatch["Typedef"] = print_typedef + _dispatch["Define"] = print_define + _dispatch["Using"] = print_using + _dispatch["Union"] = print_union def generate_function(self, t): - '''Main entry point''' + """Main entry point""" write = self.stream.write if t.manual_print: - write('/* Manual print {} */\n'.format(t.name)) + write("/* Manual print {} */\n".format(t.name)) return self._dispatch[t.type](self, t) def generate_types(self): - '''Main entry point''' + """Main entry point""" for t in self.types: self.generate_function(t) def generate_defines(self): - '''Main entry point''' + """Main entry point""" for t in self.defines: self.generate_function(t) def generate_tojson(s, modulename, stream): - '''Generate all functions to convert from API to JSON''' + """Generate all functions to convert from API to JSON""" write = stream.write - write('/* Imported API files */\n') - for i in s['Import']: - f = i.filename.replace('plugins/', '') - write('#include <{}_tojson.h>\n'.format(f)) + write("/* Imported API files */\n") + for i in s["Import"]: + f = i.filename.replace("plugins/", "") + write("#include <{}_tojson.h>\n".format(f)) - pp = ToJSON(modulename, s['types'], s['Define'], s['imported']['types'], - stream) + pp = ToJSON(modulename, s["types"], s["Define"], s["imported"]["types"], stream) pp.header() pp.generate_types() pp.generate_defines() pp.footer() - return '' + return "" def generate_fromjson(s, modulename, stream): - '''Generate all functions to convert from JSON to API''' + """Generate all functions to convert from JSON to API""" write = stream.write - write('/* Imported API files */\n') - for i in s['Import']: - f = i.filename.replace('plugins/', '') - write('#include <{}_fromjson.h>\n'.format(f)) + write("/* Imported API files */\n") + for i in s["Import"]: + f = i.filename.replace("plugins/", "") + write("#include <{}_fromjson.h>\n".format(f)) - pp = FromJSON(modulename, s['types'], s['Define'], s['imported']['types'], - stream) + pp = FromJSON(modulename, s["types"], s["Define"], s["imported"]["types"], stream) pp.header() pp.generate_types() pp.generate_defines() pp.footer() - return '' + return "" + ############################################################################### DATESTRING = datetime.datetime.utcfromtimestamp( - int(os.environ.get('SOURCE_DATE_EPOCH', time.time()))) -TOP_BOILERPLATE = '''\ + int(os.environ.get("SOURCE_DATE_EPOCH", time.time())) +) +TOP_BOILERPLATE = """\ /* * VLIB API definitions {datestring} * Input file: {input_filename} @@ -677,52 +743,65 @@ TOP_BOILERPLATE = '''\ || defined(vl_printfun) ||defined(vl_endianfun) \\ || defined(vl_api_version)||defined(vl_typedefs) \\ || defined(vl_msg_name)||defined(vl_msg_name_crc_list) \\ - || defined(vl_api_version_tuple) + || defined(vl_api_version_tuple) || defined(vl_calcsizefun) /* ok, something was selected */ #else #warning no content included from {input_filename} #endif #define VL_API_PACKED(x) x __attribute__ ((packed)) -''' -BOTTOM_BOILERPLATE = '''\ +/* + * Note: VL_API_MAX_ARRAY_SIZE is set to an arbitrarily large limit. + * + * However, any message with a ~2 billion element array is likely to break the + * api handling long before this limit causes array element endian issues. + * + * Applications should be written to create reasonable api messages. + */ +#define VL_API_MAX_ARRAY_SIZE 0x7fffffff + +""" + +BOTTOM_BOILERPLATE = """\ /****** API CRC (whole file) *****/ #ifdef vl_api_version vl_api_version({input_filename}, {file_crc:#08x}) #endif -''' +""" def msg_ids(s): - '''Generate macro to map API message id to handler''' - output = '''\ + """Generate macro to map API message id to handler""" + output = """\ /****** Message ID / handler enum ******/ #ifdef vl_msg_id -''' +""" - for t in s['Define']: - output += "vl_msg_id(VL_API_%s, vl_api_%s_t_handler)\n" % \ - (t.name.upper(), t.name) + for t in s["Define"]: + output += "vl_msg_id(VL_API_%s, vl_api_%s_t_handler)\n" % ( + t.name.upper(), + t.name, + ) output += "#endif" return output def msg_names(s): - '''Generate calls to name mapping macro''' - output = '''\ + """Generate calls to name mapping macro""" + output = """\ /****** Message names ******/ #ifdef vl_msg_name -''' +""" - for t in s['Define']: + for t in s["Define"]: dont_trace = 0 if t.dont_trace else 1 output += "vl_msg_name(vl_api_%s_t, %d)\n" % (t.name, dont_trace) output += "#endif" @@ -731,190 +810,215 @@ def msg_names(s): def msg_name_crc_list(s, suffix): - '''Generate list of names to CRC mappings''' - output = '''\ + """Generate list of names to CRC mappings""" + output = """\ /****** Message name, crc list ******/ #ifdef vl_msg_name_crc_list -''' +""" output += "#define foreach_vl_msg_name_crc_%s " % suffix - for t in s['Define']: - output += "\\\n_(VL_API_%s, %s, %08x) " % \ - (t.name.upper(), t.name, t.crc) + for t in s["Define"]: + output += "\\\n_(VL_API_%s, %s, %08x) " % (t.name.upper(), t.name, t.crc) output += "\n#endif" return output def api2c(fieldtype): - '''Map between API type names and internal VPP type names''' - mappingtable = {'string': 'vl_api_string_t', } + """Map between API type names and internal VPP type names""" + mappingtable = { + "string": "vl_api_string_t", + } if fieldtype in mappingtable: return mappingtable[fieldtype] return fieldtype def typedefs(filename): - '''Include in the main files to the types file''' - output = '''\ + """Include in the main files to the types file""" + output = """\ /****** Typedefs ******/ #ifdef vl_typedefs #include "{include}.api_types.h" #endif -'''.format(include=filename) +""".format( + include=filename + ) return output -FORMAT_STRINGS = {'u8': '%u', - 'bool': '%u', - 'i8': '%d', - 'u16': '%u', - 'i16': '%d', - 'u32': '%u', - 'i32': '%ld', - 'u64': '%llu', - 'i64': '%lld', - 'f64': '%.2f'} +FORMAT_STRINGS = { + "u8": "%u", + "bool": "%u", + "i8": "%d", + "u16": "%u", + "i16": "%d", + "u32": "%u", + "i32": "%ld", + "u64": "%llu", + "i64": "%lld", + "f64": "%.2f", +} + +class Printfun: + """Functions for pretty printing VPP API messages""" -class Printfun(): - '''Functions for pretty printing VPP API messages''' _dispatch = {} - noprint_fields = {'_vl_msg_id': None, - 'client_index': None, - 'context': None} + noprint_fields = {"_vl_msg_id": None, "client_index": None, "context": None} def __init__(self, stream): self.stream = stream @staticmethod def print_string(o, stream): - '''Pretty print a vl_api_string_t''' + """Pretty print a vl_api_string_t""" write = stream.write if o.modern_vla: - write(' if (vl_api_string_len(&a->{f}) > 0) {{\n' - .format(f=o.fieldname)) - write(' s = format(s, "\\n%U{f}: %U", ' - 'format_white_space, indent, ' - 'vl_api_format_string, (&a->{f}));\n'.format(f=o.fieldname)) - write(' } else {\n') - write(' s = format(s, "\\n%U{f}:", ' - 'format_white_space, indent);\n'.format(f=o.fieldname)) - write(' }\n') + write(" if (vl_api_string_len(&a->{f}) > 0) {{\n".format(f=o.fieldname)) + write( + ' s = format(s, "\\n%U{f}: %U", ' + "format_white_space, indent, " + "vl_api_format_string, (&a->{f}));\n".format(f=o.fieldname) + ) + write(" } else {\n") + write( + ' s = format(s, "\\n%U{f}:", ' + "format_white_space, indent);\n".format(f=o.fieldname) + ) + write(" }\n") else: - write(' s = format(s, "\\n%U{f}: %s", ' - 'format_white_space, indent, a->{f});\n' - .format(f=o.fieldname)) + write( + ' s = format(s, "\\n%U{f}: %s", ' + "format_white_space, indent, a->{f});\n".format(f=o.fieldname) + ) def print_field(self, o, stream): - '''Pretty print API field''' + """Pretty print API field""" write = stream.write if o.fieldname in self.noprint_fields: return if o.fieldtype in FORMAT_STRINGS: f = FORMAT_STRINGS[o.fieldtype] - write(' s = format(s, "\\n%U{n}: {f}", ' - 'format_white_space, indent, a->{n});\n' - .format(n=o.fieldname, f=f)) + write( + ' s = format(s, "\\n%U{n}: {f}", ' + "format_white_space, indent, a->{n});\n".format(n=o.fieldname, f=f) + ) else: - write(' s = format(s, "\\n%U{n}: %U", ' - 'format_white_space, indent, ' - 'format_{t}, &a->{n}, indent);\n' - .format(n=o.fieldname, t=o.fieldtype)) + write( + ' s = format(s, "\\n%U{n}: %U", ' + "format_white_space, indent, " + "format_{t}, &a->{n}, indent);\n".format(n=o.fieldname, t=o.fieldtype) + ) - _dispatch['Field'] = print_field + _dispatch["Field"] = print_field def print_array(self, o, stream): - '''Pretty print API array''' + """Pretty print API array""" write = stream.write - forloop = '''\ + forloop = """\ for (i = 0; i < {lfield}; i++) {{ s = format(s, "\\n%U{n}: %U", format_white_space, indent, format_{t}, &a->{n}[i], indent); }} -''' +""" - forloop_format = '''\ + forloop_format = """\ for (i = 0; i < {lfield}; i++) {{ s = format(s, "\\n%U{n}: {t}", format_white_space, indent, a->{n}[i]); }} -''' +""" - if o.fieldtype == 'string': + if o.fieldtype == "string": self.print_string(o, stream) return - if o.fieldtype == 'u8': + if o.fieldtype == "u8": if o.lengthfield: - write(' s = format(s, "\\n%U{n}: %U", format_white_space, ' - 'indent, format_hex_bytes, a->{n}, a->{lfield});\n' - .format(n=o.fieldname, lfield=o.lengthfield)) + write( + ' s = format(s, "\\n%U{n}: %U", format_white_space, ' + "indent, format_hex_bytes, a->{n}, a->{lfield});\n".format( + n=o.fieldname, lfield=o.lengthfield + ) + ) else: - write(' s = format(s, "\\n%U{n}: %U", format_white_space, ' - 'indent, format_hex_bytes, a, {lfield});\n' - .format(n=o.fieldname, lfield=o.length)) + write( + ' s = format(s, "\\n%U{n}: %U", format_white_space, ' + "indent, format_hex_bytes, a, {lfield});\n".format( + n=o.fieldname, lfield=o.length + ) + ) return - lfield = 'a->' + o.lengthfield if o.lengthfield else o.length + lfield = "a->" + o.lengthfield if o.lengthfield else o.length if o.fieldtype in FORMAT_STRINGS: - write(forloop_format.format(lfield=lfield, - t=FORMAT_STRINGS[o.fieldtype], - n=o.fieldname)) + write( + forloop_format.format( + lfield=lfield, t=FORMAT_STRINGS[o.fieldtype], n=o.fieldname + ) + ) else: write(forloop.format(lfield=lfield, t=o.fieldtype, n=o.fieldname)) - _dispatch['Array'] = print_array + _dispatch["Array"] = print_array @staticmethod def print_alias(k, v, stream): - '''Pretty print type alias''' + """Pretty print type alias""" write = stream.write - if ('length' in v.alias and v.alias['length'] and - v.alias['type'] == 'u8'): - write(' return format(s, "%U", format_hex_bytes, a, {});\n' - .format(v.alias['length'])) - elif v.alias['type'] in FORMAT_STRINGS: - write(' return format(s, "{}", *a);\n' - .format(FORMAT_STRINGS[v.alias['type']])) + if "length" in v.alias and v.alias["length"] and v.alias["type"] == "u8": + write( + ' return format(s, "%U", format_hex_bytes, a, {});\n'.format( + v.alias["length"] + ) + ) + elif v.alias["type"] in FORMAT_STRINGS: + write( + ' return format(s, "{}", *a);\n'.format( + FORMAT_STRINGS[v.alias["type"]] + ) + ) else: - write(' return format(s, "{} (print not implemented)");\n' - .format(k)) + write(' return format(s, "{} (print not implemented)");\n'.format(k)) @staticmethod def print_enum(o, stream): - '''Pretty print API enum''' + """Pretty print API enum""" write = stream.write write(" switch(*a) {\n") for b in o: write(" case %s:\n" % b[1]) write(' return format(s, "{}");\n'.format(b[0])) - write(' }\n') + write(" }\n") - _dispatch['Enum'] = print_enum - _dispatch['EnumFlag'] = print_enum + _dispatch["Enum"] = print_enum + _dispatch["EnumFlag"] = print_enum def print_obj(self, o, stream): - '''Entry point''' + """Entry point""" write = stream.write if o.type in self._dispatch: self._dispatch[o.type](self, o, stream) else: - write(' s = format(s, "\\n{} {} {} (print not implemented");\n' - .format(o.type, o.fieldtype, o.fieldname)) + write( + ' s = format(s, "\\n{} {} {} (print not implemented");\n'.format( + o.type, o.fieldtype, o.fieldname + ) + ) def printfun(objs, stream, modulename): - '''Main entry point for pretty print function generation''' + """Main entry point for pretty print function generation""" write = stream.write - h = '''\ + h = """\ /****** Print functions *****/ #ifdef vl_printfun #ifndef included_{module}_printfun @@ -928,15 +1032,18 @@ def printfun(objs, stream, modulename): #define _uword_cast long #endif -''' +#include "{module}.api_tojson.h" +#include "{module}.api_fromjson.h" + +""" - signature = '''\ -static inline void *vl_api_{name}_t_print (vl_api_{name}_t *a, void *handle) + signature = """\ +static inline u8 *vl_api_{name}_t_format (u8 *s, va_list *args) {{ - u8 *s = 0; + __attribute__((unused)) vl_api_{name}_t *a = va_arg (*args, vl_api_{name}_t *); u32 indent __attribute__((unused)) = 2; int i __attribute__((unused)); -''' +""" h = h.format(module=modulename) write(h) @@ -944,163 +1051,178 @@ static inline void *vl_api_{name}_t_print (vl_api_{name}_t *a, void *handle) pp = Printfun(stream) for t in objs: if t.manual_print: - write("/***** manual: vl_api_%s_t_print *****/\n\n" % t.name) + write("/***** manual: vl_api_%s_t_format *****/\n\n" % t.name) continue - write(signature.format(name=t.name)) - write(' /* Message definition: vl_api_{}_t: */\n'.format(t.name)) - write(" s = format(s, \"vl_api_%s_t:\");\n" % t.name) + write(signature.format(name=t.name, suffix="")) + write(" /* Message definition: vl_api_{}_t: */\n".format(t.name)) + write(' s = format(s, "vl_api_%s_t:");\n' % t.name) for o in t.block: pp.print_obj(o, stream) - write(' vec_add1(s, 0);\n') - write(' vl_print (handle, (char *)s);\n') - write(' vec_free (s);\n') - write(' return handle;\n') - write('}\n\n') + write(" return s;\n") + write("}\n\n") write("\n#endif") write("\n#endif /* vl_printfun */\n") - return '' + return "" def printfun_types(objs, stream, modulename): - '''Pretty print API types''' + """Pretty print API types""" write = stream.write pp = Printfun(stream) - h = '''\ + h = """\ /****** Print functions *****/ #ifdef vl_printfun #ifndef included_{module}_printfun_types #define included_{module}_printfun_types -''' +""" h = h.format(module=modulename) write(h) - signature = '''\ + signature = """\ static inline u8 *format_vl_api_{name}_t (u8 *s, va_list * args) {{ vl_api_{name}_t *a = va_arg (*args, vl_api_{name}_t *); u32 indent __attribute__((unused)) = va_arg (*args, u32); int i __attribute__((unused)); indent += 2; -''' +""" for t in objs: - if t.__class__.__name__ == 'Enum' or t.__class__.__name__ == 'EnumFlag': + if t.__class__.__name__ == "Enum" or t.__class__.__name__ == "EnumFlag": write(signature.format(name=t.name)) pp.print_enum(t.block, stream) - write(' return s;\n') - write('}\n\n') + write(" return s;\n") + write("}\n\n") continue if t.manual_print: - write("/***** manual: vl_api_%s_t_print *****/\n\n" % t.name) + write("/***** manual: vl_api_%s_t_format *****/\n\n" % t.name) continue - if t.__class__.__name__ == 'Using': + if t.__class__.__name__ == "Using": write(signature.format(name=t.name)) pp.print_alias(t.name, t, stream) - write('}\n\n') + write("}\n\n") continue write(signature.format(name=t.name)) for o in t.block: pp.print_obj(o, stream) - write(' return s;\n') - write('}\n\n') + write(" return s;\n") + write("}\n\n") write("\n#endif") write("\n#endif /* vl_printfun_types */\n") def generate_imports(imports): - '''Add #include matching the API import statements''' - output = '/* Imported API files */\n' - output += '#ifndef vl_api_version\n' + """Add #include matching the API import statements""" + output = "/* Imported API files */\n" + output += "#ifndef vl_api_version\n" for i in imports: - s = i.filename.replace('plugins/', '') - output += '#include <{}.h>\n'.format(s) - output += '#endif\n' + s = i.filename.replace("plugins/", "") + output += "#include <{}.h>\n".format(s) + output += "#endif\n" return output ENDIAN_STRINGS = { - 'u16': 'clib_net_to_host_u16', - 'u32': 'clib_net_to_host_u32', - 'u64': 'clib_net_to_host_u64', - 'i16': 'clib_net_to_host_i16', - 'i32': 'clib_net_to_host_i32', - 'i64': 'clib_net_to_host_i64', - 'f64': 'clib_net_to_host_f64', + "u16": "clib_net_to_host_u16", + "u32": "clib_net_to_host_u32", + "u64": "clib_net_to_host_u64", + "i16": "clib_net_to_host_i16", + "i32": "clib_net_to_host_i32", + "i64": "clib_net_to_host_i64", + "f64": "clib_net_to_host_f64", } +def get_endian_string(o, fieldtype): + """Return proper endian string conversion function""" + return ENDIAN_STRINGS[fieldtype] + + def endianfun_array(o): - '''Generate endian functions for arrays''' - forloop = '''\ + """Generate endian functions for arrays""" + forloop = """\ + ASSERT((u32){length} <= (u32)VL_API_MAX_ARRAY_SIZE); for (i = 0; i < {length}; i++) {{ a->{name}[i] = {format}(a->{name}[i]); }} -''' +""" - forloop_format = '''\ + forloop_format = """\ for (i = 0; i < {length}; i++) {{ - {type}_endian(&a->{name}[i]); + {type}_endian(&a->{name}[i], to_net); }} -''' +""" - output = '' - if o.fieldtype == 'u8' or o.fieldtype == 'string' or o.fieldtype == 'bool': - output += ' /* a->{n} = a->{n} (no-op) */\n'.format(n=o.fieldname) + output = "" + if o.fieldtype == "u8" or o.fieldtype == "string" or o.fieldtype == "bool": + output += " /* a->{n} = a->{n} (no-op) */\n".format(n=o.fieldname) else: - lfield = 'a->' + o.lengthfield if o.lengthfield else o.length + lfield = "a->" + o.lengthfield if o.lengthfield else o.length + if o.lengthfield: + output += ( + f" u32 count = to_net ? clib_host_to_net_u32(a->{o.lengthfield}) : " + f"a->{o.lengthfield};\n" + ) + lfield = "count" + else: + lfield = o.length + if o.fieldtype in ENDIAN_STRINGS: - output += (forloop - .format(length=lfield, - format=ENDIAN_STRINGS[o.fieldtype], - name=o.fieldname)) + output += forloop.format( + length=lfield, + format=get_endian_string(o, o.fieldtype), + name=o.fieldname, + ) else: - output += (forloop_format - .format(length=lfield, type=o.fieldtype, - name=o.fieldname)) + output += forloop_format.format( + length=lfield, type=o.fieldtype, name=o.fieldname + ) return output -NO_ENDIAN_CONVERSION = {'client_index': None} +NO_ENDIAN_CONVERSION = {"client_index": None} def endianfun_obj(o): - '''Generate endian conversion function for type''' - output = '' - if o.type == 'Array': + """Generate endian conversion function for type""" + output = "" + if o.type == "Array": return endianfun_array(o) - if o.type != 'Field': - output += (' s = format(s, "\\n{} {} {} (print not implemented");\n' - .format(o.type, o.fieldtype, o.fieldname)) + if o.type != "Field": + output += ' s = format(s, "\\n{} {} {} (print not implemented");\n'.format( + o.type, o.fieldtype, o.fieldname + ) return output if o.fieldname in NO_ENDIAN_CONVERSION: - output += ' /* a->{n} = a->{n} (no-op) */\n'.format(n=o.fieldname) + output += " /* a->{n} = a->{n} (no-op) */\n".format(n=o.fieldname) return output if o.fieldtype in ENDIAN_STRINGS: - output += (' a->{name} = {format}(a->{name});\n' - .format(name=o.fieldname, - format=ENDIAN_STRINGS[o.fieldtype])) - elif o.fieldtype.startswith('vl_api_'): - output += (' {type}_endian(&a->{name});\n' - .format(type=o.fieldtype, name=o.fieldname)) + output += " a->{name} = {format}(a->{name});\n".format( + name=o.fieldname, format=get_endian_string(o, o.fieldtype) + ) + elif o.fieldtype.startswith("vl_api_"): + output += " {type}_endian(&a->{name}, to_net);\n".format( + type=o.fieldtype, name=o.fieldname + ) else: - output += ' /* a->{n} = a->{n} (no-op) */\n'.format(n=o.fieldname) + output += " /* a->{n} = a->{n} (no-op) */\n".format(n=o.fieldname) return output def endianfun(objs, modulename): - '''Main entry point for endian function generation''' - output = '''\ + """Main entry point for endian function generation""" + output = """\ /****** Endian swap functions *****/\n\ #ifdef vl_endianfun @@ -1108,57 +1230,61 @@ def endianfun(objs, modulename): #define included_{module}_endianfun #undef clib_net_to_host_uword +#undef clib_host_to_net_uword #ifdef LP64 #define clib_net_to_host_uword clib_net_to_host_u64 +#define clib_host_to_net_uword clib_host_to_net_u64 #else #define clib_net_to_host_uword clib_net_to_host_u32 +#define clib_host_to_net_uword clib_host_to_net_u32 #endif -''' +""" output = output.format(module=modulename) - signature = '''\ -static inline void vl_api_{name}_t_endian (vl_api_{name}_t *a) + signature = """\ +static inline void vl_api_{name}_t_endian (vl_api_{name}_t *a, bool to_net) {{ int i __attribute__((unused)); -''' +""" for t in objs: - if t.__class__.__name__ == 'Enum' or t.__class__.__name__ == 'EnumFlag' : + if t.__class__.__name__ == "Enum" or t.__class__.__name__ == "EnumFlag": output += signature.format(name=t.name) if t.enumtype in ENDIAN_STRINGS: - output += (' *a = {}(*a);\n' - .format(ENDIAN_STRINGS[t.enumtype])) + output += " *a = {}(*a);\n".format(get_endian_string(t, t.enumtype)) else: - output += (' /* a->{name} = a->{name} (no-op) */\n' - .format(name=t.name)) + output += " /* a->{name} = a->{name} (no-op) */\n".format( + name=t.name + ) - output += '}\n\n' + output += "}\n\n" continue if t.manual_endian: output += "/***** manual: vl_api_%s_t_endian *****/\n\n" % t.name continue - if t.__class__.__name__ == 'Using': + if t.__class__.__name__ == "Using": output += signature.format(name=t.name) - if ('length' in t.alias and t.alias['length'] and - t.alias['type'] == 'u8'): - output += (' /* a->{name} = a->{name} (no-op) */\n' - .format(name=t.name)) - elif t.alias['type'] in FORMAT_STRINGS: - output += (' *a = {}(*a);\n' - .format(ENDIAN_STRINGS[t.alias['type']])) + if "length" in t.alias and t.alias["length"] and t.alias["type"] == "u8": + output += " /* a->{name} = a->{name} (no-op) */\n".format( + name=t.name + ) + elif t.alias["type"] in FORMAT_STRINGS: + output += " *a = {}(*a);\n".format( + get_endian_string(t, t.alias["type"]) + ) else: - output += ' /* Not Implemented yet {} */'.format(t.name) - output += '}\n\n' + output += " /* Not Implemented yet {} */".format(t.name) + output += "}\n\n" continue output += signature.format(name=t.name) for o in t.block: output += endianfun_obj(o) - output += '}\n\n' + output += "}\n\n" output += "\n#endif" output += "\n#endif /* vl_endianfun */\n\n" @@ -1166,19 +1292,103 @@ static inline void vl_api_{name}_t_endian (vl_api_{name}_t *a) return output +def calc_size_fun(objs, modulename): + """Main entry point for calculate size function generation""" + output = """\ + +/****** Calculate size functions *****/\n\ +#ifdef vl_calcsizefun +#ifndef included_{module}_calcsizefun +#define included_{module}_calcsizefun + +""" + output = output.format(module=modulename) + + signature = """\ +/* calculate message size of message in network byte order */ +static inline uword vl_api_{name}_t_calc_size (vl_api_{name}_t *a) +{{ +""" + + for o in objs: + tname = o.__class__.__name__ + + output += signature.format(name=o.name) + output += f" return sizeof(*a)" + if tname == "Using": + if "length" in o.alias: + try: + tmp = int(o.alias["length"]) + if tmp == 0: + raise (f"Unexpected length '0' for alias {o}") + except: + # output += f" + vl_api_{o.alias.name}_t_calc_size({o.name})" + print("culprit:") + print(o) + print(dir(o.alias)) + print(o.alias) + raise + elif tname == "Enum" or tname == "EnumFlag": + pass + else: + for b in o.block: + if b.type == "Option": + continue + elif b.type == "Field": + if b.fieldtype.startswith("vl_api_"): + output += f" - sizeof(a->{b.fieldname})" + output += f" + {b.fieldtype}_calc_size(&a->{b.fieldname})" + elif b.type == "Array": + if b.lengthfield: + m = list( + filter(lambda x: x.fieldname == b.lengthfield, o.block) + ) + if len(m) != 1: + raise Exception( + f"Expected 1 match for field '{b.lengthfield}', got '{m}'" + ) + lf = m[0] + if lf.fieldtype in ENDIAN_STRINGS: + output += f" + {get_endian_string(b, lf.fieldtype)}(a->{b.lengthfield}) * sizeof(a->{b.fieldname}[0])" + elif lf.fieldtype == "u8": + output += ( + f" + a->{b.lengthfield} * sizeof(a->{b.fieldname}[0])" + ) + else: + raise Exception( + f"Don't know how to endian swap {lf.fieldtype}" + ) + else: + # Fixed length strings decay to nul terminated u8 + if b.fieldtype == "string": + if b.modern_vla: + output += f" + vl_api_string_len(&a->{b.fieldname})" + + output += ";\n" + output += "}\n\n" + output += "\n#endif" + output += "\n#endif /* vl_calcsizefun */\n\n" + + return output + + def version_tuple(s, module): - '''Generate semantic version string''' - output = '''\ + """Generate semantic version string""" + output = """\ /****** Version tuple *****/ #ifdef vl_api_version_tuple -''' - if 'version' in s['Option']: - v = s['Option']['version'] - (major, minor, patch) = v.split('.') - output += "vl_api_version_tuple(%s, %s, %s, %s)\n" % \ - (module, major, minor, patch) +""" + if "version" in s["Option"]: + v = s["Option"]["version"] + (major, minor, patch) = v.split(".") + output += "vl_api_version_tuple(%s, %s, %s, %s)\n" % ( + module, + major, + minor, + patch, + ) output += "\n#endif /* vl_api_version_tuple */\n\n" @@ -1186,224 +1396,288 @@ def version_tuple(s, module): def generate_include_enum(s, module, stream): - '''Generate <name>.api_enum.h''' + """Generate <name>.api_enum.h""" write = stream.write - if 'Define' in s: - write('typedef enum {\n') - for t in s['Define']: - write(' VL_API_{},\n'.format(t.name.upper())) - write(' VL_MSG_{}_LAST\n'.format(module.upper())) - write('}} vl_api_{}_enum_t;\n'.format(module)) + if "Define" in s: + write("typedef enum {\n") + for t in s["Define"]: + write(" VL_API_{},\n".format(t.name.upper())) + write(" VL_MSG_{}_LAST\n".format(module.upper())) + write("}} vl_api_{}_enum_t;\n".format(module)) def generate_include_counters(s, stream): - '''Include file for the counter data model types.''' + """Include file for the counter data model types.""" write = stream.write for counters in s: csetname = counters.name - write('typedef enum {\n') + write("typedef enum {\n") for c in counters.block: - write(' {}_ERROR_{},\n' - .format(csetname.upper(), c['name'].upper())) - write(' {}_N_ERROR\n'.format(csetname.upper())) - write('}} vl_counter_{}_enum_t;\n'.format(csetname)) + write(" {}_ERROR_{},\n".format(csetname.upper(), c["name"].upper())) + write(" {}_N_ERROR\n".format(csetname.upper())) + write("}} vl_counter_{}_enum_t;\n".format(csetname)) - write('extern vlib_error_desc_t {}_error_counters[];\n'.format(csetname)) + write("extern vlib_error_desc_t {}_error_counters[];\n".format(csetname)) def generate_include_types(s, module, stream): - '''Generate separate API _types file.''' + """Generate separate API _types file.""" write = stream.write - write('#ifndef included_{module}_api_types_h\n'.format(module=module)) - write('#define included_{module}_api_types_h\n'.format(module=module)) - - if 'version' in s['Option']: - v = s['Option']['version'] - (major, minor, patch) = v.split('.') - write('#define VL_API_{m}_API_VERSION_MAJOR {v}\n' - .format(m=module.upper(), v=major)) - write('#define VL_API_{m}_API_VERSION_MINOR {v}\n' - .format(m=module.upper(), v=minor)) - write('#define VL_API_{m}_API_VERSION_PATCH {v}\n' - .format(m=module.upper(), v=patch)) - - if 'Import' in s: - write('/* Imported API files */\n') - for i in s['Import']: - filename = i.filename.replace('plugins/', '') - write('#include <{}_types.h>\n'.format(filename)) - - for o in s['types'] + s['Define']: + write("#ifndef included_{module}_api_types_h\n".format(module=module)) + write("#define included_{module}_api_types_h\n".format(module=module)) + + if "version" in s["Option"]: + v = s["Option"]["version"] + (major, minor, patch) = v.split(".") + write( + "#define VL_API_{m}_API_VERSION_MAJOR {v}\n".format( + m=module.upper(), v=major + ) + ) + write( + "#define VL_API_{m}_API_VERSION_MINOR {v}\n".format( + m=module.upper(), v=minor + ) + ) + write( + "#define VL_API_{m}_API_VERSION_PATCH {v}\n".format( + m=module.upper(), v=patch + ) + ) + + if "Import" in s: + write("/* Imported API files */\n") + for i in s["Import"]: + filename = i.filename.replace("plugins/", "") + write("#include <{}_types.h>\n".format(filename)) + + for o in itertools.chain(s["types"], s["Define"]): tname = o.__class__.__name__ - if tname == 'Using': - if 'length' in o.alias: - write('typedef %s vl_api_%s_t[%s];\n' % - (o.alias['type'], o.name, o.alias['length'])) + if tname == "Using": + if "length" in o.alias: + write( + "typedef %s vl_api_%s_t[%s];\n" + % (o.alias["type"], o.name, o.alias["length"]) + ) else: - write('typedef %s vl_api_%s_t;\n' % (o.alias['type'], o.name)) - elif tname == 'Enum' or tname == 'EnumFlag': - if o.enumtype == 'u32': + write("typedef %s vl_api_%s_t;\n" % (o.alias["type"], o.name)) + elif tname == "Enum" or tname == "EnumFlag": + if o.enumtype == "u32": write("typedef enum {\n") else: write("typedef enum __attribute__((packed)) {\n") for b in o.block: write(" %s = %s,\n" % (b[0], b[1])) - write('} vl_api_%s_t;\n' % o.name) - if o.enumtype != 'u32': - size1 = 'sizeof(vl_api_%s_t)' % o.name - size2 = 'sizeof(%s)' % o.enumtype - err_str = 'size of API enum %s is wrong' % o.name - write('STATIC_ASSERT(%s == %s, "%s");\n' - % (size1, size2, err_str)) + write("} vl_api_%s_t;\n" % o.name) + if o.enumtype != "u32": + size1 = "sizeof(vl_api_%s_t)" % o.name + size2 = "sizeof(%s)" % o.enumtype + err_str = "size of API enum %s is wrong" % o.name + write('STATIC_ASSERT(%s == %s, "%s");\n' % (size1, size2, err_str)) else: - if tname == 'Union': - write("typedef union __attribute__ ((packed)) _vl_api_%s {\n" - % o.name) + if tname == "Union": + write("typedef union __attribute__ ((packed)) _vl_api_%s {\n" % o.name) else: - write(("typedef struct __attribute__ ((packed)) _vl_api_%s {\n") - % o.name) + write( + ("typedef struct __attribute__ ((packed)) _vl_api_%s {\n") % o.name + ) for b in o.block: - if b.type == 'Option': + if b.type == "Option": continue - if b.type == 'Field': - write(" %s %s;\n" % (api2c(b.fieldtype), - b.fieldname)) - elif b.type == 'Array': + if b.type == "Field": + write(" %s %s;\n" % (api2c(b.fieldtype), b.fieldname)) + elif b.type == "Array": if b.lengthfield: - write(" %s %s[0];\n" % (api2c(b.fieldtype), - b.fieldname)) + write(" %s %s[0];\n" % (api2c(b.fieldtype), b.fieldname)) else: # Fixed length strings decay to nul terminated u8 - if b.fieldtype == 'string': + if b.fieldtype == "string": if b.modern_vla: - write(' {} {};\n' - .format(api2c(b.fieldtype), - b.fieldname)) + write( + " {} {};\n".format( + api2c(b.fieldtype), b.fieldname + ) + ) else: - write(' u8 {}[{}];\n' - .format(b.fieldname, b.length)) + write(" u8 {}[{}];\n".format(b.fieldname, b.length)) else: - write(" %s %s[%s];\n" % - (api2c(b.fieldtype), b.fieldname, - b.length)) + write( + " %s %s[%s];\n" + % (api2c(b.fieldtype), b.fieldname, b.length) + ) else: - raise ValueError("Error in processing type {} for {}" - .format(b, o.name)) - - write('} vl_api_%s_t;\n' % o.name) - - for t in s['Define']: - write('#define VL_API_{ID}_CRC "{n}_{crc:08x}"\n' - .format(n=t.name, ID=t.name.upper(), crc=t.crc)) + raise ValueError( + "Error in processing type {} for {}".format(b, o.name) + ) + + write("} vl_api_%s_t;\n" % o.name) + write( + f"#define VL_API_{o.name.upper()}_IS_CONSTANT_SIZE ({0 if o.vla else 1})\n\n" + ) + + for t in s["Define"]: + write( + '#define VL_API_{ID}_CRC "{n}_{crc:08x}"\n'.format( + n=t.name, ID=t.name.upper(), crc=t.crc + ) + ) write("\n#endif\n") -def generate_c_boilerplate(services, defines, counters, file_crc, - module, stream): - '''VPP side plugin.''' +def generate_c_boilerplate(services, defines, counters, file_crc, module, stream): + """VPP side plugin.""" write = stream.write define_hash = {d.name: d for d in defines} - hdr = '''\ + hdr = """\ #define vl_endianfun /* define message structures */ #include "{module}.api.h" #undef vl_endianfun +#define vl_calcsizefun +#include "{module}.api.h" +#undef vl_calsizefun + /* instantiate all the print functions we know about */ -#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) #define vl_printfun #include "{module}.api.h" #undef vl_printfun -''' +#include "{module}.api_json.h" +""" write(hdr.format(module=module)) - write('static u16\n') - write('setup_message_id_table (void) {\n') - write(' api_main_t *am = my_api_main;\n') - write(' vl_msg_api_msg_config_t c;\n') - write(' u16 msg_id_base = vl_msg_api_get_msg_ids ("{}_{crc:08x}", ' - 'VL_MSG_{m}_LAST);\n' - .format(module, crc=file_crc, m=module.upper())) + if len(defines) > 0: + write("static u16\n") + write("setup_message_id_table (void) {\n") + write(" api_main_t *am = my_api_main;\n") + write(" vl_msg_api_msg_config_t c;\n") + write( + ' u16 msg_id_base = vl_msg_api_get_msg_ids ("{}_{crc:08x}", ' + "VL_MSG_{m}_LAST);\n".format(module, crc=file_crc, m=module.upper()) + ) + write(f" vec_add1(am->json_api_repr, (u8 *)json_api_repr_{module});\n") for d in defines: - write(' vl_msg_api_add_msg_name_crc (am, "{n}_{crc:08x}",\n' - ' VL_API_{ID} + msg_id_base);\n' - .format(n=d.name, ID=d.name.upper(), crc=d.crc)) + write( + ' vl_msg_api_add_msg_name_crc (am, "{n}_{crc:08x}",\n' + " VL_API_{ID} + msg_id_base);\n".format( + n=d.name, ID=d.name.upper(), crc=d.crc + ) + ) for s in services: d = define_hash[s.caller] - write(' c = (vl_msg_api_msg_config_t) ' - ' {{.id = VL_API_{ID} + msg_id_base,\n' - ' .name = "{n}",\n' - ' .handler = vl_api_{n}_t_handler,\n' - ' .cleanup = vl_noop_handler,\n' - ' .endian = vl_api_{n}_t_endian,\n' - ' .print = vl_api_{n}_t_print,\n' - ' .traced = 1,\n' - ' .replay = 1,\n' - ' .is_autoendian = {auto}}};\n' - .format(n=s.caller, ID=s.caller.upper(), - auto=d.autoendian)) - write(' vl_msg_api_config (&c);\n') + write( + " c = (vl_msg_api_msg_config_t) " + " {{.id = VL_API_{ID} + msg_id_base,\n" + ' .name = "{n}",\n' + " .handler = vl_api_{n}_t_handler,\n" + " .endian = vl_api_{n}_t_endian,\n" + " .format_fn = vl_api_{n}_t_format,\n" + " .traced = 1,\n" + " .replay = 1,\n" + " .tojson = vl_api_{n}_t_tojson,\n" + " .fromjson = vl_api_{n}_t_fromjson,\n" + " .calc_size = vl_api_{n}_t_calc_size,\n" + " .is_autoendian = {auto}}};\n".format( + n=s.caller, ID=s.caller.upper(), auto=d.autoendian + ) + ) + write(" vl_msg_api_config (&c);\n") try: d = define_hash[s.reply] - write(' c = (vl_msg_api_msg_config_t) ' - '{{.id = VL_API_{ID} + msg_id_base,\n' - ' .name = "{n}",\n' - ' .handler = 0,\n' - ' .cleanup = vl_noop_handler,\n' - ' .endian = vl_api_{n}_t_endian,\n' - ' .print = vl_api_{n}_t_print,\n' - ' .is_autoendian = {auto}}};\n' - .format(n=s.reply, ID=s.reply.upper(), - auto=d.autoendian)) - write(' vl_msg_api_config (&c);\n') + write( + " c = (vl_msg_api_msg_config_t) " + "{{.id = VL_API_{ID} + msg_id_base,\n" + ' .name = "{n}",\n' + " .handler = 0,\n" + " .endian = vl_api_{n}_t_endian,\n" + " .format_fn = vl_api_{n}_t_format,\n" + " .traced = 1,\n" + " .replay = 1,\n" + " .tojson = vl_api_{n}_t_tojson,\n" + " .fromjson = vl_api_{n}_t_fromjson,\n" + " .calc_size = vl_api_{n}_t_calc_size,\n" + " .is_autoendian = {auto}}};\n".format( + n=s.reply, ID=s.reply.upper(), auto=d.autoendian + ) + ) + write(" vl_msg_api_config (&c);\n") except KeyError: pass - write(' return msg_id_base;\n') - write('}\n') + try: + if s.stream: + d = define_hash[s.stream_message] + write( + " c = (vl_msg_api_msg_config_t) " + "{{.id = VL_API_{ID} + msg_id_base,\n" + ' .name = "{n}",\n' + " .handler = 0,\n" + " .endian = vl_api_{n}_t_endian,\n" + " .format_fn = vl_api_{n}_t_format,\n" + " .traced = 1,\n" + " .replay = 1,\n" + " .tojson = vl_api_{n}_t_tojson,\n" + " .fromjson = vl_api_{n}_t_fromjson,\n" + " .calc_size = vl_api_{n}_t_calc_size,\n" + " .is_autoendian = {auto}}};\n".format( + n=s.stream_message, + ID=s.stream_message.upper(), + auto=d.autoendian, + ) + ) + write(" vl_msg_api_config (&c);\n") + except KeyError: + pass + if len(defines) > 0: + write(" return msg_id_base;\n") + write("}\n") - severity = {'error': 'VL_COUNTER_SEVERITY_ERROR', - 'info': 'VL_COUNTER_SEVERITY_INFO', - 'warn': 'VL_COUNTER_SEVERITY_WARN'} + severity = { + "error": "VL_COUNTER_SEVERITY_ERROR", + "info": "VL_COUNTER_SEVERITY_INFO", + "warn": "VL_COUNTER_SEVERITY_WARN", + } for cnt in counters: csetname = cnt.name - write('vlib_error_desc_t {}_error_counters[] = {{\n'.format(csetname)) + write("vlib_error_desc_t {}_error_counters[] = {{\n".format(csetname)) for c in cnt.block: - write(' {\n') - write(' .name = "{}",\n'.format(c['name'])) - write(' .desc = "{}",\n'.format(c['description'])) - write(' .severity = {},\n'.format(severity[c['severity']])) - write(' },\n') - write('};\n') + write(" {\n") + write(' .name = "{}",\n'.format(c["name"])) + write(' .desc = "{}",\n'.format(c["description"])) + write(" .severity = {},\n".format(severity[c["severity"]])) + write(" },\n") + write("};\n") -def generate_c_test_boilerplate(services, defines, file_crc, module, plugin, - stream): - '''Generate code for legacy style VAT. To be deleted.''' +def generate_c_test_boilerplate(services, defines, file_crc, module, plugin, stream): + """Generate code for legacy style VAT. To be deleted.""" write = stream.write define_hash = {d.name: d for d in defines} - hdr = '''\ + hdr = """\ #define vl_endianfun /* define message structures */ #include "{module}.api.h" #undef vl_endianfun +#define vl_calcsizefun +#include "{module}.api.h" +#undef vl_calsizefun + /* instantiate all the print functions we know about */ -#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) #define vl_printfun #include "{module}.api.h" #undef vl_printfun -''' +""" write(hdr.format(module=module)) for s in services: @@ -1412,109 +1686,133 @@ def generate_c_test_boilerplate(services, defines, file_crc, module, plugin, except KeyError: continue if d.manual_print: - write('/*\n' - ' * Manual definition requested for: \n' - ' * vl_api_{n}_t_handler()\n' - ' */\n' - .format(n=s.reply)) + write( + "/*\n" + " * Manual definition requested for: \n" + " * vl_api_{n}_t_handler()\n" + " */\n".format(n=s.reply) + ) continue if not define_hash[s.caller].autoreply: - write('/* Generation not supported (vl_api_{n}_t_handler()) */\n' - .format(n=s.reply)) + write( + "/* Generation not supported (vl_api_{n}_t_handler()) */\n".format( + n=s.reply + ) + ) continue - write('#ifndef VL_API_{n}_T_HANDLER\n'.format(n=s.reply.upper())) - write('static void\n') - write('vl_api_{n}_t_handler (vl_api_{n}_t * mp) {{\n' - .format(n=s.reply)) - write(' vat_main_t * vam = {}_test_main.vat_main;\n'.format(module)) - write(' i32 retval = ntohl(mp->retval);\n') - write(' if (vam->async_mode) {\n') - write(' vam->async_errors += (retval < 0);\n') - write(' } else {\n') - write(' vam->retval = retval;\n') - write(' vam->result_ready = 1;\n') - write(' }\n') - write('}\n') - write('#endif\n') + write("#ifndef VL_API_{n}_T_HANDLER\n".format(n=s.reply.upper())) + write("static void\n") + write("vl_api_{n}_t_handler (vl_api_{n}_t * mp) {{\n".format(n=s.reply)) + write(" vat_main_t * vam = {}_test_main.vat_main;\n".format(module)) + write(" i32 retval = ntohl(mp->retval);\n") + write(" if (vam->async_mode) {\n") + write(" vam->async_errors += (retval < 0);\n") + write(" } else {\n") + write(" vam->retval = retval;\n") + write(" vam->result_ready = 1;\n") + write(" }\n") + write("}\n") + write("#endif\n") for e in s.events: if define_hash[e].manual_print: continue - write('static void\n') - write('vl_api_{n}_t_handler (vl_api_{n}_t * mp) {{\n'.format(n=e)) - write(' vl_print(0, "{n} event called:");\n'.format(n=e)) - write(' vl_api_{n}_t_print(mp, 0);\n'.format(n=e)) - write('}\n') - - write('static void\n') - write('setup_message_id_table (vat_main_t * vam, u16 msg_id_base) {\n') + write("static void\n") + write("vl_api_{n}_t_handler (vl_api_{n}_t * mp) {{\n".format(n=e)) + write(' vlib_cli_output(0, "{n} event called:");\n'.format(n=e)) + write( + ' vlib_cli_output(0, "%U", vl_api_{n}_t_format, mp);\n'.format(n=e) + ) + write("}\n") + + write("static void\n") + write("setup_message_id_table (vat_main_t * vam, u16 msg_id_base) {\n") for s in services: - write(' vl_msg_api_set_handlers(VL_API_{ID} + msg_id_base, ' - ' "{n}",\n' - ' vl_api_{n}_t_handler, ' - ' vl_noop_handler,\n' - ' vl_api_{n}_t_endian, ' - ' vl_api_{n}_t_print,\n' - ' sizeof(vl_api_{n}_t), 1);\n' - .format(n=s.reply, ID=s.reply.upper())) - write(' hash_set_mem (vam->function_by_name, "{n}", api_{n});\n' - .format(n=s.caller)) + write( + " vl_msg_api_config (&(vl_msg_api_msg_config_t){{\n" + " .id = VL_API_{ID} + msg_id_base,\n" + ' .name = "{n}",\n' + " .handler = vl_api_{n}_t_handler,\n" + " .endian = vl_api_{n}_t_endian,\n" + " .format_fn = vl_api_{n}_t_format,\n" + " .size = sizeof(vl_api_{n}_t),\n" + " .traced = 1,\n" + " .tojson = vl_api_{n}_t_tojson,\n" + " .fromjson = vl_api_{n}_t_fromjson,\n" + " .calc_size = vl_api_{n}_t_calc_size,\n" + " }});".format(n=s.reply, ID=s.reply.upper()) + ) + write( + ' hash_set_mem (vam->function_by_name, "{n}", api_{n});\n'.format( + n=s.caller + ) + ) try: - write(' hash_set_mem (vam->help_by_name, "{n}", "{help}");\n' - .format(n=s.caller, - help=define_hash[s.caller].options['vat_help'])) + write( + ' hash_set_mem (vam->help_by_name, "{n}", "{help}");\n'.format( + n=s.caller, help=define_hash[s.caller].options["vat_help"] + ) + ) except KeyError: pass # Events for e in s.events: - write(' vl_msg_api_set_handlers(VL_API_{ID} + msg_id_base, ' - ' "{n}",\n' - ' vl_api_{n}_t_handler, ' - ' vl_noop_handler,\n' - ' vl_api_{n}_t_endian, ' - ' vl_api_{n}_t_print,\n' - ' sizeof(vl_api_{n}_t), 1);\n' - .format(n=e, ID=e.upper())) - - write('}\n') - if plugin: - write('clib_error_t * vat_plugin_register (vat_main_t *vam)\n') - else: - write('clib_error_t * vat_{}_plugin_register (vat_main_t *vam)\n' - .format(module)) - write('{\n') - write(' {n}_test_main_t * mainp = &{n}_test_main;\n'.format(n=module)) - write(' mainp->vat_main = vam;\n') - write(' mainp->msg_id_base = vl_client_get_first_plugin_msg_id ' - ' ("{n}_{crc:08x}");\n' - .format(n=module, crc=file_crc)) - write(' if (mainp->msg_id_base == (u16) ~0)\n') - write(' return clib_error_return (0, "{} plugin not loaded...");\n' - .format(module)) - write(' setup_message_id_table (vam, mainp->msg_id_base);\n') - write('#ifdef VL_API_LOCAL_SETUP_MESSAGE_ID_TABLE\n') - write(' VL_API_LOCAL_SETUP_MESSAGE_ID_TABLE(vam);\n') - write('#endif\n') - write(' return 0;\n') - write('}\n') + write( + " vl_msg_api_config (&(vl_msg_api_msg_config_t){{\n" + " .id = VL_API_{ID} + msg_id_base,\n" + ' .name = "{n}",\n' + " .handler = vl_api_{n}_t_handler,\n" + " .endian = vl_api_{n}_t_endian,\n" + " .format_fn = vl_api_{n}_t_format,\n" + " .size = sizeof(vl_api_{n}_t),\n" + " .traced = 1,\n" + " .tojson = vl_api_{n}_t_tojson,\n" + " .fromjson = vl_api_{n}_t_fromjson,\n" + " .calc_size = vl_api_{n}_t_calc_size,\n" + " }});".format(n=e, ID=e.upper()) + ) + + write("}\n") + write("clib_error_t * vat_plugin_register (vat_main_t *vam)\n") + write("{\n") + write(" {n}_test_main_t * mainp = &{n}_test_main;\n".format(n=module)) + write(" mainp->vat_main = vam;\n") + write( + " mainp->msg_id_base = vl_client_get_first_plugin_msg_id " + ' ("{n}_{crc:08x}");\n'.format(n=module, crc=file_crc) + ) + write(" if (mainp->msg_id_base == (u16) ~0)\n") + write( + ' return clib_error_return (0, "{} plugin not loaded...");\n'.format( + module + ) + ) + write(" setup_message_id_table (vam, mainp->msg_id_base);\n") + write("#ifdef VL_API_LOCAL_SETUP_MESSAGE_ID_TABLE\n") + write(" VL_API_LOCAL_SETUP_MESSAGE_ID_TABLE(vam);\n") + write("#endif\n") + write(" return 0;\n") + write("}\n") def apifunc(func): - '''Check if a method is generated already.''' + """Check if a method is generated already.""" + def _f(module, d, processed, *args): if d.name in processed: return None processed[d.name] = True return func(module, d, *args) + return _f def c_test_api_service(s, dump, stream): - '''Generate JSON code for a service.''' + """Generate JSON code for a service.""" write = stream.write - req_reply_template = '''\ + req_reply_template = """\ static cJSON * api_{n} (cJSON *o) {{ @@ -1528,9 +1826,9 @@ api_{n} (cJSON *o) }} mp->_vl_msg_id = vac_get_msg_index(VL_API_{N}_CRC); - vl_api_{n}_t_endian(mp); + vl_api_{n}_t_endian(mp, 1); vac_write((char *)mp, len); - free(mp); + cJSON_free(mp); /* Read reply */ char *p; @@ -1543,12 +1841,12 @@ api_{n} (cJSON *o) return 0; }} vl_api_{r}_t *rmp = (vl_api_{r}_t *)p; - vl_api_{r}_t_endian(rmp); + vl_api_{r}_t_endian(rmp, 0); return vl_api_{r}_t_tojson(rmp); }} -''' - dump_details_template = '''\ +""" + dump_details_template = """\ static cJSON * api_{n} (cJSON *o) {{ @@ -1561,9 +1859,9 @@ api_{n} (cJSON *o) return 0; }} mp->_vl_msg_id = msg_id; - vl_api_{n}_t_endian(mp); + vl_api_{n}_t_endian(mp, 1); vac_write((char *)mp, len); - free(mp); + cJSON_free(mp); vat2_control_ping(123); // FIX CONTEXT cJSON *reply = cJSON_CreateArray(); @@ -1595,15 +1893,15 @@ api_{n} (cJSON *o) return 0; }} vl_api_{r}_t *rmp = (vl_api_{r}_t *)p; - vl_api_{r}_t_endian(rmp); + vl_api_{r}_t_endian(rmp, 0); cJSON_AddItemToArray(reply, vl_api_{r}_t_tojson(rmp)); }} }} return reply; }} -''' - gets_details_reply_template = '''\ +""" + gets_details_reply_template = """\ static cJSON * api_{n} (cJSON *o) {{ @@ -1617,9 +1915,9 @@ api_{n} (cJSON *o) }} mp->_vl_msg_id = msg_id; - vl_api_{n}_t_endian(mp); + vl_api_{n}_t_endian(mp, 1); vac_write((char *)mp, len); - free(mp); + cJSON_free(mp); cJSON *reply = cJSON_CreateArray(); @@ -1638,46 +1936,56 @@ api_{n} (cJSON *o) u16 msg_id = ntohs(*((u16 *)p)); if (msg_id == reply_msg_id) {{ vl_api_{r}_t *rmp = (vl_api_{r}_t *)p; - vl_api_{r}_t_endian(rmp); + vl_api_{r}_t_endian(rmp, 0); cJSON_AddItemToArray(reply, vl_api_{r}_t_tojson(rmp)); break; }} if (msg_id == details_msg_id) {{ vl_api_{d}_t *rmp = (vl_api_{d}_t *)p; - vl_api_{d}_t_endian(rmp); + vl_api_{d}_t_endian(rmp, 0); cJSON_AddItemToArray(reply, vl_api_{d}_t_tojson(rmp)); }} }} return reply; }} -''' +""" if dump: if s.stream_message: - write(gets_details_reply_template - .format(n=s.caller, r=s.reply, N=s.caller.upper(), - R=s.reply.upper(), d=s.stream_message, - D=s.stream_message.upper())) + write( + gets_details_reply_template.format( + n=s.caller, + r=s.reply, + N=s.caller.upper(), + R=s.reply.upper(), + d=s.stream_message, + D=s.stream_message.upper(), + ) + ) else: - write(dump_details_template.format(n=s.caller, r=s.reply, - N=s.caller.upper(), - R=s.reply.upper())) + write( + dump_details_template.format( + n=s.caller, r=s.reply, N=s.caller.upper(), R=s.reply.upper() + ) + ) else: - write(req_reply_template.format(n=s.caller, r=s.reply, - N=s.caller.upper(), - R=s.reply.upper())) + write( + req_reply_template.format( + n=s.caller, r=s.reply, N=s.caller.upper(), R=s.reply.upper() + ) + ) def generate_c_test2_boilerplate(services, defines, module, stream): - '''Generate code for VAT2 plugin.''' + """Generate code for VAT2 plugin.""" write = stream.write define_hash = {d.name: d for d in defines} # replies = {} - hdr = '''\ + hdr = """\ #include <vlibapi/api.h> #include <vlibmemory/api.h> #include <vppinfra/error.h> @@ -1686,8 +1994,8 @@ def generate_c_test2_boilerplate(services, defines, module, stream): #define vl_typedefs /* define message structures */ #include <vlibmemory/vl_memory_api_h.h> -#include <vpp/api/vpe_types.api.h> -#include <vpp/api/vpe.api.h> +#include <vlibmemory/vlib.api_types.h> +#include <vlibmemory/vlib.api.h> #undef vl_typedefs #include "{module}.api_enum.h" @@ -1697,7 +2005,10 @@ def generate_c_test2_boilerplate(services, defines, module, stream): #include "{module}.api.h" #undef vl_endianfun -#define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) +#define vl_calcsizefun +#include "{module}.api.h" +#undef vl_calsizefun + #define vl_printfun #include "{module}.api.h" #undef vl_printfun @@ -1708,7 +2019,7 @@ def generate_c_test2_boilerplate(services, defines, module, stream): #include <vat2/vat2_helpers.h> -''' +""" write(hdr.format(module=module)) @@ -1717,119 +2028,123 @@ def generate_c_test2_boilerplate(services, defines, module, stream): continue c_test_api_service(s, s.stream, stream) - write('void vat2_register_function(char *, cJSON * (*)(cJSON *));\n') + write( + "void vat2_register_function(char *, cJSON * (*)(cJSON *), cJSON * (*)(void *), u32);\n" + ) # write('__attribute__((constructor))') - write('clib_error_t *\n') - write('vat2_register_plugin (void) {\n') + write("clib_error_t *\n") + write("vat2_register_plugin (void) {\n") for s in services: - write(' vat2_register_function("{n}", api_{n});\n' - .format(n=s.caller)) - write(' return 0;\n') - write('}\n') + if s.reply not in define_hash: + continue + crc = define_hash[s.caller].crc + write( + ' vat2_register_function("{n}", api_{n}, (cJSON * (*)(void *))vl_api_{n}_t_tojson, 0x{crc:08x});\n'.format( + n=s.caller, crc=crc + ) + ) + write(" return 0;\n") + write("}\n") # # Plugin entry point # -def run(args, apifilename, s): - '''Main plugin entry point.''' +def run(output_dir, apifilename, s): + """Main plugin entry point.""" stream = StringIO() - if not args.outputdir: - sys.stderr.write('Missing --outputdir argument') + if not output_dir: + sys.stderr.write("Missing --outputdir argument") return None basename = os.path.basename(apifilename) filename, _ = os.path.splitext(basename) - modulename = filename.replace('.', '_') - filename_enum = os.path.join(args.outputdir + '/' + basename + '_enum.h') - filename_types = os.path.join(args.outputdir + '/' + basename + '_types.h') - filename_c = os.path.join(args.outputdir + '/' + basename + '.c') - filename_c_test = os.path.join(args.outputdir + '/' + basename + '_test.c') - filename_c_test2 = (os.path.join(args.outputdir + '/' + basename + - '_test2.c')) - filename_c_tojson = (os.path.join(args.outputdir + - '/' + basename + '_tojson.h')) - filename_c_fromjson = (os.path.join(args.outputdir + '/' + - basename + '_fromjson.h')) + modulename = filename.replace(".", "_") + filename_enum = os.path.join(output_dir + "/" + basename + "_enum.h") + filename_types = os.path.join(output_dir + "/" + basename + "_types.h") + filename_c = os.path.join(output_dir + "/" + basename + ".c") + filename_c_test = os.path.join(output_dir + "/" + basename + "_test.c") + filename_c_test2 = os.path.join(output_dir + "/" + basename + "_test2.c") + filename_c_tojson = os.path.join(output_dir + "/" + basename + "_tojson.h") + filename_c_fromjson = os.path.join(output_dir + "/" + basename + "_fromjson.h") # Generate separate types file st = StringIO() generate_include_types(s, modulename, st) - with open(filename_types, 'w') as fd: + with open(filename_types, "w") as fd: st.seek(0) shutil.copyfileobj(st, fd) st.close() # Generate separate enum file st = StringIO() - st.write('#ifndef included_{}_api_enum_h\n'.format(modulename)) - st.write('#define included_{}_api_enum_h\n'.format(modulename)) + st.write("#ifndef included_{}_api_enum_h\n".format(modulename)) + st.write("#define included_{}_api_enum_h\n".format(modulename)) generate_include_enum(s, modulename, st) - generate_include_counters(s['Counters'], st) - st.write('#endif\n') - with open(filename_enum, 'w') as fd: + generate_include_counters(s["Counters"], st) + st.write("#endif\n") + with open(filename_enum, "w") as fd: st.seek(0) shutil.copyfileobj(st, fd) st.close() # Generate separate C file st = StringIO() - generate_c_boilerplate(s['Service'], s['Define'], s['Counters'], - s['file_crc'], modulename, st) - with open(filename_c, 'w') as fd: + generate_c_boilerplate( + s["Service"], s["Define"], s["Counters"], s["file_crc"], modulename, st + ) + with open(filename_c, "w") as fd: st.seek(0) shutil.copyfileobj(st, fd) st.close() # Generate separate C test file st = StringIO() - plugin = bool('plugin' in apifilename) - generate_c_test_boilerplate(s['Service'], s['Define'], - s['file_crc'], - modulename, plugin, st) - with open(filename_c_test, 'w') as fd: + plugin = bool("plugin" in apifilename) + generate_c_test_boilerplate( + s["Service"], s["Define"], s["file_crc"], modulename, plugin, st + ) + with open(filename_c_test, "w") as fd: st.seek(0) shutil.copyfileobj(st, fd) st.close() # Fully autogenerated VATv2 C test file st = StringIO() - generate_c_test2_boilerplate(s['Service'], s['Define'], - modulename, st) - with open(filename_c_test2, 'w') as fd: + generate_c_test2_boilerplate(s["Service"], s["Define"], modulename, st) + with open(filename_c_test2, "w") as fd: st.seek(0) shutil.copyfileobj(st, fd) - st.close() # + st.close() # # Generate separate JSON file st = StringIO() generate_tojson(s, modulename, st) - with open(filename_c_tojson, 'w') as fd: + with open(filename_c_tojson, "w") as fd: st.seek(0) shutil.copyfileobj(st, fd) st.close() st = StringIO() generate_fromjson(s, modulename, st) - with open(filename_c_fromjson, 'w') as fd: + with open(filename_c_fromjson, "w") as fd: st.seek(0) shutil.copyfileobj(st, fd) st.close() - output = TOP_BOILERPLATE.format(datestring=DATESTRING, - input_filename=basename) - output += generate_imports(s['Import']) + output = TOP_BOILERPLATE.format(datestring=DATESTRING, input_filename=basename) + output += generate_imports(s["Import"]) output += msg_ids(s) output += msg_names(s) output += msg_name_crc_list(s, filename) output += typedefs(modulename) - printfun_types(s['types'], stream, modulename) - printfun(s['Define'], stream, modulename) + printfun_types(s["types"], stream, modulename) + printfun(s["Define"], stream, modulename) output += stream.getvalue() stream.close() - output += endianfun(s['types'] + s['Define'], modulename) + output += endianfun(s["types"] + s["Define"], modulename) + output += calc_size_fun(s["types"] + s["Define"], modulename) output += version_tuple(s, basename) - output += BOTTOM_BOILERPLATE.format(input_filename=basename, - file_crc=s['file_crc']) + output += BOTTOM_BOILERPLATE.format(input_filename=basename, file_crc=s["file_crc"]) return output diff --git a/src/tools/vppapigen/vppapigen_crc.py b/src/tools/vppapigen/vppapigen_crc.py index 791e347292e..f7e8296af3e 100644 --- a/src/tools/vppapigen/vppapigen_crc.py +++ b/src/tools/vppapigen/vppapigen_crc.py @@ -7,16 +7,15 @@ process_imports = True # # Plugin entry point # -def run(args, input_filename, s): +def run(output_dir, input_filename, s): j = {} major = 0 minor = 0 patch = 0 - if 'version' in s['Option']: - v = s['Option']['version'] - (major, minor, patch) = v.split('.') - j['_version'] = {'major': major, 'minor': minor, 'patch': patch} - for t in s['Define']: - j[t.name] = {'crc': f'{t.crc:#08x}', 'version': major, - 'options': t.options} - return json.dumps(j, indent=4, separators=(',', ': ')) + if "version" in s["Option"]: + v = s["Option"]["version"] + (major, minor, patch) = v.split(".") + j["_version"] = {"major": major, "minor": minor, "patch": patch} + for t in s["Define"]: + j[t.name] = {"crc": f"{t.crc:#08x}", "version": major, "options": t.options} + return json.dumps(j, indent=4, separators=(",", ": ")) diff --git a/src/tools/vppapigen/vppapigen_json.py b/src/tools/vppapigen/vppapigen_json.py index 5fa839f9854..7239d1ea732 100644 --- a/src/tools/vppapigen/vppapigen_json.py +++ b/src/tools/vppapigen/vppapigen_json.py @@ -1,5 +1,7 @@ # JSON generation import json +import sys +import os process_imports = True @@ -14,7 +16,7 @@ def walk_imports(s): def walk_counters(s, pathset): r = [] for e in s: - r2 = {'name': e.name, 'elements': e.block} + r2 = {"name": e.name, "elements": e.block} r.append(r2) r3 = [] @@ -31,7 +33,7 @@ def walk_enums(s): d.append(e.name) for b in e.block: d.append(b) - d.append({'enumtype': e.enumtype}) + d.append({"enumtype": e.enumtype}) r.append(d) return r @@ -39,13 +41,13 @@ def walk_enums(s): def walk_services(s): r = {} for e in s: - d = {'reply': e.reply} + d = {"reply": e.reply} if e.stream: - d['stream'] = True + d["stream"] = True if e.stream_message: - d['stream_msg'] = e.stream_message + d["stream_msg"] = e.stream_message if e.events: - d['events'] = e.events + d["events"] = e.events r[e.caller] = d return r @@ -56,28 +58,29 @@ def walk_defs(s, is_message=False): d = [] d.append(t.name) for b in t.block: - if b.type == 'Option': + if b.type == "Option": continue - if b.type == 'Field': + if b.type == "Field": if b.limit: d.append([b.fieldtype, b.fieldname, b.limit]) else: d.append([b.fieldtype, b.fieldname]) - elif b.type == 'Array': + elif b.type == "Array": if b.lengthfield: - d.append([b.fieldtype, b.fieldname, - b.length, b.lengthfield]) + d.append([b.fieldtype, b.fieldname, b.length, b.lengthfield]) else: d.append([b.fieldtype, b.fieldname, b.length]) - elif b.type == 'Union': + elif b.type == "Union": pass else: raise ValueError("Error in processing array type %s" % b) if is_message and t.crc: c = {} - c['crc'] = "{0:#0{1}x}".format(t.crc, 10) - c['options'] = t.options + c["crc"] = "{0:#0{1}x}".format(t.crc, 10) + c["options"] = t.options + if t.comment: + c["comment"] = t.comment d.append(c) r.append(d) @@ -87,22 +90,46 @@ def walk_defs(s, is_message=False): # # Plugin entry point # -def run(args, filename, s): + + +def contents_to_c_string(contents): + # Escape backslashes and double quotes + contents = contents.replace("\\", "\\\\").replace('"', '\\"') + # Replace newlines with \n + contents = contents.replace("\n", "\\n") + return '"' + contents + '"' + + +def run(output_dir, apifilename, s): + if not output_dir: + sys.stderr.write("Missing --outputdir argument") + return None + + basename = os.path.basename(apifilename) + filename_json_repr = os.path.join(output_dir + "/" + basename + "_json.h") + filename, _ = os.path.splitext(basename) + modulename = filename.replace(".", "_") + j = {} - j['types'] = (walk_defs([o for o in s['types'] - if o.__class__.__name__ == 'Typedef'])) - j['messages'] = walk_defs(s['Define'], True) - j['unions'] = (walk_defs([o for o in s['types'] - if o.__class__.__name__ == 'Union'])) - j['enums'] = (walk_enums([o for o in s['types'] - if o.__class__.__name__ == 'Enum'])) - j['enumflags'] = (walk_enums([o for o in s['types'] - if o.__class__.__name__ == 'EnumFlag'])) - j['services'] = walk_services(s['Service']) - j['options'] = s['Option'] - j['aliases'] = {o.name:o.alias for o in s['types'] if o.__class__.__name__ == 'Using'} - j['vl_api_version'] = hex(s['file_crc']) - j['imports'] = walk_imports(i for i in s['Import']) - j['counters'], j['paths'] = walk_counters(s['Counters'], s['Paths']) - return json.dumps(j, indent=4, separators=(',', ': ')) + j["types"] = walk_defs([o for o in s["types"] if o.__class__.__name__ == "Typedef"]) + j["messages"] = walk_defs(s["Define"], True) + j["unions"] = walk_defs([o for o in s["types"] if o.__class__.__name__ == "Union"]) + j["enums"] = walk_enums([o for o in s["types"] if o.__class__.__name__ == "Enum"]) + j["enumflags"] = walk_enums( + [o for o in s["types"] if o.__class__.__name__ == "EnumFlag"] + ) + j["services"] = walk_services(s["Service"]) + j["options"] = s["Option"] + j["aliases"] = { + o.name: o.alias for o in s["types"] if o.__class__.__name__ == "Using" + } + j["vl_api_version"] = hex(s["file_crc"]) + j["imports"] = walk_imports(i for i in s["Import"]) + j["counters"], j["paths"] = walk_counters(s["Counters"], s["Paths"]) + r = json.dumps(j, indent=4, separators=(",", ": ")) + c_string = contents_to_c_string(r) + with open(filename_json_repr, "w", encoding="UTF-8") as f: + print(f"const char *json_api_repr_{modulename} = {c_string};", file=f) + # return json.dumps(j, indent=4, separators=(",", ": ")) + return r |