Refactor GoVPP

Squashed commit of the following:

commit 348930db31575e9f59b3834d9fec07411f011e05
Author: Ondrej Fabry <ofabry@cisco.com>
Date:   Wed Aug 15 11:30:13 2018 +0200

    Use debug level for log about different context

commit 9fc963c559cea67a41b85c6cdadc322fb3b1fc7c
Author: Ondrej Fabry <ofabry@cisco.com>
Date:   Wed Aug 15 11:22:03 2018 +0200

    Remove annoying logs and add env vars for debugging

commit fdc9fd624d13feadb602e0d03d58f8a44b7a565f
Author: Ondrej Fabry <ofabry@cisco.com>
Date:   Wed Aug 15 11:18:47 2018 +0200

    Fix printing unknown VPPApiError

commit 8f968be36a91de4d4a8ea17593ba314f82aa9583
Author: Ondrej Fabry <ofabry@cisco.com>
Date:   Tue Aug 14 17:25:10 2018 +0200

    Refactor entire GoVPP

    - fix some cases with inconsistent VPP messages, causing messages
      to be incorrectly identified as event or request
    - simplify API, remove direct access to internal Go channels
    - add module name with message to registration of messages
    - start watching filesystem only when vpe-api file does not exist
    - simplify code in message codec and remove unneeded parts
    - retrieve IDs of all registered messages after connect to VPP
    - define fallback for control ping in core to avoid duplicate registration
    - add SetLogLevel function to set logger level more easily
    - remove lot of unused code

commit 34dd1b7e10ef0324aa8c4e4cc42375bd6021c6cb
Author: Ondrej Fabry <ofabry@cisco.com>
Date:   Mon Aug 13 11:29:54 2018 +0200

    Rename VnetError to VPPApiError

commit c6549d6f77847a1367a12ff47fb716e2955e973a
Author: Ondrej Fabry <ofabry@cisco.com>
Date:   Mon Aug 13 10:23:43 2018 +0200

    Fix examples and regenerate binapi

commit 4612e36b416779771f5efab4fc654c2766d2cb1c
Author: Ondrej Fabry <ofabry@cisco.com>
Date:   Mon Aug 13 09:51:22 2018 +0200

    Add parsing and generation for services

commit ac9c5280c5aa27e325f327609e2364cc66f3723f
Author: Ondrej Fabry <ofabry@cisco.com>
Date:   Fri Aug 10 14:30:15 2018 +0200

    Fix exit status on error and add continue-onerror flag

commit 9b3f4ef9fc7c8c62037fa107085eae18a8725314
Author: Ondrej Fabry <ofabry@cisco.com>
Date:   Thu Aug 9 15:20:56 2018 +0200

    Return VnetError when Retval != 0

commit 8fd21a907b5e628ec4d2026215b83d15da96c297
Author: Ondrej Fabry <ofabry@cisco.com>
Date:   Thu Aug 9 14:59:05 2018 +0200

    Add all missing errors from api_errno.h

commit 08450f288d3046ebaecf40203174ae342a07f1eb
Author: Ondrej Fabry <ofabry@cisco.com>
Date:   Thu Aug 9 14:29:27 2018 +0200

    Update README

commit d8dced0728dd62243539be741868fb7d9b8de4cc
Author: Ondrej Fabry <ofabry@cisco.com>
Date:   Thu Aug 9 13:59:59 2018 +0200

    Regenerate vpe in core

commit 254da7592cdbf634cf7aa46ae36ce7bb6d4ee555
Author: Ondrej Fabry <ofabry@cisco.com>
Date:   Thu Aug 9 13:37:00 2018 +0200

    Add VnetError type for Retvals

commit 4475c1087fb53ab4c788e530bc7fef7cfc89d2cd
Author: Ondrej Fabry <ofabry@cisco.com>
Date:   Thu Aug 9 13:36:07 2018 +0200

    Add registration API

commit 892a3ea5a2c703e2f7c29331663f6a6fa706bff5
Author: Ondrej Fabry <ofabry@cisco.com>
Date:   Thu Aug 9 13:30:43 2018 +0200

    Generate registration for messages and check all IDs on connect

commit 389ed03b6e7082260281866c3449d72d72347c99
Author: Ondrej Fabry <ofabry@cisco.com>
Date:   Thu Aug 9 11:32:41 2018 +0200

    Show error for empty adapter (on Darwin/Windows)

commit ef1ea040d656ade64242432dc5f06810ed8dcde6
Author: Ondrej Fabry <ofabry@cisco.com>
Date:   Thu Aug 9 11:31:37 2018 +0200

    Improve logged info

commit d4adae3b14ed54c8d693060dd857fa9ba5ec8e06
Author: Ondrej Fabry <ofabry@cisco.com>
Date:   Thu Aug 9 11:27:48 2018 +0200

    Update examples

commit 63921e1346014701a22639a2611129563bb1eb78
Author: Ondrej Fabry <ofabry@cisco.com>
Date:   Thu Aug 9 11:02:56 2018 +0200

    Generate unions and fix some issues

    - add comments between sections
    - define structs on single line if it has no fields
    - generate unions with setters/getters for each field
    - fix detection of message type

commit 6ab3e3fa590b245898306a6ffaf32c7721eab60c
Author: Ondrej Fabry <ofabry@cisco.com>
Date:   Wed Aug 8 15:37:10 2018 +0200

    Refactor binapi-generator

    - split JSON parsing from code generation
    - parse and generate enums
    - parse unions (no generation yet)
    - change output file suffix to '.ba.go'
    - cleanup and simplify code
    - split code into files
    - add flag for debug mode

Change-Id: I58f685e0d4c7a38e9a7b6ea0a1f47792d95d7399
Signed-off-by: Ondrej Fabry <ofabry@cisco.com>

1 files changed, 547 insertions, 0 deletions

diff --git a/cmd/binapi-generator/parse.go b/cmd/binapi-generator/parse.go
new file mode 100644
index 0000000..7f7880b
--- /dev/null
+++ b/cmd/binapi-generator/parse.go
@@ -0,0 +1,547 @@
+// Copyright (c) 2018 Cisco and/or its affiliates.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"sort"
+	"strings"
+
+	"github.com/bennyscetbun/jsongo"
+)
+
+// Package represents collection of objects parsed from VPP binary API JSON data
+type Package struct {
+	APIVersion string
+	Enums      []Enum
+	Unions     []Union
+	Types      []Type
+	Messages   []Message
+	Services   []Service
+	RefMap     map[string]string
+}
+
+// MessageType represents the type of a VPP message
+type MessageType int
+
+const (
+	requestMessage MessageType = iota // VPP request message
+	replyMessage                      // VPP reply message
+	eventMessage                      // VPP event message
+	otherMessage                      // other VPP message
+)
+
+// Message represents VPP binary API message
+type Message struct {
+	Name   string
+	CRC    string
+	Fields []Field
+}
+
+// Type represents VPP binary API type
+type Type struct {
+	Name   string
+	CRC    string
+	Fields []Field
+}
+
+// Union represents VPP binary API union
+type Union struct {
+	Name   string
+	CRC    string
+	Fields []Field
+}
+
+// Field represents VPP binary API object field
+type Field struct {
+	Name     string
+	Type     string
+	Length   int
+	SizeFrom string
+}
+
+func (f *Field) IsArray() bool {
+	return f.Length > 0 || f.SizeFrom != ""
+}
+
+// Enum represents VPP binary API enum
+type Enum struct {
+	Name    string
+	Type    string
+	Entries []EnumEntry
+}
+
+// EnumEntry represents VPP binary API enum entry
+type EnumEntry struct {
+	Name  string
+	Value interface{}
+}
+
+// Service represents VPP binary API service
+type Service struct {
+	RequestType string
+	ReplyType   string
+	Stream      bool
+	Events      []string
+}
+
+func getSizeOfType(typ *Type) (size int) {
+	for _, field := range typ.Fields {
+		if n := getBinapiTypeSize(field.Type); n > 0 {
+			if field.Length > 0 {
+				size += n * field.Length
+			} else {
+				size += n
+			}
+		}
+	}
+	return size
+}
+
+func getTypeByRef(ctx *context, ref string) *Type {
+	for _, typ := range ctx.packageData.Types {
+		if ref == toApiType(typ.Name) {
+			return &typ
+		}
+	}
+	return nil
+}
+
+func getUnionSize(ctx *context, union *Union) (maxSize int) {
+	for _, field := range union.Fields {
+		if typ := getTypeByRef(ctx, field.Type); typ != nil {
+			if size := getSizeOfType(typ); size > maxSize {
+				maxSize = size
+			}
+		}
+	}
+	return
+}
+
+// toApiType returns name that is used as type reference in VPP binary API
+func toApiType(name string) string {
+	return fmt.Sprintf("vl_api_%s_t", name)
+}
+
+// parsePackage parses provided JSON data into objects prepared for code generation
+func parsePackage(ctx *context, jsonRoot *jsongo.JSONNode) (*Package, error) {
+	logf(" %s contains: %d services, %d messages, %d types, %d enums, %d unions (version: %s)",
+		ctx.packageName,
+		jsonRoot.Map("services").Len(),
+		jsonRoot.Map("messages").Len(),
+		jsonRoot.Map("types").Len(),
+		jsonRoot.Map("enums").Len(),
+		jsonRoot.Map("unions").Len(),
+		jsonRoot.Map("vl_api_version").Get(),
+	)
+
+	pkg := Package{
+		APIVersion: jsonRoot.Map("vl_api_version").Get().(string),
+		RefMap:     make(map[string]string),
+	}
+
+	// parse enums
+	enums := jsonRoot.Map("enums")
+	pkg.Enums = make([]Enum, enums.Len())
+	for i := 0; i < enums.Len(); i++ {
+		enumNode := enums.At(i)
+
+		enum, err := parseEnum(ctx, enumNode)
+		if err != nil {
+			return nil, err
+		}
+		pkg.Enums[i] = *enum
+		pkg.RefMap[toApiType(enum.Name)] = enum.Name
+	}
+
+	// parse types
+	types := jsonRoot.Map("types")
+	pkg.Types = make([]Type, types.Len())
+	for i := 0; i < types.Len(); i++ {
+		typNode := types.At(i)
+
+		typ, err := parseType(ctx, typNode)
+		if err != nil {
+			return nil, err
+		}
+		pkg.Types[i] = *typ
+		pkg.RefMap[toApiType(typ.Name)] = typ.Name
+	}
+
+	// parse unions
+	unions := jsonRoot.Map("unions")
+	pkg.Unions = make([]Union, unions.Len())
+	for i := 0; i < unions.Len(); i++ {
+		unionNode := unions.At(i)
+
+		union, err := parseUnion(ctx, unionNode)
+		if err != nil {
+			return nil, err
+		}
+		pkg.Unions[i] = *union
+		pkg.RefMap[toApiType(union.Name)] = union.Name
+	}
+
+	// parse messages
+	messages := jsonRoot.Map("messages")
+	pkg.Messages = make([]Message, messages.Len())
+	for i := 0; i < messages.Len(); i++ {
+		msgNode := messages.At(i)
+
+		msg, err := parseMessage(ctx, msgNode)
+		if err != nil {
+			return nil, err
+		}
+		pkg.Messages[i] = *msg
+	}
+
+	// parse services
+	services := jsonRoot.Map("services")
+	if services.GetType() == jsongo.TypeMap {
+		pkg.Services = make([]Service, services.Len())
+		for i, key := range services.GetKeys() {
+			svcNode := services.At(key)
+
+			svc, err := parseService(ctx, key.(string), svcNode)
+			if err != nil {
+				return nil, err
+			}
+			pkg.Services[i] = *svc
+		}
+
+		// sort services
+		sort.Slice(pkg.Services, func(i, j int) bool {
+			// dumps first
+			if pkg.Services[i].Stream != pkg.Services[j].Stream {
+				return pkg.Services[i].Stream
+			}
+			return pkg.Services[i].RequestType < pkg.Services[j].RequestType
+		})
+	}
+
+	printPackage(&pkg)
+
+	return &pkg, nil
+}
+
+// printPackage prints all loaded objects for package
+func printPackage(pkg *Package) {
+	if len(pkg.Enums) > 0 {
+		logf("loaded %d enums:", len(pkg.Enums))
+		for k, enum := range pkg.Enums {
+			logf(" - enum #%d\t%+v", k, enum)
+		}
+	}
+	if len(pkg.Unions) > 0 {
+		logf("loaded %d unions:", len(pkg.Unions))
+		for k, union := range pkg.Unions {
+			logf(" - union #%d\t%+v", k, union)
+		}
+	}
+	if len(pkg.Types) > 0 {
+		logf("loaded %d types:", len(pkg.Types))
+		for _, typ := range pkg.Types {
+			logf(" - type: %q (%d fields)", typ.Name, len(typ.Fields))
+		}
+	}
+	if len(pkg.Messages) > 0 {
+		logf("loaded %d messages:", len(pkg.Messages))
+		for _, msg := range pkg.Messages {
+			logf(" - message: %q (%d fields)", msg.Name, len(msg.Fields))
+		}
+	}
+	if len(pkg.Services) > 0 {
+		logf("loaded %d services:", len(pkg.Services))
+		for _, svc := range pkg.Services {
+			var info string
+			if svc.Stream {
+				info = "(STREAM)"
+			} else if len(svc.Events) > 0 {
+				info = fmt.Sprintf("(EVENTS: %v)", svc.Events)
+			}
+			logf(" - service: %q -> %q %s", svc.RequestType, svc.ReplyType, info)
+		}
+	}
+}
+
+// parseEnum parses VPP binary API enum object from JSON node
+func parseEnum(ctx *context, enumNode *jsongo.JSONNode) (*Enum, error) {
+	if enumNode.Len() == 0 || enumNode.At(0).GetType() != jsongo.TypeValue {
+		return nil, errors.New("invalid JSON for enum specified")
+	}
+
+	enumName, ok := enumNode.At(0).Get().(string)
+	if !ok {
+		return nil, fmt.Errorf("enum name is %T, not a string", enumNode.At(0).Get())
+	}
+	enumType, ok := enumNode.At(enumNode.Len() - 1).At("enumtype").Get().(string)
+	if !ok {
+		return nil, fmt.Errorf("enum type invalid or missing")
+	}
+
+	enum := Enum{
+		Name: enumName,
+		Type: enumType,
+	}
+
+	// loop through enum entries, skip first (name) and last (enumtype)
+	for j := 1; j < enumNode.Len()-1; j++ {
+		if enumNode.At(j).GetType() == jsongo.TypeArray {
+			entry := enumNode.At(j)
+
+			if entry.Len() < 2 || entry.At(0).GetType() != jsongo.TypeValue || entry.At(1).GetType() != jsongo.TypeValue {
+				return nil, errors.New("invalid JSON for enum entry specified")
+			}
+
+			entryName, ok := entry.At(0).Get().(string)
+			if !ok {
+				return nil, fmt.Errorf("enum entry name is %T, not a string", entry.At(0).Get())
+			}
+			entryVal := entry.At(1).Get()
+
+			enum.Entries = append(enum.Entries, EnumEntry{
+				Name:  entryName,
+				Value: entryVal,
+			})
+		}
+	}
+
+	return &enum, nil
+}
+
+// parseUnion parses VPP binary API union object from JSON node
+func parseUnion(ctx *context, unionNode *jsongo.JSONNode) (*Union, error) {
+	if unionNode.Len() == 0 || unionNode.At(0).GetType() != jsongo.TypeValue {
+		return nil, errors.New("invalid JSON for union specified")
+	}
+
+	unionName, ok := unionNode.At(0).Get().(string)
+	if !ok {
+		return nil, fmt.Errorf("union name is %T, not a string", unionNode.At(0).Get())
+	}
+	unionCRC, ok := unionNode.At(unionNode.Len() - 1).At("crc").Get().(string)
+	if !ok {
+		return nil, fmt.Errorf("union crc invalid or missing")
+	}
+
+	union := Union{
+		Name: unionName,
+		CRC:  unionCRC,
+	}
+
+	// loop through union fields, skip first (name) and last (crc)
+	for j := 1; j < unionNode.Len()-1; j++ {
+		if unionNode.At(j).GetType() == jsongo.TypeArray {
+			fieldNode := unionNode.At(j)
+
+			field, err := parseField(ctx, fieldNode)
+			if err != nil {
+				return nil, err
+			}
+
+			union.Fields = append(union.Fields, *field)
+		}
+	}
+
+	return &union, nil
+}
+
+// parseType parses VPP binary API type object from JSON node
+func parseType(ctx *context, typeNode *jsongo.JSONNode) (*Type, error) {
+	if typeNode.Len() == 0 || typeNode.At(0).GetType() != jsongo.TypeValue {
+		return nil, errors.New("invalid JSON for type specified")
+	}
+
+	typeName, ok := typeNode.At(0).Get().(string)
+	if !ok {
+		return nil, fmt.Errorf("type name is %T, not a string", typeNode.At(0).Get())
+	}
+	typeCRC, ok := typeNode.At(typeNode.Len() - 1).At("crc").Get().(string)
+	if !ok {
+		return nil, fmt.Errorf("type crc invalid or missing")
+	}
+
+	typ := Type{
+		Name: typeName,
+		CRC:  typeCRC,
+	}
+
+	// loop through type fields, skip first (name) and last (crc)
+	for j := 1; j < typeNode.Len()-1; j++ {
+		if typeNode.At(j).GetType() == jsongo.TypeArray {
+			fieldNode := typeNode.At(j)
+
+			field, err := parseField(ctx, fieldNode)
+			if err != nil {
+				return nil, err
+			}
+
+			typ.Fields = append(typ.Fields, *field)
+		}
+	}
+
+	return &typ, nil
+}
+
+// parseMessage parses VPP binary API message object from JSON node
+func parseMessage(ctx *context, msgNode *jsongo.JSONNode) (*Message, error) {
+	if msgNode.Len() == 0 || msgNode.At(0).GetType() != jsongo.TypeValue {
+		return nil, errors.New("invalid JSON for message specified")
+	}
+
+	msgName, ok := msgNode.At(0).Get().(string)
+	if !ok {
+		return nil, fmt.Errorf("message name is %T, not a string", msgNode.At(0).Get())
+	}
+	msgCRC, ok := msgNode.At(msgNode.Len() - 1).At("crc").Get().(string)
+	if !ok {
+		return nil, fmt.Errorf("message crc invalid or missing")
+	}
+
+	msg := Message{
+		Name: msgName,
+		CRC:  msgCRC,
+	}
+
+	// loop through message fields, skip first (name) and last (crc)
+	for j := 1; j < msgNode.Len()-1; j++ {
+		if msgNode.At(j).GetType() == jsongo.TypeArray {
+			fieldNode := msgNode.At(j)
+
+			field, err := parseField(ctx, fieldNode)
+			if err != nil {
+				return nil, err
+			}
+
+			msg.Fields = append(msg.Fields, *field)
+		}
+	}
+
+	return &msg, nil
+}
+
+// parseField parses VPP binary API object field from JSON node
+func parseField(ctx *context, field *jsongo.JSONNode) (*Field, error) {
+	if field.Len() < 2 || field.At(0).GetType() != jsongo.TypeValue || field.At(1).GetType() != jsongo.TypeValue {
+		return nil, errors.New("invalid JSON for field specified")
+	}
+
+	fieldType, ok := field.At(0).Get().(string)
+	if !ok {
+		return nil, fmt.Errorf("field type is %T, not a string", field.At(0).Get())
+	}
+	fieldName, ok := field.At(1).Get().(string)
+	if !ok {
+		return nil, fmt.Errorf("field name is %T, not a string", field.At(1).Get())
+	}
+	var fieldLength float64
+	if field.Len() >= 3 {
+		fieldLength, ok = field.At(2).Get().(float64)
+		if !ok {
+			return nil, fmt.Errorf("field length is %T, not an int", field.At(2).Get())
+		}
+	}
+	var fieldLengthFrom string
+	if field.Len() >= 4 {
+		fieldLengthFrom, ok = field.At(3).Get().(string)
+		if !ok {
+			return nil, fmt.Errorf("field length from is %T, not a string", field.At(3).Get())
+		}
+	}
+
+	return &Field{
+		Name:     fieldName,
+		Type:     fieldType,
+		Length:   int(fieldLength),
+		SizeFrom: fieldLengthFrom,
+	}, nil
+}
+
+// parseService parses VPP binary API service object from JSON node
+func parseService(ctx *context, svcName string, svcNode *jsongo.JSONNode) (*Service, error) {
+	if svcNode.Len() == 0 || svcNode.At("reply").GetType() != jsongo.TypeValue {
+		return nil, errors.New("invalid JSON for service specified")
+	}
+
+	svc := Service{
+		RequestType: svcName,
+	}
+
+	if replyNode := svcNode.At("reply"); replyNode.GetType() == jsongo.TypeValue {
+		reply, ok := replyNode.Get().(string)
+		if !ok {
+			return nil, fmt.Errorf("service reply is %T, not a string", replyNode.Get())
+		}
+		// some binapi messages might have `null` reply (for example: memclnt)
+		if reply != "null" {
+			svc.ReplyType = reply
+		}
+	}
+
+	// stream service (dumps)
+	if streamNode := svcNode.At("stream"); streamNode.GetType() == jsongo.TypeValue {
+		var ok bool
+		svc.Stream, ok = streamNode.Get().(bool)
+		if !ok {
+			return nil, fmt.Errorf("service stream is %T, not a string", streamNode.Get())
+		}
+	}
+
+	// events service (event subscription)
+	if eventsNode := svcNode.At("events"); eventsNode.GetType() == jsongo.TypeArray {
+		for j := 0; j < eventsNode.Len(); j++ {
+			event := eventsNode.At(j).Get().(string)
+			svc.Events = append(svc.Events, event)
+		}
+	}
+
+	// validate service
+	if svc.Stream {
+		if !strings.HasSuffix(svc.RequestType, "_dump") ||
+			!strings.HasSuffix(svc.ReplyType, "_details") {
+			fmt.Printf("Invalid STREAM SERVICE: %+v\n", svc)
+		}
+	} else if len(svc.Events) > 0 {
+		if (!strings.HasSuffix(svc.RequestType, "_events") &&
+			!strings.HasSuffix(svc.RequestType, "_stats")) ||
+			!strings.HasSuffix(svc.ReplyType, "_reply") {
+			fmt.Printf("Invalid EVENTS SERVICE: %+v\n", svc)
+		}
+	} else if svc.ReplyType != "" {
+		if !strings.HasSuffix(svc.ReplyType, "_reply") {
+			fmt.Printf("Invalid SERVICE: %+v\n", svc)
+		}
+	}
+
+	return &svc, nil
+}
+
+// convertToGoType translates the VPP binary API type into Go type
+func convertToGoType(ctx *context, binapiType string) (typ string) {
+	if t, ok := binapiTypes[binapiType]; ok {
+		// basic types
+		typ = t
+	} else if r, ok := ctx.packageData.RefMap[binapiType]; ok {
+		// specific types (enums/types/unions)
+		typ = camelCaseName(r)
+	} else {
+		// fallback type
+		log.Printf("found unknown VPP binary API type %q, using byte", binapiType)
+		typ = "byte"
+	}
+	return typ
+}