From a3bb834db727a3ac9a1ffcfeae9265e5dead851f Mon Sep 17 00:00:00 2001 From: Ondrej Fabry Date: Wed, 15 Aug 2018 12:59:25 +0200 Subject: Refactor GoVPP Squashed commit of the following: commit 348930db31575e9f59b3834d9fec07411f011e05 Author: Ondrej Fabry Date: Wed Aug 15 11:30:13 2018 +0200 Use debug level for log about different context commit 9fc963c559cea67a41b85c6cdadc322fb3b1fc7c Author: Ondrej Fabry Date: Wed Aug 15 11:22:03 2018 +0200 Remove annoying logs and add env vars for debugging commit fdc9fd624d13feadb602e0d03d58f8a44b7a565f Author: Ondrej Fabry Date: Wed Aug 15 11:18:47 2018 +0200 Fix printing unknown VPPApiError commit 8f968be36a91de4d4a8ea17593ba314f82aa9583 Author: Ondrej Fabry Date: Tue Aug 14 17:25:10 2018 +0200 Refactor entire GoVPP - fix some cases with inconsistent VPP messages, causing messages to be incorrectly identified as event or request - simplify API, remove direct access to internal Go channels - add module name with message to registration of messages - start watching filesystem only when vpe-api file does not exist - simplify code in message codec and remove unneeded parts - retrieve IDs of all registered messages after connect to VPP - define fallback for control ping in core to avoid duplicate registration - add SetLogLevel function to set logger level more easily - remove lot of unused code commit 34dd1b7e10ef0324aa8c4e4cc42375bd6021c6cb Author: Ondrej Fabry Date: Mon Aug 13 11:29:54 2018 +0200 Rename VnetError to VPPApiError commit c6549d6f77847a1367a12ff47fb716e2955e973a Author: Ondrej Fabry Date: Mon Aug 13 10:23:43 2018 +0200 Fix examples and regenerate binapi commit 4612e36b416779771f5efab4fc654c2766d2cb1c Author: Ondrej Fabry Date: Mon Aug 13 09:51:22 2018 +0200 Add parsing and generation for services commit ac9c5280c5aa27e325f327609e2364cc66f3723f Author: Ondrej Fabry Date: Fri Aug 10 14:30:15 2018 +0200 Fix exit status on error and add continue-onerror flag commit 9b3f4ef9fc7c8c62037fa107085eae18a8725314 Author: Ondrej Fabry Date: Thu Aug 9 15:20:56 2018 +0200 Return VnetError when Retval != 0 commit 8fd21a907b5e628ec4d2026215b83d15da96c297 Author: Ondrej Fabry Date: Thu Aug 9 14:59:05 2018 +0200 Add all missing errors from api_errno.h commit 08450f288d3046ebaecf40203174ae342a07f1eb Author: Ondrej Fabry Date: Thu Aug 9 14:29:27 2018 +0200 Update README commit d8dced0728dd62243539be741868fb7d9b8de4cc Author: Ondrej Fabry Date: Thu Aug 9 13:59:59 2018 +0200 Regenerate vpe in core commit 254da7592cdbf634cf7aa46ae36ce7bb6d4ee555 Author: Ondrej Fabry Date: Thu Aug 9 13:37:00 2018 +0200 Add VnetError type for Retvals commit 4475c1087fb53ab4c788e530bc7fef7cfc89d2cd Author: Ondrej Fabry Date: Thu Aug 9 13:36:07 2018 +0200 Add registration API commit 892a3ea5a2c703e2f7c29331663f6a6fa706bff5 Author: Ondrej Fabry Date: Thu Aug 9 13:30:43 2018 +0200 Generate registration for messages and check all IDs on connect commit 389ed03b6e7082260281866c3449d72d72347c99 Author: Ondrej Fabry Date: Thu Aug 9 11:32:41 2018 +0200 Show error for empty adapter (on Darwin/Windows) commit ef1ea040d656ade64242432dc5f06810ed8dcde6 Author: Ondrej Fabry Date: Thu Aug 9 11:31:37 2018 +0200 Improve logged info commit d4adae3b14ed54c8d693060dd857fa9ba5ec8e06 Author: Ondrej Fabry Date: Thu Aug 9 11:27:48 2018 +0200 Update examples commit 63921e1346014701a22639a2611129563bb1eb78 Author: Ondrej Fabry Date: Thu Aug 9 11:02:56 2018 +0200 Generate unions and fix some issues - add comments between sections - define structs on single line if it has no fields - generate unions with setters/getters for each field - fix detection of message type commit 6ab3e3fa590b245898306a6ffaf32c7721eab60c Author: Ondrej Fabry Date: Wed Aug 8 15:37:10 2018 +0200 Refactor binapi-generator - split JSON parsing from code generation - parse and generate enums - parse unions (no generation yet) - change output file suffix to '.ba.go' - cleanup and simplify code - split code into files - add flag for debug mode Change-Id: I58f685e0d4c7a38e9a7b6ea0a1f47792d95d7399 Signed-off-by: Ondrej Fabry --- cmd/binapi-generator/generate.go | 565 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 565 insertions(+) create mode 100644 cmd/binapi-generator/generate.go (limited to 'cmd/binapi-generator/generate.go') diff --git a/cmd/binapi-generator/generate.go b/cmd/binapi-generator/generate.go new file mode 100644 index 0000000..251d39d --- /dev/null +++ b/cmd/binapi-generator/generate.go @@ -0,0 +1,565 @@ +// Copyright (c) 2017 Cisco and/or its affiliates. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "bytes" + "fmt" + "io" + "path/filepath" + "strings" + "unicode" +) + +const ( + govppApiImportPath = "git.fd.io/govpp.git/api" // import path of the govpp API package + inputFileExt = ".api.json" // file extension of the VPP binary API files + outputFileExt = ".ba.go" // file extension of the Go generated files +) + +// context is a structure storing data for code generation +type context struct { + inputFile string // input file with VPP API in JSON + outputFile string // output file with generated Go package + + inputData []byte // contents of the input file + inputBuff *bytes.Buffer // contents of the input file currently being read + inputLine int // currently processed line in the input file + + moduleName string // name of the source VPP module + packageName string // name of the Go package being generated + + packageData *Package // parsed package data +} + +// getContext returns context details of the code generation task +func getContext(inputFile, outputDir string) (*context, error) { + if !strings.HasSuffix(inputFile, inputFileExt) { + return nil, fmt.Errorf("invalid input file name: %q", inputFile) + } + + ctx := &context{ + inputFile: inputFile, + } + + // package name + inputFileName := filepath.Base(inputFile) + ctx.moduleName = inputFileName[:strings.Index(inputFileName, ".")] + + // alter package names for modules that are reserved keywords in Go + switch ctx.moduleName { + case "interface": + ctx.packageName = "interfaces" + case "map": + ctx.packageName = "maps" + default: + ctx.packageName = ctx.moduleName + } + + // output file + packageDir := filepath.Join(outputDir, ctx.packageName) + outputFileName := ctx.packageName + outputFileExt + ctx.outputFile = filepath.Join(packageDir, outputFileName) + + return ctx, nil +} + +// generatePackage generates code for the parsed package data and writes it into w +func generatePackage(ctx *context, w *bufio.Writer) error { + logf("generating package %q", ctx.packageName) + + // generate file header + generateHeader(ctx, w) + generateImports(ctx, w) + + if *includeAPIVer { + const APIVerConstName = "VlAPIVersion" + fmt.Fprintf(w, "// %s represents version of the API.\n", APIVerConstName) + fmt.Fprintf(w, "const %s = %v\n", APIVerConstName, ctx.packageData.APIVersion) + fmt.Fprintln(w) + } + + // generate enums + if len(ctx.packageData.Enums) > 0 { + fmt.Fprintf(w, "/* Enums */\n\n") + + ctx.inputBuff = bytes.NewBuffer(ctx.inputData) + ctx.inputLine = 0 + for _, enum := range ctx.packageData.Enums { + generateEnum(ctx, w, &enum) + } + } + + // generate types + if len(ctx.packageData.Types) > 0 { + fmt.Fprintf(w, "/* Types */\n\n") + + ctx.inputBuff = bytes.NewBuffer(ctx.inputData) + ctx.inputLine = 0 + for _, typ := range ctx.packageData.Types { + generateType(ctx, w, &typ) + } + } + + // generate unions + if len(ctx.packageData.Unions) > 0 { + fmt.Fprintf(w, "/* Unions */\n\n") + + ctx.inputBuff = bytes.NewBuffer(ctx.inputData) + ctx.inputLine = 0 + for _, union := range ctx.packageData.Unions { + generateUnion(ctx, w, &union) + } + } + + // generate messages + if len(ctx.packageData.Messages) > 0 { + fmt.Fprintf(w, "/* Messages */\n\n") + + ctx.inputBuff = bytes.NewBuffer(ctx.inputData) + ctx.inputLine = 0 + for _, msg := range ctx.packageData.Messages { + generateMessage(ctx, w, &msg) + } + } + + // generate services + if len(ctx.packageData.Services) > 0 { + fmt.Fprintf(w, "/* Services */\n\n") + + fmt.Fprintf(w, "type %s interface {\n", "Services") + ctx.inputBuff = bytes.NewBuffer(ctx.inputData) + ctx.inputLine = 0 + for _, svc := range ctx.packageData.Services { + generateService(ctx, w, &svc) + } + fmt.Fprintln(w, "}") + } + + // TODO: generate implementation for Services interface + + // generate message registrations + fmt.Fprintln(w) + fmt.Fprintln(w, "func init() {") + for _, msg := range ctx.packageData.Messages { + name := camelCaseName(msg.Name) + fmt.Fprintf(w, "\tapi.RegisterMessage((*%s)(nil), \"%s\")\n", name, ctx.moduleName+"."+name) + } + fmt.Fprintln(w, "}") + + // flush the data: + if err := w.Flush(); err != nil { + return fmt.Errorf("flushing data to %s failed: %v", ctx.outputFile, err) + } + + return nil +} + +// generateHeader writes generated package header into w +func generateHeader(ctx *context, w io.Writer) { + fmt.Fprintln(w, "// Code generated by GoVPP binapi-generator. DO NOT EDIT.") + fmt.Fprintf(w, "// source: %s\n", ctx.inputFile) + fmt.Fprintln(w) + + fmt.Fprintln(w, "/*") + fmt.Fprintf(w, "Package %s is a generated VPP binary API of the '%s' VPP module.\n", ctx.packageName, ctx.moduleName) + fmt.Fprintln(w) + fmt.Fprintln(w, "It is generated from this file:") + fmt.Fprintf(w, "\t%s\n", filepath.Base(ctx.inputFile)) + fmt.Fprintln(w) + fmt.Fprintln(w, "It contains these VPP binary API objects:") + var printObjNum = func(obj string, num int) { + if num > 0 { + if num > 1 { + obj += "s" + } + fmt.Fprintf(w, "\t%d %s\n", num, obj) + } + } + printObjNum("message", len(ctx.packageData.Messages)) + printObjNum("type", len(ctx.packageData.Types)) + printObjNum("enum", len(ctx.packageData.Enums)) + printObjNum("union", len(ctx.packageData.Unions)) + printObjNum("service", len(ctx.packageData.Services)) + fmt.Fprintln(w, "*/") + fmt.Fprintf(w, "package %s\n", ctx.packageName) + fmt.Fprintln(w) +} + +// generateImports writes generated package imports into w +func generateImports(ctx *context, w io.Writer) { + fmt.Fprintf(w, "import \"%s\"\n", govppApiImportPath) + fmt.Fprintf(w, "import \"%s\"\n", "github.com/lunixbochs/struc") + fmt.Fprintf(w, "import \"%s\"\n", "bytes") + fmt.Fprintln(w) + + fmt.Fprintf(w, "// Reference imports to suppress errors if they are not otherwise used.\n") + fmt.Fprintf(w, "var _ = struc.Pack\n") + fmt.Fprintf(w, "var _ = bytes.NewBuffer\n") + fmt.Fprintln(w) +} + +// generateComment writes generated comment for the object into w +func generateComment(ctx *context, w io.Writer, goName string, vppName string, objKind string) { + fmt.Fprintf(w, "// %s represents the VPP binary API %s '%s'.\n", goName, objKind, vppName) + + var isNotSpace = func(r rune) bool { + return !unicode.IsSpace(r) + } + + // print out the source of the generated object + objFound := false + objTitle := fmt.Sprintf(`"%s",`, vppName) + var indent int + for { + line, err := ctx.inputBuff.ReadString('\n') + if err != nil { + break + } + ctx.inputLine++ + + if !objFound { + indent = strings.Index(line, objTitle) + if indent == -1 { + continue + } + // If no other non-whitespace character then we are at the message header. + if trimmed := strings.TrimSpace(line); trimmed == objTitle { + objFound = true + fmt.Fprintf(w, "// Generated from '%s', line %d:\n", filepath.Base(ctx.inputFile), ctx.inputLine) + fmt.Fprintln(w, "//") + } + } else { + if strings.IndexFunc(line, isNotSpace) < indent { + break // end of the object definition in JSON + } + } + fmt.Fprint(w, "//", line) + } + + fmt.Fprintln(w, "//") +} + +// generateEnum writes generated code for the enum into w +func generateEnum(ctx *context, w io.Writer, enum *Enum) { + name := camelCaseName(enum.Name) + typ := binapiTypes[enum.Type] + + logf(" writing enum %q (%s) with %d entries", enum.Name, name, len(enum.Entries)) + + // generate enum comment + generateComment(ctx, w, name, enum.Name, "enum") + + // generate enum definition + fmt.Fprintf(w, "type %s %s\n", name, typ) + fmt.Fprintln(w) + + fmt.Fprintln(w, "const (") + + // generate enum entries + for _, entry := range enum.Entries { + fmt.Fprintf(w, "\t%s %s = %v\n", entry.Name, name, entry.Value) + } + + fmt.Fprintln(w, ")") + + fmt.Fprintln(w) +} + +// generateType writes generated code for the type into w +func generateType(ctx *context, w io.Writer, typ *Type) { + name := camelCaseName(typ.Name) + + logf(" writing type %q (%s) with %d fields", typ.Name, name, len(typ.Fields)) + + // generate struct comment + generateComment(ctx, w, name, typ.Name, "type") + + // generate struct definition + fmt.Fprintf(w, "type %s struct {\n", name) + + // generate struct fields + for i, field := range typ.Fields { + // skip internal fields + switch strings.ToLower(field.Name) { + case "crc", "_vl_msg_id": + continue + } + + generateField(ctx, w, typ.Fields, i) + } + + // generate end of the struct + fmt.Fprintln(w, "}") + + // generate name getter + generateTypeNameGetter(w, name, typ.Name) + + // generate CRC getter + generateCrcGetter(w, name, typ.CRC) + + fmt.Fprintln(w) +} + +// generateUnion writes generated code for the union into w +func generateUnion(ctx *context, w io.Writer, union *Union) { + name := camelCaseName(union.Name) + + logf(" writing union %q (%s) with %d fields", union.Name, name, len(union.Fields)) + + // generate struct comment + generateComment(ctx, w, name, union.Name, "union") + + // generate struct definition + fmt.Fprintln(w, "type", name, "struct {") + + // maximum size for union + maxSize := getUnionSize(ctx, union) + + // generate data field + fieldName := "Union_data" + fmt.Fprintf(w, "\t%s [%d]byte\n", fieldName, maxSize) + + // generate end of the struct + fmt.Fprintln(w, "}") + + // generate name getter + generateTypeNameGetter(w, name, union.Name) + + // generate CRC getter + generateCrcGetter(w, name, union.CRC) + + // generate getters for fields + for _, field := range union.Fields { + fieldName := camelCaseName(field.Name) + fieldType := convertToGoType(ctx, field.Type) + generateUnionGetterSetter(w, name, fieldName, fieldType) + } + + // generate union methods + //generateUnionMethods(w, name) + + fmt.Fprintln(w) +} + +// generateUnionMethods generates methods that implement struc.Custom +// interface to allow having Union_data field unexported +// TODO: do more testing when unions are actually used in some messages +func generateUnionMethods(w io.Writer, structName string) { + // generate struc.Custom implementation for union + fmt.Fprintf(w, ` +func (u *%[1]s) Pack(p []byte, opt *struc.Options) (int, error) { + var b = new(bytes.Buffer) + if err := struc.PackWithOptions(b, u.union_data, opt); err != nil { + return 0, err + } + copy(p, b.Bytes()) + return b.Len(), nil +} +func (u *%[1]s) Unpack(r io.Reader, length int, opt *struc.Options) error { + return struc.UnpackWithOptions(r, u.union_data[:], opt) +} +func (u *%[1]s) Size(opt *struc.Options) int { + return len(u.union_data) +} +func (u *%[1]s) String() string { + return string(u.union_data[:]) +} +`, structName) +} + +func generateUnionGetterSetter(w io.Writer, structName string, getterField, getterStruct string) { + fmt.Fprintf(w, ` +func (u *%[1]s) Set%[2]s(a %[3]s) { + var b = new(bytes.Buffer) + if err := struc.Pack(b, &a); err != nil { + return + } + copy(u.Union_data[:], b.Bytes()) +} +func (u *%[1]s) Get%[2]s() (a %[3]s) { + var b = bytes.NewReader(u.Union_data[:]) + struc.Unpack(b, &a) + return +} +`, structName, getterField, getterStruct) +} + +// generateMessage writes generated code for the message into w +func generateMessage(ctx *context, w io.Writer, msg *Message) { + name := camelCaseName(msg.Name) + + logf(" writing message %q (%s) with %d fields", msg.Name, name, len(msg.Fields)) + + // generate struct comment + generateComment(ctx, w, name, msg.Name, "message") + + // generate struct definition + fmt.Fprintf(w, "type %s struct {", name) + + msgType := otherMessage + wasClientIndex := false + + // generate struct fields + n := 0 + for i, field := range msg.Fields { + if i == 1 { + if field.Name == "client_index" { + // "client_index" as the second member, this might be an event message or a request + msgType = eventMessage + wasClientIndex = true + } else if field.Name == "context" { + // reply needs "context" as the second member + msgType = replyMessage + } + } else if i == 2 { + if wasClientIndex && field.Name == "context" { + // request needs "client_index" as the second member and "context" as the third member + msgType = requestMessage + } + } + + // skip internal fields + switch strings.ToLower(field.Name) { + case "crc", "_vl_msg_id": + continue + case "client_index", "context": + if n == 0 { + continue + } + } + n++ + if n == 1 { + fmt.Fprintln(w) + } + + generateField(ctx, w, msg.Fields, i) + } + + // generate end of the struct + fmt.Fprintln(w, "}") + + // generate name getter + generateMessageNameGetter(w, name, msg.Name) + + // generate CRC getter + generateCrcGetter(w, name, msg.CRC) + + // generate message type getter method + generateMessageTypeGetter(w, name, msgType) + + // generate message factory + generateMessageFactory(w, name) +} + +// generateField writes generated code for the field into w +func generateField(ctx *context, w io.Writer, fields []Field, i int) { + field := fields[i] + + fieldName := strings.TrimPrefix(field.Name, "_") + fieldName = camelCaseName(fieldName) + + dataType := convertToGoType(ctx, field.Type) + + fieldType := dataType + if field.IsArray() { + if dataType == "uint8" { + dataType = "byte" + } + fieldType = "[]" + dataType + } + fmt.Fprintf(w, "\t%s %s", fieldName, fieldType) + + if field.Length > 0 { + // fixed size array + fmt.Fprintf(w, "\t`struc:\"[%d]%s\"`", field.Length, dataType) + } else { + for _, f := range fields { + if f.SizeFrom == field.Name { + // variable sized array + sizeOfName := camelCaseName(f.Name) + fmt.Fprintf(w, "\t`struc:\"sizeof=%s\"`", sizeOfName) + } + } + } + + fmt.Fprintln(w) +} + +// generateService writes generated code for the service into w +func generateService(ctx *context, w io.Writer, svc *Service) { + reqTyp := camelCaseName(svc.RequestType) + + // method name is same as parameter type name by default + method := reqTyp + if svc.Stream { + // use Dump as prefix instead of suffix for stream services + if m := strings.TrimSuffix(method, "Dump"); method != m { + method = "Dump" + m + } + } + params := fmt.Sprintf("*%s", reqTyp) + returns := "error" + if replyTyp := camelCaseName(svc.ReplyType); replyTyp != "" { + returns = fmt.Sprintf("(*%s, error)", replyTyp) + } + + fmt.Fprintf(w, "\t%s(%s) %s\n", method, params, returns) +} + +// generateMessageNameGetter generates getter for original VPP message name into the provider writer +func generateMessageNameGetter(w io.Writer, structName string, msgName string) { + fmt.Fprintln(w, "func (*"+structName+") GetMessageName() string {") + fmt.Fprintln(w, "\treturn \""+msgName+"\"") + fmt.Fprintln(w, "}") +} + +// generateTypeNameGetter generates getter for original VPP type name into the provider writer +func generateTypeNameGetter(w io.Writer, structName string, msgName string) { + fmt.Fprintln(w, "func (*"+structName+") GetTypeName() string {") + fmt.Fprintln(w, "\treturn \""+msgName+"\"") + fmt.Fprintln(w, "}") +} + +// generateCrcGetter generates getter for CRC checksum of the message definition into the provider writer +func generateCrcGetter(w io.Writer, structName string, crc string) { + crc = strings.TrimPrefix(crc, "0x") + fmt.Fprintln(w, "func (*"+structName+") GetCrcString() string {") + fmt.Fprintln(w, "\treturn \""+crc+"\"") + fmt.Fprintln(w, "}") +} + +// generateMessageTypeGetter generates message factory for the generated message into the provider writer +func generateMessageTypeGetter(w io.Writer, structName string, msgType MessageType) { + fmt.Fprintln(w, "func (*"+structName+") GetMessageType() api.MessageType {") + if msgType == requestMessage { + fmt.Fprintln(w, "\treturn api.RequestMessage") + } else if msgType == replyMessage { + fmt.Fprintln(w, "\treturn api.ReplyMessage") + } else if msgType == eventMessage { + fmt.Fprintln(w, "\treturn api.EventMessage") + } else { + fmt.Fprintln(w, "\treturn api.OtherMessage") + } + fmt.Fprintln(w, "}") +} + +// generateMessageFactory generates message factory for the generated message into the provider writer +func generateMessageFactory(w io.Writer, structName string) { + fmt.Fprintln(w, "func New"+structName+"() api.Message {") + fmt.Fprintln(w, "\treturn &"+structName+"{}") + fmt.Fprintln(w, "}") +} -- cgit 1.2.3-korg