Improve binapi generator

- simplified Size/Marshal/Unmarshal methods
- replace struc in unions with custom marshal/unmarshal
- fix imports in generated files
- fix mock adapter
- generate rpc service using low-level stream API (dumps generate control ping or stream msg..)
- move examples/binapi to binapi and generate all API for latest release
- add binapigen.Plugin for developing custom generator plugins
- optionally generate HTTP handlers (REST API) for RPC services
- add govpp program for browsing VPP API

Change-Id: I092e9ed2b0c17972b3476463c3d4b14dd76ed42b
Signed-off-by: Ondrej Fabry <ofabry@cisco.com>

4 files changed, 305 insertions, 152 deletions

diff --git a/cmd/binapi-generator/main.go b/cmd/binapi-generator/main.go
index e30aaf2..732b4f3 100644
--- a/cmd/binapi-generator/main.go
+++ b/cmd/binapi-generator/main.go
@@ -18,93 +18,82 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"path/filepath"
+	"strings"
+	"unicode"
 
 	"github.com/sirupsen/logrus"
 
 	"git.fd.io/govpp.git/binapigen"
 	"git.fd.io/govpp.git/binapigen/vppapi"
-	"git.fd.io/govpp.git/version"
+	"git.fd.io/govpp.git/internal/version"
 )
 
 func init() {
 	flag.Usage = func() {
-		fmt.Fprintf(flag.CommandLine.Output(), "Usage: %s [OPTION]... [API]...\n", os.Args[0])
-		fmt.Fprintln(flag.CommandLine.Output(), "Generate code for each API.")
-		fmt.Fprintf(flag.CommandLine.Output(), "Example: %s -output-dir=binapi acl interface l2\n", os.Args[0])
-		fmt.Fprintln(flag.CommandLine.Output())
-		fmt.Fprintln(flag.CommandLine.Output(), "Options:")
-		flag.CommandLine.PrintDefaults()
+		fmt.Fprintf(os.Stderr, "Usage: %s [OPTION] API_FILES\n", os.Args[0])
+		fmt.Fprintln(os.Stderr, "Parse API_FILES and generate Go bindings based on the options given:")
+		flag.PrintDefaults()
 	}
 }
 
 func main() {
 	var (
-		theInputFile = flag.String("input-file", "", "Input VPP API file. (DEPRECATED: Use program arguments to define VPP API files)")
-		theApiDir    = flag.String("input-dir", vppapi.DefaultAPIDir, "Directory with VPP API files.")
-		theOutputDir = flag.String("output-dir", ".", "Output directory where code will be generated.")
+		theApiDir        = flag.String("input-dir", vppapi.DefaultDir, "Input directory containing API files.")
+		theInputFile     = flag.String("input-file", "", "DEPRECATED: Use program arguments to define files to generate.")
+		theOutputDir     = flag.String("output-dir", "binapi", "Output directory where code will be generated.")
+		importPrefix     = flag.String("import-prefix", "", "Define import path prefix to be used to import types.")
+		generatorPlugins = flag.String("gen", "rpc", "List of generator plugins to run for files.")
 
-		importPrefix       = flag.String("import-prefix", "", "Define import path prefix to be used to import types.")
-		importTypes        = flag.Bool("import-types", true, "Generate packages for imported types.")
-		includeAPIVer      = flag.Bool("include-apiver", true, "Include APIVersion constant for each module.")
-		includeServices    = flag.Bool("include-services", true, "Include RPC service api and client implementation.")
-		includeComments    = flag.Bool("include-comments", false, "Include JSON API source in comments for each object.")
-		includeBinapiNames = flag.Bool("include-binapi-names", true, "Include binary API names in struct tag.")
-		includeVppVersion  = flag.Bool("include-vpp-version", true, "Include version of the VPP that provided input files.")
-
-		debugMode    = flag.Bool("debug", os.Getenv("DEBUG_GOVPP") != "", "Enable debug mode.")
 		printVersion = flag.Bool("version", false, "Prints version and exits.")
+		debugLog     = flag.Bool("debug", false, "Enable verbose logging.")
 	)
 	flag.Parse()
+
 	if *printVersion {
 		fmt.Fprintln(os.Stdout, version.Info())
 		os.Exit(0)
 	}
-	if flag.NArg() == 1 && flag.Arg(0) == "version" {
-		fmt.Fprintln(os.Stdout, version.Verbose())
-		os.Exit(0)
+
+	if *debugLog {
+		logrus.SetLevel(logrus.DebugLevel)
 	}
 
-	// prepare options
-	var opts binapigen.Options
+	var filesToGenerate []string
 	if *theInputFile != "" {
 		if flag.NArg() > 0 {
 			fmt.Fprintln(os.Stderr, "input-file cannot be combined with files to generate in arguments")
 			os.Exit(1)
 		}
-		opts.FilesToGenerate = append(opts.FilesToGenerate, *theInputFile)
-	} else {
-		opts.FilesToGenerate = append(opts.FilesToGenerate, flag.Args()...)
-	}
-	if ver := os.Getenv("VPP_API_VERSION"); ver != "" {
-		// use version from env var if set
-		opts.VPPVersion = ver
+		filesToGenerate = append(filesToGenerate, *theInputFile)
 	} else {
-		opts.VPPVersion = ResolveVppVersion(*theApiDir)
+		filesToGenerate = append(filesToGenerate, flag.Args()...)
 	}
-	opts.IncludeAPIVersion = *includeAPIVer
-	opts.IncludeComments = *includeComments
-	opts.IncludeBinapiNames = *includeBinapiNames
-	opts.IncludeServices = *includeServices
-	opts.IncludeVppVersion = *includeVppVersion
-	opts.ImportPrefix = *importPrefix
-	opts.ImportTypes = *importTypes
 
-	if *debugMode {
-		logrus.SetLevel(logrus.DebugLevel)
-		logrus.Debug("debug mode enabled")
+	opts := binapigen.Options{
+		ImportPrefix: *importPrefix,
+		OutputDir:    *theOutputDir,
+	}
+	if opts.OutputDir == "binapi" {
+		if wd, _ := os.Getwd(); filepath.Base(wd) == "binapi" {
+			opts.OutputDir = "."
+		}
 	}
-
 	apiDir := *theApiDir
-	outputDir := *theOutputDir
+	genPlugins := strings.FieldsFunc(*generatorPlugins, func(c rune) bool {
+		return !unicode.IsLetter(c) && !unicode.IsNumber(c)
+	})
 
-	binapigen.Run(apiDir, opts, func(g *binapigen.Generator) error {
-		for _, file := range g.Files {
+	binapigen.Run(apiDir, filesToGenerate, opts, func(gen *binapigen.Generator) error {
+		for _, file := range gen.Files {
 			if !file.Generate {
 				continue
 			}
-			binapigen.GenerateBinapi(g, file, outputDir)
-			if g.IncludeServices && file.Service != nil {
-				binapigen.GenerateRPC(g, file, outputDir)
+			binapigen.GenerateAPI(gen, file)
+			for _, p := range genPlugins {
+				if err := binapigen.RunPlugin(p, gen, file); err != nil {
+					return err
+				}
 			}
 		}
 		return nil
diff --git a/cmd/binapi-generator/util.go b/cmd/binapi-generator/util.go
deleted file mode 100644
index 8738963..0000000
--- a/cmd/binapi-generator/util.go
+++ /dev/null
@@ -1,81 +0,0 @@
-//  Copyright (c) 2020 Cisco and/or its affiliates.
-//
-//  Licensed under the Apache License, Version 2.0 (the "License");
-//  you may not use this file except in compliance with the License.
-//  You may obtain a copy of the License at:
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-//  Unless required by applicable law or agreed to in writing, software
-//  distributed under the License is distributed on an "AS IS" BASIS,
-//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-//  See the License for the specific language governing permissions and
-//  limitations under the License.
-
-package main
-
-import (
-	"io/ioutil"
-	"os/exec"
-	"path"
-	"strings"
-
-	"github.com/sirupsen/logrus"
-)
-
-const (
-	versionScriptPath = "./src/scripts/version"
-	defaultVppApiDir  = "/usr/share/vpp/api"
-)
-
-func ResolveVppVersion(inputDir string) string {
-	// assuming VPP package is installed
-	if inputDir == defaultVppApiDir {
-		// resolve VPP version using dpkg
-		cmd := exec.Command("dpkg-query", "-f", "${Version}", "-W", "vpp")
-		out, err := cmd.CombinedOutput()
-		if err != nil {
-			logrus.Warnf("resolving VPP version from installed package failed: %v", err)
-			logrus.Warnf("command output: %s", out)
-		} else {
-			version := strings.TrimSpace(string(out))
-			logrus.Debugf("resolved VPP version from installed package: %v", version)
-			return version
-		}
-	}
-	// check if inside VPP git repo
-	if inputDir != "" {
-		repo := findVppGitRepo(inputDir)
-		if repo != "" {
-			cmd := exec.Command(versionScriptPath)
-			cmd.Dir = repo
-			out, err := cmd.CombinedOutput()
-			if err != nil {
-				logrus.Warnf("resolving VPP version from version script failed: %v", err)
-				logrus.Warnf("command output: %s", out)
-			} else {
-				version := strings.TrimSpace(string(out))
-				logrus.Debugf("resolved VPP version from version script: %v", version)
-				return version
-			}
-		}
-		file, err := ioutil.ReadFile(path.Join(inputDir, "VPP_VERSION"))
-		if err == nil {
-			return strings.TrimSpace(string(file))
-		}
-	}
-	logrus.Warnf("VPP version could not be resolved, you can set it manually using VPP_API_VERSION env var")
-	return "unknown"
-}
-
-func findVppGitRepo(dir string) string {
-	cmd := exec.Command("git", "rev-parse", "--show-toplevel")
-	cmd.Dir = dir
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		logrus.Warnf("checking VPP git repo failed: %v", err)
-		logrus.Warnf("command output: %s", out)
-		return ""
-	}
-	return strings.TrimSpace(string(out))
-}
diff --git a/cmd/govpp/main.go b/cmd/govpp/main.go
new file mode 100644
index 0000000..f1ad5d8
--- /dev/null
+++ b/cmd/govpp/main.go
@@ -0,0 +1,265 @@
+//  Copyright (c) 2020 Cisco and/or its affiliates.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at:
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+	"text/tabwriter"
+
+	"git.fd.io/govpp.git"
+	"git.fd.io/govpp.git/adapter/socketclient"
+	"git.fd.io/govpp.git/binapi/vpe"
+	"git.fd.io/govpp.git/binapigen"
+	"git.fd.io/govpp.git/binapigen/vppapi"
+)
+
+func main() {
+	flag.Parse()
+
+	apifiles, err := vppapi.Parse()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	switch cmd := flag.Arg(0); cmd {
+	case "server":
+		runServer(apifiles, ":7777")
+	case "vppapi":
+		showVPPAPI(os.Stdout, apifiles)
+	case "vppapijson":
+		if flag.NArg() == 1 {
+			writeAsJSON(os.Stdout, apifiles)
+		} else {
+			f := flag.Arg(1)
+			var found bool
+			for _, apifile := range apifiles {
+				if apifile.Name == f {
+					writeAsJSON(os.Stdout, apifile)
+					found = true
+					break
+				}
+			}
+			if !found {
+				log.Fatalf("VPP API file %q not found", f)
+			}
+		}
+	case "rpc":
+		showRPC(apifiles)
+	case "cli":
+		args := flag.Args()
+		if len(args) == 0 {
+			args = []string{"?"}
+		}
+		sendCLI(args[1:])
+	default:
+		log.Fatalf("invalid command: %q", cmd)
+	}
+
+}
+
+func writeAsJSON(w io.Writer, data interface{}) {
+	b, err := json.MarshalIndent(data, "", "  ")
+	if err != nil {
+		log.Fatal(err)
+		return
+	}
+	if _, err := w.Write(b); err != nil {
+		panic(err)
+	}
+}
+
+func showRPC(apifiles []*vppapi.File) {
+	for _, apifile := range apifiles {
+		fmt.Printf("%s.api\n", apifile.Name)
+		if apifile.Service == nil {
+			continue
+		}
+		for _, rpc := range apifile.Service.RPCs {
+			req := rpc.Request
+			reply := rpc.Reply
+			if rpc.Stream {
+				reply = "stream " + reply
+			}
+			fmt.Printf(" rpc (%s) --> (%s)\n", req, reply)
+		}
+	}
+}
+
+func showVPPAPI(out io.Writer, apifiles []*vppapi.File) {
+	binapigen.SortFilesByImports(apifiles)
+
+	var buf bytes.Buffer
+	w := tabwriter.NewWriter(&buf, 0, 0, 3, ' ', 0)
+	fmt.Fprintf(w, "API\tOPTIONS\tCRC\tPATH\tIMPORTED\tTYPES\t\n")
+
+	for _, apifile := range apifiles {
+		importedTypes := binapigen.ListImportedTypes(apifiles, apifile)
+		var options []string
+		for k, v := range apifile.Options {
+			options = append(options, fmt.Sprintf("%s=%v", k, v))
+		}
+		imports := fmt.Sprintf("%d apis, %2d types", len(apifile.Imports), len(importedTypes))
+		path := strings.TrimPrefix(apifile.Path, vppapi.DefaultDir+"/")
+		types := fmt.Sprintf("%2d enum, %2d alias, %2d struct, %2d union, %2d msg",
+			len(apifile.EnumTypes), len(apifile.AliasTypes), len(apifile.StructTypes), len(apifile.UnionTypes), len(apifile.Messages))
+		fmt.Fprintf(w, " %s\t%s\t%s\t%s\t%v\t%s\t\n",
+			apifile.Name, strings.Join(options, " "), apifile.CRC, path, imports, types)
+	}
+
+	if err := w.Flush(); err != nil {
+		log.Fatal(err)
+	}
+	fmt.Fprint(out, buf.String())
+}
+
+func sendCLI(args []string) {
+	cmd := strings.Join(args, " ")
+	fmt.Printf("# %s\n", cmd)
+
+	conn, err := govpp.Connect("/run/vpp/api.sock")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer conn.Disconnect()
+
+	ch, err := conn.NewAPIChannel()
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer ch.Close()
+
+	if err := ch.CheckCompatiblity(vpe.AllMessages()...); err != nil {
+		log.Fatal(err)
+	}
+
+	client := vpe.NewServiceClient(conn)
+	reply, err := client.CliInband(context.Background(), &vpe.CliInband{
+		Cmd: cmd,
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Print(reply.Reply)
+}
+
+func runServer(apifiles []*vppapi.File, addr string) {
+	apiRoutes(apifiles, http.DefaultServeMux)
+
+	conn, err := govpp.Connect(socketclient.DefaultSocketName)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	vpeRPC := vpe.NewServiceClient(conn)
+	c := vpe.RESTHandler(vpeRPC)
+
+	http.Handle("/", c)
+
+	log.Printf("listening on %v", addr)
+
+	if err := http.ListenAndServe(addr, nil); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func apiRoutes(apifiles []*vppapi.File, mux *http.ServeMux) {
+	for _, apifile := range apifiles {
+		name := apifile.Name
+		mux.HandleFunc("/vppapi/"+name, apiFileHandler(apifile))
+		mux.HandleFunc("/raw/"+name, rawHandler(apifile))
+		mux.HandleFunc("/rpc/"+name, rpcHandler(apifile))
+	}
+	mux.HandleFunc("/vppapi", apiHandler(apifiles))
+}
+
+func rpcHandler(apifile *vppapi.File) func(http.ResponseWriter, *http.Request) {
+	return func(w http.ResponseWriter, req *http.Request) {
+		msgName := strings.TrimPrefix(req.URL.Path, "/rpc/"+apifile.Name+"/")
+		if msgName == "" {
+			http.Error(w, "no message name", 500)
+			return
+		}
+
+		input, err := ioutil.ReadAll(req.Body)
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+			return
+		}
+
+		msgReq := make(map[string]interface{})
+		err = json.Unmarshal(input, &msgReq)
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+			return
+		}
+
+		var msg *vppapi.Message
+		for _, m := range apifile.Messages {
+			if m.Name == msgName {
+				msg = &m
+				break
+			}
+		}
+		if msg == nil {
+			http.Error(w, "unknown message name: "+msgName, 500)
+			return
+		}
+
+	}
+}
+
+func apiHandler(apifiles []*vppapi.File) func(http.ResponseWriter, *http.Request) {
+	return func(w http.ResponseWriter, req *http.Request) {
+		b, err := json.MarshalIndent(apifiles, "", "  ")
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+			return
+		}
+		w.Write(b)
+	}
+}
+
+func apiFileHandler(apifile *vppapi.File) func(http.ResponseWriter, *http.Request) {
+	return func(w http.ResponseWriter, req *http.Request) {
+		b, err := json.MarshalIndent(apifile, "", "  ")
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+			return
+		}
+		w.Write(b)
+	}
+}
+
+func rawHandler(apifile *vppapi.File) func(http.ResponseWriter, *http.Request) {
+	return func(w http.ResponseWriter, req *http.Request) {
+		b, err := ioutil.ReadFile(apifile.Path)
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+			return
+		}
+		w.Write(b)
+	}
+}
diff --git a/cmd/vpp-proxy/main.go b/cmd/vpp-proxy/main.go
index 5221218..d1af5df 100644
--- a/cmd/vpp-proxy/main.go
+++ b/cmd/vpp-proxy/main.go
@@ -15,18 +15,16 @@
 package main
 
 import (
-	"context"
 	"encoding/gob"
 	"flag"
-	"io"
 	"log"
 
 	"git.fd.io/govpp.git/adapter/socketclient"
 	"git.fd.io/govpp.git/adapter/statsclient"
 	"git.fd.io/govpp.git/api"
+	interfaces "git.fd.io/govpp.git/binapi/interface"
+	"git.fd.io/govpp.git/binapi/vpe"
 	_ "git.fd.io/govpp.git/core"
-	"git.fd.io/govpp.git/examples/binapi/interfaces"
-	"git.fd.io/govpp.git/examples/binapi/vpe"
 	"git.fd.io/govpp.git/proxy"
 )
 
@@ -93,30 +91,12 @@ func runClient() {
 		panic(err)
 	}
 
-	// - using binapi message directly
 	req := &vpe.CliInband{Cmd: "show version"}
 	reply := new(vpe.CliInbandReply)
 	if err := binapiChannel.SendRequest(req).ReceiveReply(reply); err != nil {
 		log.Fatalln("binapi request failed:", err)
 	}
 	log.Printf("VPP version: %+v", reply.Reply)
-
-	// - or using generated rpc service
-	svc := interfaces.NewServiceClient(binapiChannel)
-	stream, err := svc.DumpSwInterface(context.Background(), &interfaces.SwInterfaceDump{})
-	if err != nil {
-		log.Fatalln("binapi request failed:", err)
-	}
-	for {
-		iface, err := stream.Recv()
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			log.Fatalln(err)
-		}
-		log.Printf("- interface: %+v", iface)
-	}
 }
 
 func runServer() {