Support for Golang GRPC (Experimental) (#4082)

* support for grpc golang

* refactored grpc go generator

* added grpc-go test and refactored

* refactored idl_gen_grpc.cpp

* fixed grpc generate method name

* refactored flatc and fixed line length issue

* added codec to go lib and fixed formatting issues

* fixed spacing issues

CMakeLists.txt

@@ -50,6 +50,8 @@ set(FlatBuffers_Compiler_SRCS
   grpc/src/compiler/schema_interface.h
   grpc/src/compiler/cpp_generator.h
   grpc/src/compiler/cpp_generator.cc
+  grpc/src/compiler/go_generator.h
+  grpc/src/compiler/go_generator.cc
 )
 
 set(FlatHash_SRCS

go/grpc.go

+package flatbuffers
+
+// FlatbuffersCodec implements gRPC-go Codec which is used to encode and decode messages.
+var Codec string = "flatbuffers"
+
+type FlatbuffersCodec struct{}
+
+func (FlatbuffersCodec) Marshal(v interface{}) ([]byte, error) {
+	return v.(*Builder).FinishedBytes(), nil
+}
+
+func (FlatbuffersCodec) Unmarshal(data []byte, v interface{}) error {
+	v.(flatbuffersInit).Init(data, GetUOffsetT(data))
+	return nil
+}
+
+func (FlatbuffersCodec) String() string {
+	return Codec
+}
+
+type flatbuffersInit interface {
+	Init(data []byte, i UOffsetT)
+}

grpc/src/compiler/go_generator.h

+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_INTERNAL_COMPILER_GO_GENERATOR_H
+#define GRPC_INTERNAL_COMPILER_GO_GENERATOR_H
+
+//go generator is used to generate GRPC code for serialization system, such as flatbuffers
+#include <memory>
+#include <vector>
+
+#include "src/compiler/schema_interface.h"
+
+namespace grpc_go_generator {
+
+struct Parameters {
+	//Defines the custom parameter types for methods
+	//eg: flatbuffers uses flatbuffers.Builder as input for the client and output for the server
+	grpc::string custom_method_io_type;
+
+	//Package name for the service
+	grpc::string package_name;
+};
+
+// Return the source of the generated service file.
+grpc::string GenerateServiceSource(grpc_generator::File *file,
+                                   const grpc_generator::Service *service,
+                                   grpc_go_generator::Parameters *parameters);
+
+}
+
+#endif  // GRPC_INTERNAL_COMPILER_GO_GENERATOR_H

grpc/src/compiler/schema_interface.h

@@ -59,6 +59,8 @@ namespace grpc_generator {
 
     virtual grpc::string input_type_name() const = 0;
     virtual grpc::string output_type_name() const = 0;
+    virtual grpc::string input_name() const = 0;
+    virtual grpc::string output_name() const = 0;
 
     virtual bool NoStreaming() const = 0;
     virtual bool ClientOnlyStreaming() const = 0;
@@ -98,6 +100,7 @@ namespace grpc_generator {
     virtual grpc::string package() const = 0;
     virtual std::vector<grpc::string> package_parts() const = 0;
     virtual grpc::string additional_headers() const = 0;
+    virtual grpc::string additional_imports() const = 0;
 
     virtual int service_count() const = 0;
     virtual std::unique_ptr<const Service> service(int i) const = 0;

grpc/tests/go_test.go

+package testing
+
+import (
+	"../../tests/MyGame/Example"
+
+	"net"
+	"testing"
+
+	"github.com/google/flatbuffers/go"
+	"golang.org/x/net/context"
+	"google.golang.org/grpc"
+)
+
+type server struct{}
+
+// test used to send and receive in grpc methods
+var test string = "Flatbuffers"
+var addr string = "0.0.0.0:50051"
+
+// gRPC server store method
+func (s *server) Store(context context.Context, in *Example.Monster) (*flatbuffers.Builder, error) {
+	b := flatbuffers.NewBuilder(0)
+	i := b.CreateString(test)
+	Example.StatStart(b)
+	Example.StatAddId(b, i)
+	b.Finish(Example.StatEnd(b))
+	return b, nil
+
+}
+
+// gRPC server retrieve method
+func (s *server) Retrieve(context context.Context, in *Example.Stat) (*flatbuffers.Builder, error) {
+	b := flatbuffers.NewBuilder(0)
+	i := b.CreateString(test)
+	Example.MonsterStart(b)
+	Example.MonsterAddName(b, i)
+	b.Finish(Example.MonsterEnd(b))
+	return b, nil
+}
+
+func StoreClient(c Example.MonsterStorageClient, t *testing.T) {
+	b := flatbuffers.NewBuilder(0)
+	i := b.CreateString(test)
+	Example.MonsterStart(b)
+	Example.MonsterAddName(b, i)
+	b.Finish(Example.MonsterEnd(b))
+	out, err := c.Store(context.Background(), b)
+	if err != nil {
+		t.Fatal("Store client failed: %v", err)
+	}
+	if string(out.Id()) != test {
+		t.Errorf("StoreClient failed: expected=%s, got=%s\n", test, out.Id())
+		t.Fail()
+	}
+}
+
+func RetrieveClient(c Example.MonsterStorageClient, t *testing.T) {
+	b := flatbuffers.NewBuilder(0)
+	i := b.CreateString(test)
+	Example.StatStart(b)
+	Example.StatAddId(b, i)
+	b.Finish(Example.StatEnd(b))
+	out, err := c.Retrieve(context.Background(), b)
+	if err != nil {
+		t.Fatal("Retrieve client failed: %v", err)
+	}
+	if string(out.Name()) != test {
+		t.Errorf("RetrieveClient failed: expected=%s, got=%s\n", test, out.Name())
+		t.Fail()
+	}
+}
+
+func TestGRPC(t *testing.T) {
+	lis, err := net.Listen("tcp", addr)
+	if err != nil {
+		t.Fatalf("Failed to listen: %v", err)
+	}
+	ser := grpc.NewServer(grpc.CustomCodec(flatbuffers.FlatbuffersCodec{}))
+	Example.RegisterMonsterStorageServer(ser, &server{})
+	go func() {
+		if err := ser.Serve(lis); err != nil {
+			t.Fatalf("Failed to serve: %v", err)
+			t.FailNow()
+		}
+	}()
+	conn, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithCodec(flatbuffers.FlatbuffersCodec{}))
+	if err != nil {
+		t.Fatal("Failed to connect: %v", err)
+	}
+	defer conn.Close()
+	client := Example.NewMonsterStorageClient(conn)
+	StoreClient(client, t)
+	RetrieveClient(client, t)
+}

include/flatbuffers/idl.h

@@ -708,9 +708,15 @@ extern std::string BinaryMakeRule(const Parser &parser,
                                   const std::string &path,
                                   const std::string &file_name);
 
-// Generate GRPC interfaces.
+// Generate GRPC Cpp interfaces.
 // See idl_gen_grpc.cpp.
-bool GenerateGRPC(const Parser &parser,
+bool GenerateCppGRPC(const Parser &parser,
+                     const std::string &path,
+                     const std::string &file_name);
+
+// Generate GRPC Go interfaces.
+// See idl_gen_grpc.cpp.
+bool GenerateGoGRPC(const Parser &parser,
                     const std::string &path,
                     const std::string &file_name);
 

src/flatc.cpp

@@ -33,6 +33,9 @@ struct Generator {
   const char *generator_opt_short;
   const char *generator_opt_long;
   const char *lang_name;
+  bool (*generateGRPC)(const flatbuffers::Parser &parser,
+                       const std::string &path,
+                       const std::string &file_name);
   flatbuffers::IDLOptions::Language lang;
   const char *generator_help;
 
@@ -43,45 +46,50 @@ struct Generator {
 
 const Generator generators[] = {
   { flatbuffers::GenerateBinary,   "-b", "--binary", "binary",
+    nullptr,
     flatbuffers::IDLOptions::kMAX,
     "Generate wire format binaries for any data definitions",
     flatbuffers::BinaryMakeRule },
   { flatbuffers::GenerateTextFile, "-t", "--json", "text",
+    nullptr,
     flatbuffers::IDLOptions::kMAX,
     "Generate text output for any data definitions",
     flatbuffers::TextMakeRule },
   { flatbuffers::GenerateCPP,      "-c", "--cpp", "C++",
+    flatbuffers::GenerateCppGRPC,
     flatbuffers::IDLOptions::kMAX,
     "Generate C++ headers for tables/structs",
     flatbuffers::CPPMakeRule },
   { flatbuffers::GenerateGo,       "-g", "--go", "Go",
+    flatbuffers::GenerateGoGRPC,
     flatbuffers::IDLOptions::kGo,
     "Generate Go files for tables/structs",
     flatbuffers::GeneralMakeRule },
   { flatbuffers::GenerateGeneral,  "-j", "--java", "Java",
+    nullptr,
     flatbuffers::IDLOptions::kJava,
     "Generate Java classes for tables/structs",
     flatbuffers::GeneralMakeRule },
   { flatbuffers::GenerateJS,       "-s", "--js", "JavaScript",
+    nullptr,
     flatbuffers::IDLOptions::kMAX,
     "Generate JavaScript code for tables/structs",
     flatbuffers::JSMakeRule },
   { flatbuffers::GenerateGeneral,  "-n", "--csharp", "C#",
+    nullptr,
     flatbuffers::IDLOptions::kCSharp,
     "Generate C# classes for tables/structs",
     flatbuffers::GeneralMakeRule },
   { flatbuffers::GeneratePython,   "-p", "--python", "Python",
+    nullptr,
     flatbuffers::IDLOptions::kMAX,
     "Generate Python files for tables/structs",
     flatbuffers::GeneralMakeRule },
   { flatbuffers::GeneratePhp, nullptr, "--php", "PHP",
+    nullptr,
     flatbuffers::IDLOptions::kMAX,
     "Generate PHP files for tables/structs",
     flatbuffers::GeneralMakeRule },
-  { flatbuffers::GenerateGRPC, nullptr, "--grpc", "GRPC",
-    flatbuffers::IDLOptions::kMAX,
-    "Generate GRPC interfaces",
-    flatbuffers::CPPMakeRule },
 };
 
 const char *g_program_name = nullptr;
@@ -170,6 +178,7 @@ int main(int argc, const char *argv[]) {
   bool print_make_rules = false;
   bool raw_binary = false;
   bool schema_binary = false;
+  bool grpc_enabled = false;
   std::vector<std::string> filenames;
   std::vector<const char *> include_directories;
   std::vector<const char *> conform_include_directories;
@@ -243,6 +252,8 @@ int main(int argc, const char *argv[]) {
       } else if(arg == "--version") {
         printf("flatc version %s\n", FLATC_VERSION);
         exit(0);
+      } else if(arg == "--grpc") {
+        grpc_enabled = true;
       } else {
         for (size_t i = 0; i < num_generators; ++i) {
           if (arg == generators[i].generator_opt_long ||
@@ -360,6 +371,17 @@ int main(int argc, const char *argv[]) {
               printf("%s\n", flatbuffers::WordWrap(
                   make_rule, 80, " ", " \\").c_str());
           }
+          if (grpc_enabled) {
+            if (generators[i].generateGRPC != nullptr) {
+              if (!generators[i].generateGRPC(*g_parser, output_path, filebase)) {
+                    Error(std::string("Unable to generate GRPC interface for") +
+                          generators[i].lang_name);
+              }
+            } else {
+                Error(std::string("GRPC interface generator not implemented for ") +
+                      generators[i].lang_name);
+            }
+          }
         }
       }
 

src/idl_gen_grpc.cpp

@@ -19,8 +19,10 @@
 #include "flatbuffers/flatbuffers.h"
 #include "flatbuffers/idl.h"
 #include "flatbuffers/util.h"
+#include "flatbuffers/code_generators.h"
 
 #include "src/compiler/cpp_generator.h"
+#include "src/compiler/go_generator.h"
 
 namespace flatbuffers {
 
@@ -52,6 +54,14 @@ class FlatBufMethod : public grpc_generator::Method {
     return GRPCType(*method_->response);
   }
 
+  std::string input_name() const {
+    return (*method_->request).name;
+  }
+
+  std::string output_name() const {
+    return (*method_->response).name;
+  }
+
   bool NoStreaming() const { return streaming_ == kNone; }
   bool ClientOnlyStreaming() const { return streaming_ == kClient; }
   bool ServerOnlyStreaming() const { return streaming_ == kServer; }
@@ -159,6 +169,10 @@ class FlatBufFile : public grpc_generator::File {
     return "#include \"flatbuffers/grpc.h\"\n";
   }
 
+  std::string additional_imports() const {
+    return "import \"github.com/google/flatbuffers/go\"";
+  }
+
   int service_count() const {
     return static_cast<int>(parser_.services_.vec.size());
   };
@@ -178,7 +192,47 @@ class FlatBufFile : public grpc_generator::File {
   const std::string &file_name_;
 };
 
-bool GenerateGRPC(const Parser &parser,
+class GoGRPCGenerator : public flatbuffers::BaseGenerator {
+ public:
+  GoGRPCGenerator(const Parser &parser, const std::string &path,
+                  const std::string &file_name)
+    : BaseGenerator(parser, path, file_name, "", "" /*Unused*/),
+      parser_(parser), path_(path), file_name_(file_name) {}
+
+  bool generate() {
+    FlatBufFile file(parser_, file_name_);
+    grpc_go_generator::Parameters p;
+    p.custom_method_io_type = "flatbuffers.Builder";
+    for (int i = 0; i < file.service_count(); i++) {
+      auto service = file.service(i);
+      const Definition *def = parser_.services_.vec[i];
+      p.package_name = LastNamespacePart(*(def->defined_namespace));
+      std::string output = grpc_go_generator::GenerateServiceSource(&file, service.get(), &p);
+      std::string filename = NamespaceDir(*def->defined_namespace) + def->name + "_grpc.go";
+      if (!flatbuffers::SaveFile(filename.c_str(), output, false))
+        return false;
+    }
+    return true;
+  }
+
+ protected:
+  const Parser &parser_;
+  const std::string &path_, &file_name_;
+};
+
+bool GenerateGoGRPC(const Parser &parser,
+                    const std::string &path,
+                    const std::string &file_name) {
+  int nservices = 0;
+  for (auto it = parser.services_.vec.begin();
+       it != parser.services_.vec.end(); ++it) {
+    if (!(*it)->generated) nservices++;
+  }
+  if (!nservices) return true;
+  return GoGRPCGenerator(parser, path, file_name).generate();
+}
+
+bool GenerateCppGRPC(const Parser &parser,
                   const std::string &/*path*/,
                   const std::string &file_name) {
 

tests/MyGame/Example/MonsterStorage_grpc.go

+//Generated by gRPC Go plugin
+//If you make any local changes, they will be lost
+//source: monster_test
+
+package Example
+
+import "github.com/google/flatbuffers/go"
+
+import (
+  context "golang.org/x/net/context"
+  grpc "google.golang.org/grpc"
+)
+
+// Client API for MonsterStorage service
+type MonsterStorageClient interface{
+  Store(ctx context.Context, in *flatbuffers.Builder, 
+  	opts... grpc.CallOption) (* Stat, error)  
+  Retrieve(ctx context.Context, in *flatbuffers.Builder, 
+  	opts... grpc.CallOption) (* Monster, error)  
+}
+
+type monsterStorageClient struct {
+  cc *grpc.ClientConn
+}
+
+func NewMonsterStorageClient(cc *grpc.ClientConn) MonsterStorageClient {
+  return &monsterStorageClient{cc}
+}
+
+func (c *monsterStorageClient) Store(ctx context.Context, in *flatbuffers.Builder, 
+	opts... grpc.CallOption) (* Stat, error) {
+  out := new(Stat)
+  err := grpc.Invoke(ctx, "/Example.MonsterStorage/Store", in, out, c.cc, opts...)
+  if err != nil { return nil, err }
+  return out, nil
+}
+
+func (c *monsterStorageClient) Retrieve(ctx context.Context, in *flatbuffers.Builder, 
+	opts... grpc.CallOption) (* Monster, error) {
+  out := new(Monster)
+  err := grpc.Invoke(ctx, "/Example.MonsterStorage/Retrieve", in, out, c.cc, opts...)
+  if err != nil { return nil, err }
+  return out, nil
+}
+
+// Server API for MonsterStorage service
+type MonsterStorageServer interface {
+  Store(context.Context, *Monster) (*flatbuffers.Builder, error)  
+  Retrieve(context.Context, *Stat) (*flatbuffers.Builder, error)  
+}
+
+func RegisterMonsterStorageServer(s *grpc.Server, srv MonsterStorageServer) {
+  s.RegisterService(&_MonsterStorage_serviceDesc, srv)
+}
+
+func _MonsterStorage_Store_Handler(srv interface{}, ctx context.Context,
+	dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+  in := new(Monster)
+  if err := dec(in); err != nil { return nil, err }
+  if interceptor == nil { return srv.(MonsterStorageServer).Store(ctx, in) }
+  info := &grpc.UnaryServerInfo{
+    Server: srv,
+    FullMethod: "/Example.MonsterStorage/Store",
+  }
+  
+  handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+    return srv.(MonsterStorageServer).Store(ctx, req.(* Monster))
+  }
+  return interceptor(ctx, in, info, handler)
+}
+
+
+func _MonsterStorage_Retrieve_Handler(srv interface{}, ctx context.Context,
+	dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+  in := new(Stat)
+  if err := dec(in); err != nil { return nil, err }
+  if interceptor == nil { return srv.(MonsterStorageServer).Retrieve(ctx, in) }
+  info := &grpc.UnaryServerInfo{
+    Server: srv,
+    FullMethod: "/Example.MonsterStorage/Retrieve",
+  }
+  
+  handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+    return srv.(MonsterStorageServer).Retrieve(ctx, req.(* Stat))
+  }
+  return interceptor(ctx, in, info, handler)
+}
+
+
+var _MonsterStorage_serviceDesc = grpc.ServiceDesc{
+  ServiceName: "Example.MonsterStorage",
+  HandlerType: (*MonsterStorageServer)(nil),
+  Methods: []grpc.MethodDesc{
+    {
+      MethodName: "Store",
+      Handler: _MonsterStorage_Store_Handler, 
+    },
+    {
+      MethodName: "Retrieve",
+      Handler: _MonsterStorage_Retrieve_Handler, 
+    },
+  },
+  Streams: []grpc.StreamDesc{
+  },
+}
+