Include services in reflection data (fixes #4639) (#4713)

* include service in reflection data (fixes #4639)

* changes from review

* regenerated test data

.gitignore

@@ -93,3 +93,4 @@ js/flatbuffers.mjs
 .ninja_log
 build.ninja
 rules.ninja
+.vscode

include/flatbuffers/idl.h

@@ -306,7 +306,7 @@ inline size_t InlineAlignment(const Type &type) {
 struct EnumVal {
   EnumVal(const std::string &_name, int64_t _val) : name(_name), value(_val) {}
 
-  Offset<reflection::EnumVal> Serialize(FlatBufferBuilder *builder) const;
+  Offset<reflection::EnumVal> Serialize(FlatBufferBuilder *builder, const Parser &parser) const;
 
   std::string name;
   std::vector<std::string> doc_comment;
@@ -326,8 +326,7 @@ struct EnumDef : public Definition {
     return nullptr;
   }
 
-  Offset<reflection::Enum> Serialize(FlatBufferBuilder *builder,
-                                     const Parser &parser) const;
+  Offset<reflection::Enum> Serialize(FlatBufferBuilder *builder, const Parser &parser) const;
 
   SymbolTable<EnumVal> vals;
   bool is_union;
@@ -342,14 +341,15 @@ inline bool EqualByName(const Type &a, const Type &b) {
          (a.enum_def == b.enum_def || a.enum_def->name == b.enum_def->name);
 }
 
-struct RPCCall {
-  std::string name;
-  SymbolTable<Value> attributes;
+struct RPCCall : public Definition {
+  Offset<reflection::RPCCall> Serialize(FlatBufferBuilder *builder, const Parser &parser) const;
+
   StructDef *request, *response;
-  std::vector<std::string> rpc_comment;
 };
 
 struct ServiceDef : public Definition {
+  Offset<reflection::Service> Serialize(FlatBufferBuilder *builder, const Parser &parser) const;
+
   SymbolTable<RPCCall> calls;
 };
 
@@ -382,6 +382,7 @@ struct IDLOptions {
   std::string include_prefix;
   bool keep_include_path;
   bool binary_schema_comments;
+  bool binary_schema_builtins;
   bool skip_flatbuffers_import;
   std::string go_import;
   std::string go_namespace;
@@ -440,6 +441,7 @@ struct IDLOptions {
         allow_non_utf8(false),
         keep_include_path(false),
         binary_schema_comments(false),
+        binary_schema_builtins(false),
         skip_flatbuffers_import(false),
         reexport_ts_modules(true),
         protobuf_ascii_alike(false),

reflection/reflection.fbs

@@ -44,6 +44,7 @@ table EnumVal {
     value:long (key);
     object:Object;  // Will be deprecated in favor of union_type in the future.
     union_type:Type;
+    documentation:[string];
 }
 
 table Enum {
@@ -79,12 +80,28 @@ table Object {  // Used for both tables and structs.
     documentation:[string];
 }
 
+table RPCCall {
+    name:string (required, key);
+    request:Object (required);      // must be a table (not a struct)
+    response:Object (required);     // must be a table (not a struct)
+    attributes:[KeyValue];
+    documentation:[string];
+}
+
+table Service {
+    name:string (required, key);
+    calls:[RPCCall];
+    attributes:[KeyValue];
+    documentation:[string];
+}
+
 table Schema {
     objects:[Object] (required);    // Sorted.
     enums:[Enum] (required);        // Sorted.
     file_ident:string;
     file_ext:string;
     root_table:Object;
+    services:[Service];             // Sorted.
 }
 
 root_type Schema;

src/flatc.cpp

@@ -106,6 +106,7 @@ std::string FlatCompiler::GetUsageString(const char *program_name) const {
     "  --grpc             Generate GRPC interfaces for the specified languages\n"
     "  --schema           Serialize schemas instead of JSON (use with -b)\n"
     "  --bfbs-comments    Add doc comments to the binary schema files.\n"
+    "  --bfbs-builtins    Add builtin attributes to the binary schema files.\n"
     "  --conform FILE     Specify a schema the following schemas should be\n"
     "                     an evolution of. Gives errors if not.\n"
     "  --conform-includes Include path for the schema given with --conform\n"
@@ -253,6 +254,8 @@ int FlatCompiler::Compile(int argc, const char **argv) {
         grpc_enabled = true;
       } else if (arg == "--bfbs-comments") {
         opts.binary_schema_comments = true;
+      } else if (arg == "--bfbs-builtins") {
+        opts.binary_schema_builtins = true;
       } else if (arg == "--no-fb-import") {
         opts.skip_flatbuffers_import = true;
       } else if (arg == "--no-ts-reexport") {

src/idl_gen_grpc.cpp

@@ -50,7 +50,7 @@ class FlatBufMethod : public grpc_generator::Method {
   grpc::string GetLeadingComments(const grpc::string) const { return ""; }
   grpc::string GetTrailingComments(const grpc::string) const { return ""; }
   std::vector<grpc::string> GetAllComments() const {
-    return method_->rpc_comment;
+    return method_->doc_comment;
   }
 
   std::string name() const { return method_->name; }

src/idl_parser.cpp

@@ -1724,7 +1724,7 @@ CheckedError Parser::ParseService() {
   ECHECK(ParseMetaData(&service_def.attributes));
   EXPECT('{');
   do {
-    std::vector<std::string> rpc_comment = doc_comment_;
+    std::vector<std::string> doc_comment = doc_comment_;
     auto rpc_name = attribute_;
     EXPECT(kTokenIdentifier);
     EXPECT('(');
@@ -1740,7 +1740,7 @@ CheckedError Parser::ParseService() {
     rpc.name = rpc_name;
     rpc.request = reqtype.struct_def;
     rpc.response = resptype.struct_def;
-    rpc.rpc_comment = rpc_comment;
+    rpc.doc_comment = doc_comment;
     if (service_def.calls.Add(rpc_name, &rpc))
       return Error("rpc already exists: " + rpc_name);
     ECHECK(ParseMetaData(&rpc.attributes));
@@ -2527,12 +2527,20 @@ void Parser::Serialize() {
     enum_offsets.push_back(offset);
     (*it)->serialized_location = offset.o;
   }
+  std::vector<Offset<reflection::Service>> service_offsets;
+  for (auto it = services_.vec.begin(); it != services_.vec.end(); ++it) {
+    auto offset = (*it)->Serialize(&builder_, *this);
+    service_offsets.push_back(offset);
+    (*it)->serialized_location = offset.o;
+  }
   auto schema_offset = reflection::CreateSchema(
-      builder_, builder_.CreateVectorOfSortedTables(&object_offsets),
+      builder_,
+      builder_.CreateVectorOfSortedTables(&object_offsets),
       builder_.CreateVectorOfSortedTables(&enum_offsets),
       builder_.CreateString(file_identifier_),
       builder_.CreateString(file_extension_),
-      root_struct_def_ ? root_struct_def_->serialized_location : 0);
+      (root_struct_def_ ? root_struct_def_->serialized_location : 0),
+      builder_.CreateVectorOfSortedTables(&service_offsets));
   if (opts.size_prefixed) {
     builder_.FinishSizePrefixed(schema_offset, reflection::SchemaIdentifier());
   } else {
@@ -2549,9 +2557,12 @@ Offset<reflection::Object> StructDef::Serialize(FlatBufferBuilder *builder,
   }
   auto qualified_name = defined_namespace->GetFullyQualifiedName(name);
   return reflection::CreateObject(
-      *builder, builder->CreateString(qualified_name),
-      builder->CreateVectorOfSortedTables(&field_offsets), fixed,
-      static_cast<int>(minalign), static_cast<int>(bytesize),
+      *builder,
+      builder->CreateString(qualified_name),
+      builder->CreateVectorOfSortedTables(&field_offsets),
+      fixed,
+      static_cast<int>(minalign),
+      static_cast<int>(bytesize),
       SerializeAttributes(builder, parser),
       parser.opts.binary_schema_comments
           ? builder->CreateVectorOfStrings(doc_comment)
@@ -2575,33 +2586,72 @@ Offset<reflection::Field> FieldDef::Serialize(FlatBufferBuilder *builder,
   // space by sharing it. Same for common values of value.type.
 }
 
+Offset<reflection::RPCCall> RPCCall::Serialize(FlatBufferBuilder *builder,
+                                               const Parser &parser) const {
+  return reflection::CreateRPCCall(
+      *builder,
+      builder->CreateString(name),
+      request->serialized_location,
+      response->serialized_location,
+      SerializeAttributes(builder, parser),
+      parser.opts.binary_schema_comments
+          ? builder->CreateVectorOfStrings(doc_comment)
+          : 0);
+}
+
+Offset<reflection::Service> ServiceDef::Serialize(FlatBufferBuilder *builder,
+                                                  const Parser &parser) const {
+  std::vector<Offset<reflection::RPCCall>> servicecall_offsets;
+  for (auto it = calls.vec.begin(); it != calls.vec.end(); ++it) {
+    servicecall_offsets.push_back((*it)->Serialize(builder, parser));
+  }
+  auto qualified_name = defined_namespace->GetFullyQualifiedName(name);
+  return reflection::CreateService(
+      *builder,
+      builder->CreateString(qualified_name),
+      builder->CreateVector(servicecall_offsets),
+      SerializeAttributes(builder, parser),
+      parser.opts.binary_schema_comments
+          ? builder->CreateVectorOfStrings(doc_comment)
+          : 0);
+}
+
 Offset<reflection::Enum> EnumDef::Serialize(FlatBufferBuilder *builder,
                                             const Parser &parser) const {
   std::vector<Offset<reflection::EnumVal>> enumval_offsets;
   for (auto it = vals.vec.begin(); it != vals.vec.end(); ++it) {
-    enumval_offsets.push_back((*it)->Serialize(builder));
+    enumval_offsets.push_back((*it)->Serialize(builder, parser));
   }
   auto qualified_name = defined_namespace->GetFullyQualifiedName(name);
   return reflection::CreateEnum(
-      *builder, builder->CreateString(qualified_name),
-      builder->CreateVector(enumval_offsets), is_union,
-      underlying_type.Serialize(builder), SerializeAttributes(builder, parser),
+      *builder,
+      builder->CreateString(qualified_name),
+      builder->CreateVector(enumval_offsets),
+      is_union,
+      underlying_type.Serialize(builder),
+      SerializeAttributes(builder, parser),
       parser.opts.binary_schema_comments
           ? builder->CreateVectorOfStrings(doc_comment)
           : 0);
 }
 
-Offset<reflection::EnumVal> EnumVal::Serialize(
-    FlatBufferBuilder *builder) const {
+Offset<reflection::EnumVal> EnumVal::Serialize(FlatBufferBuilder *builder,
+                                               const Parser &parser) const {
   return reflection::CreateEnumVal(
-      *builder, builder->CreateString(name), value,
+      *builder,
+      builder->CreateString(name),
+      value,
       union_type.struct_def ? union_type.struct_def->serialized_location : 0,
-      union_type.Serialize(builder));
+      union_type.Serialize(builder),
+      parser.opts.binary_schema_comments
+          ? builder->CreateVectorOfStrings(doc_comment)
+          : 0);
 }
 
 Offset<reflection::Type> Type::Serialize(FlatBufferBuilder *builder) const {
   return reflection::CreateType(
-      *builder, static_cast<reflection::BaseType>(base_type),
+      *builder,
+      static_cast<reflection::BaseType>(base_type),
       static_cast<reflection::BaseType>(element),
       struct_def ? struct_def->index : (enum_def ? enum_def->index : -1));
 }
@@ -2614,7 +2664,7 @@ Definition::SerializeAttributes(FlatBufferBuilder *builder,
   for (auto kv = attributes.dict.begin(); kv != attributes.dict.end(); ++kv) {
     auto it = parser.known_attributes_.find(kv->first);
     FLATBUFFERS_ASSERT(it != parser.known_attributes_.end());
-    if (!it->second) {  // Custom attribute.
+    if (parser.opts.binary_schema_builtins || !it->second) {
       attrs.push_back(reflection::CreateKeyValue(
           *builder, builder->CreateString(kv->first),
           builder->CreateString(kv->second->constant)));

tests/namespace_test/namespace_test2_generated.ts

 // automatically generated by the FlatBuffers compiler, do not modify
 
-import * as NS11563891686210618450 from "./namespace_test1_generated";
+import * as NS9459827973991502386 from "./namespace_test1_generated";
 /**
  * @constructor
  */
@@ -39,24 +39,24 @@ static getRootAsTableInFirstNS(bb:flatbuffers.ByteBuffer, obj?:TableInFirstNS):T
  * @param {NamespaceA.NamespaceB.TableInNestedNS=} obj
  * @returns {NamespaceA.NamespaceB.TableInNestedNS|null}
  */
-fooTable(obj?:NS11563891686210618450.NamespaceA.NamespaceB.TableInNestedNS):NS11563891686210618450.NamespaceA.NamespaceB.TableInNestedNS|null {
+fooTable(obj?:NS9459827973991502386.NamespaceA.NamespaceB.TableInNestedNS):NS9459827973991502386.NamespaceA.NamespaceB.TableInNestedNS|null {
   var offset = this.bb!.__offset(this.bb_pos, 4);
-  return offset ? (obj || new NS11563891686210618450.NamespaceA.NamespaceB.TableInNestedNS).__init(this.bb!.__indirect(this.bb_pos + offset), this.bb!) : null;
+  return offset ? (obj || new NS9459827973991502386.NamespaceA.NamespaceB.TableInNestedNS).__init(this.bb!.__indirect(this.bb_pos + offset), this.bb!) : null;
 };
 
 /**
  * @returns {NamespaceA.NamespaceB.EnumInNestedNS}
  */
-fooEnum():NS11563891686210618450.NamespaceA.NamespaceB.EnumInNestedNS {
+fooEnum():NS9459827973991502386.NamespaceA.NamespaceB.EnumInNestedNS {
   var offset = this.bb!.__offset(this.bb_pos, 6);
-  return offset ? /** @type {NamespaceA.NamespaceB.EnumInNestedNS} */ (this.bb!.readInt8(this.bb_pos + offset)) : NS11563891686210618450.NamespaceA.NamespaceB.EnumInNestedNS.A;
+  return offset ? /** @type {NamespaceA.NamespaceB.EnumInNestedNS} */ (this.bb!.readInt8(this.bb_pos + offset)) : NS9459827973991502386.NamespaceA.NamespaceB.EnumInNestedNS.A;
 };
 
 /**
  * @param {NamespaceA.NamespaceB.EnumInNestedNS} value
  * @returns {boolean}
  */
-mutate_foo_enum(value:NS11563891686210618450.NamespaceA.NamespaceB.EnumInNestedNS):boolean {
+mutate_foo_enum(value:NS9459827973991502386.NamespaceA.NamespaceB.EnumInNestedNS):boolean {
   var offset = this.bb!.__offset(this.bb_pos, 6);
 
   if (offset === 0) {
@@ -71,9 +71,9 @@ mutate_foo_enum(value:NS11563891686210618450.NamespaceA.NamespaceB.EnumInNestedN
  * @param {NamespaceA.NamespaceB.StructInNestedNS=} obj
  * @returns {NamespaceA.NamespaceB.StructInNestedNS|null}
  */
-fooStruct(obj?:NS11563891686210618450.NamespaceA.NamespaceB.StructInNestedNS):NS11563891686210618450.NamespaceA.NamespaceB.StructInNestedNS|null {
+fooStruct(obj?:NS9459827973991502386.NamespaceA.NamespaceB.StructInNestedNS):NS9459827973991502386.NamespaceA.NamespaceB.StructInNestedNS|null {
   var offset = this.bb!.__offset(this.bb_pos, 8);
-  return offset ? (obj || new NS11563891686210618450.NamespaceA.NamespaceB.StructInNestedNS).__init(this.bb_pos + offset, this.bb!) : null;
+  return offset ? (obj || new NS9459827973991502386.NamespaceA.NamespaceB.StructInNestedNS).__init(this.bb_pos + offset, this.bb!) : null;
 };
 
 /**
@@ -95,8 +95,8 @@ static addFooTable(builder:flatbuffers.Builder, fooTableOffset:flatbuffers.Offse
  * @param {flatbuffers.Builder} builder
  * @param {NamespaceA.NamespaceB.EnumInNestedNS} fooEnum
  */
-static addFooEnum(builder:flatbuffers.Builder, fooEnum:NS11563891686210618450.NamespaceA.NamespaceB.EnumInNestedNS) {
-  builder.addFieldInt8(1, fooEnum, NS11563891686210618450.NamespaceA.NamespaceB.EnumInNestedNS.A);
+static addFooEnum(builder:flatbuffers.Builder, fooEnum:NS9459827973991502386.NamespaceA.NamespaceB.EnumInNestedNS) {
+  builder.addFieldInt8(1, fooEnum, NS9459827973991502386.NamespaceA.NamespaceB.EnumInNestedNS.A);
 };
 
 /**