Enable flatbuffer to initialize Parser from bfbs (#4283) (#5077)

* Enable flatbuffer to initialize Parser from bfbs (#4283)

Now its possible to generate json data from bfbs data type and flatbuffers data
and visa versa.

* add deserialize functionality in parser from bfbs
* add small usage sample

* Fix build break

* Merge branch 'pr/1' into fix-issue4283

* Fix buildbreak

* Build monster_test.bfbs with --bfbs-builtins

Attribute flexbuffer has be included in bfbs. Only with this attribute test
will run. By initialization a parser by a bfbs the attribute has to be known
for this filed. monsterdata_test.golden has a flexbuffer field so parse would
fail.

* Fix generate_code.sh

* Revert automatic indent changes by IDE

* Auto detect size prefixed binary schema files

* Use identifier (bfbs) to detect schema files

.gitignore

@@ -46,6 +46,8 @@ flatsamplebinary
 flatsamplebinary.exe
 flatsampletext
 flatsampletext.exe
+flatsamplebfbs
+flatsamplebfbs.exe
 grpctest
 grpctest.exe
 snapshot.sh

CMakeLists.txt

@@ -123,6 +123,14 @@ set(FlatBuffers_Sample_Text_SRCS
   ${CMAKE_CURRENT_BINARY_DIR}/samples/monster_generated.h
 )
 
+set(FlatBuffers_Sample_BFBS_SRCS
+  ${FlatBuffers_Library_SRCS}
+  src/idl_gen_general.cpp
+  samples/sample_bfbs.cpp
+  # file generated by running compiler on samples/monster.fbs
+  ${CMAKE_CURRENT_BINARY_DIR}/samples/monster_generated.h
+)
+
 set(FlatBuffers_GRPCTest_SRCS
   include/flatbuffers/flatbuffers.h
   include/flatbuffers/grpc.h
@@ -316,6 +324,7 @@ if(FLATBUFFERS_BUILD_TESTS)
   include_directories(${CMAKE_CURRENT_BINARY_DIR}/samples)
   add_executable(flatsamplebinary ${FlatBuffers_Sample_Binary_SRCS})
   add_executable(flatsampletext ${FlatBuffers_Sample_Text_SRCS})
+  add_executable(flatsamplebfbs ${FlatBuffers_Sample_BFBS_SRCS})
 endif()
 
 if(FLATBUFFERS_BUILD_GRPCTEST)

include/flatbuffers/flatc.h

@@ -80,6 +80,9 @@ class FlatCompiler {
                  const std::string &contents,
                  std::vector<const char *> &include_directories) const;
 
+  void LoadBinarySchema(Parser &parser, const std::string &filename,
+                        const std::string &contents);
+
   void Warn(const std::string &warn, bool show_exe_name = true) const;
 
   void Error(const std::string &err, bool usage = true,

include/flatbuffers/idl.h

@@ -153,6 +153,8 @@ struct Type {
 
   Offset<reflection::Type> Serialize(FlatBufferBuilder *builder) const;
 
+  bool Deserialize(const Parser &parser, const reflection::Type *type);
+
   BaseType base_type;
   BaseType element;       // only set if t == BASE_TYPE_VECTOR
   StructDef *struct_def;  // only set if t or element == BASE_TYPE_STRUCT
@@ -235,6 +237,9 @@ struct Definition {
       flatbuffers::Vector<flatbuffers::Offset<reflection::KeyValue>>>
   SerializeAttributes(FlatBufferBuilder *builder, const Parser &parser) const;
 
+  bool DeserializeAttributes(Parser &parser,
+                             const Vector<Offset<reflection::KeyValue>> *attrs);
+
   std::string name;
   std::string file;
   std::vector<std::string> doc_comment;
@@ -261,6 +266,8 @@ struct FieldDef : public Definition {
   Offset<reflection::Field> Serialize(FlatBufferBuilder *builder, uint16_t id,
                                       const Parser &parser) const;
 
+  bool Deserialize(Parser &parser, const reflection::Field *field);
+
   Value value;
   bool deprecated;  // Field is allowed to be present in old data, but can't be.
                     // written in new data nor accessed in new code.
@@ -291,6 +298,8 @@ struct StructDef : public Definition {
   Offset<reflection::Object> Serialize(FlatBufferBuilder *builder,
                                        const Parser &parser) const;
 
+  bool Deserialize(Parser &parser, const reflection::Object *object);
+
   SymbolTable<FieldDef> fields;
 
   bool fixed;       // If it's struct, not a table.
@@ -317,9 +326,12 @@ inline size_t InlineAlignment(const Type &type) {
 
 struct EnumVal {
   EnumVal(const std::string &_name, int64_t _val) : name(_name), value(_val) {}
+  EnumVal(){};
 
   Offset<reflection::EnumVal> Serialize(FlatBufferBuilder *builder, const Parser &parser) const;
 
+  bool Deserialize(const Parser &parser, const reflection::EnumVal *val);
+
   std::string name;
   std::vector<std::string> doc_comment;
   int64_t value;
@@ -340,6 +352,8 @@ struct EnumDef : public Definition {
 
   Offset<reflection::Enum> Serialize(FlatBufferBuilder *builder, const Parser &parser) const;
 
+  bool Deserialize(Parser &parser, const reflection::Enum *values);
+
   SymbolTable<EnumVal> vals;
   bool is_union;
   // Type is a union which uses type aliases where at least one type is
@@ -358,11 +372,14 @@ inline bool EqualByName(const Type &a, const Type &b) {
 struct RPCCall : public Definition {
   Offset<reflection::RPCCall> Serialize(FlatBufferBuilder *builder, const Parser &parser) const;
 
+  bool Deserialize(Parser &parser, const reflection::RPCCall *call);
+
   StructDef *request, *response;
 };
 
 struct ServiceDef : public Definition {
   Offset<reflection::Service> Serialize(FlatBufferBuilder *builder, const Parser &parser) const;
+  bool Deserialize(Parser &parser, const reflection::Service *service);
 
   SymbolTable<RPCCall> calls;
 };
@@ -647,6 +664,15 @@ class Parser : public ParserState {
   // See reflection/reflection.fbs
   void Serialize();
 
+  // Deserialize a schema buffer
+  bool Deserialize(const uint8_t *buf, const size_t size);
+
+  // Fills internal structure as if the schema passed had been loaded by parsing
+  // with Parse except that included filenames will not be populated.
+  bool Deserialize(const reflection::Schema* schema);
+
+  Type* DeserializeType(const reflection::Type* type);
+  
   // Checks that the schema represented by this parser is a safe evolution
   // of the schema provided. Returns non-empty error on any problems.
   std::string ConformTo(const Parser &base);
@@ -661,6 +687,8 @@ class Parser : public ParserState {
 
   StructDef *LookupStruct(const std::string &id) const;
 
+  std::string UnqualifiedName(std::string fullQualifiedName);
+
  private:
   void Message(const std::string &msg);
   void Warning(const std::string &msg);

samples/sample_bfbs.cpp

+/*
+ * Copyright 2014 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "flatbuffers/idl.h"
+#include "flatbuffers/util.h"
+
+#include "monster_test_generated.h"
+#include "monster_generated.h" // Already includes "flatbuffers/flatbuffers.h".
+
+using namespace MyGame::Sample;
+
+// This is an example of parsing text straight into a buffer and then
+// generating flatbuffer (JSON) text from the buffer.
+int main(int /*argc*/, const char * /*argv*/[]) {
+  // load FlatBuffer schema (.fbs) and JSON from disk
+  std::string schema_file;
+  std::string json_file;
+  std::string bfbs_file;
+  bool ok =
+      flatbuffers::LoadFile("tests/monster_test.fbs", false, &schema_file) &&
+      flatbuffers::LoadFile("tests/monsterdata_test.golden", false, &json_file) &&
+      flatbuffers::LoadFile("tests/monster_test.bfbs", true, &bfbs_file);
+  if (!ok) {
+    printf("couldn't load files!\n");
+    return 1;
+  }
+
+  const char *include_directories[] = { "samples", "tests",
+                                        "tests/include_test", nullptr };
+  // parse fbs schema
+  flatbuffers::Parser parser1;
+  ok = parser1.Parse(schema_file.c_str(), include_directories);
+  assert(ok);
+  
+  // inizialize parser by deserializing bfbs schema
+  flatbuffers::Parser parser2;
+  ok = parser2.Deserialize((uint8_t *)bfbs_file.c_str(), bfbs_file.length());
+  assert(ok);
+
+  // parse json in parser from fbs and bfbs
+  ok = parser1.Parse(json_file.c_str(), include_directories);
+  ok = parser2.Parse(json_file.c_str(), include_directories);
+  assert(ok);
+
+  // to ensure it is correct, we now generate text back from the binary,
+  // and compare the two:
+  std::string jsongen1;
+  if (!GenerateText(parser1, parser1.builder_.GetBufferPointer(), &jsongen1)) {
+    printf("Couldn't serialize parsed data to JSON!\n");
+    return 1;
+  }
+
+  std::string jsongen2;
+  if (!GenerateText(parser2, parser2.builder_.GetBufferPointer(), &jsongen2)) {
+    printf("Couldn't serialize parsed data to JSON!\n");
+    return 1;
+  }
+
+  if (jsongen1 != jsongen2) {
+    printf("%s----------------\n%s", jsongen1.c_str(), jsongen2.c_str());
+  }
+
+  printf("The FlatBuffer has been parsed from JSON successfully.\n");
+}

src/flatc.cpp

@@ -36,6 +36,15 @@ void FlatCompiler::ParseFile(
   include_directories.pop_back();
 }
 
+void FlatCompiler::LoadBinarySchema(flatbuffers::Parser &parser,
+                                    const std::string &filename,
+                                    const std::string &contents) {
+  if (!parser.Deserialize(reinterpret_cast<const uint8_t *>(contents.c_str()),
+      contents.size())) {
+    Error("failed to load binary schema: " + filename, false, false);
+  }
+}
+
 void FlatCompiler::Warn(const std::string &warn, bool show_exe_name) const {
   params_.warn_fn(this, warn, show_exe_name);
 }
@@ -127,8 +136,9 @@ std::string FlatCompiler::GetUsageString(const char *program_name) const {
     "  --force-defaults   Emit default values in binary output from JSON\n"
     "  --force-empty      When serializing from object API representation,\n"
     "                     force strings and vectors to empty rather than null.\n"
-    "FILEs may be schemas (must end in .fbs), or JSON files (conforming to preceding\n"
-    "schema). FILEs after the -- must be binary flatbuffer format files.\n"
+    "FILEs may be schemas (must end in .fbs), binary schemas (must end in .bfbs),\n"
+    "or JSON files (conforming to preceding schema). FILEs after the -- must be\n"
+    "binary flatbuffer format files.\n"
     "Output files are named using the base file name of the input,\n"
     "and written to the current directory or the path given by -o.\n"
     "example: " << program_name << " -c -b schema1.fbs schema2.fbs data.json\n";
@@ -323,8 +333,14 @@ int FlatCompiler::Compile(int argc, const char **argv) {
     std::string contents;
     if (!flatbuffers::LoadFile(conform_to_schema.c_str(), true, &contents))
       Error("unable to load schema: " + conform_to_schema);
-    ParseFile(conform_parser, conform_to_schema, contents,
-              conform_include_directories);
+
+    if (flatbuffers::GetExtension(conform_to_schema) ==
+        reflection::SchemaExtension()) {
+      LoadBinarySchema(conform_parser, conform_to_schema, contents);
+    } else {
+      ParseFile(conform_parser, conform_to_schema, contents,
+                conform_include_directories);
+    }
   }
 
   std::unique_ptr<flatbuffers::Parser> parser(new flatbuffers::Parser(opts));
@@ -340,6 +356,7 @@ int FlatCompiler::Compile(int argc, const char **argv) {
         static_cast<size_t>(file_it - filenames.begin()) >= binary_files_from;
     auto ext = flatbuffers::GetExtension(filename);
     auto is_schema = ext == "fbs" || ext == "proto";
+    auto is_binary_schema = ext == reflection::SchemaExtension();
     if (is_binary) {
       parser->builder_.Clear();
       parser->builder_.PushFlatBuffer(
@@ -366,7 +383,7 @@ int FlatCompiler::Compile(int argc, const char **argv) {
       }
     } else {
       // Check if file contains 0 bytes.
-      if (contents.length() != strlen(contents.c_str())) {
+      if (!is_binary_schema && contents.length() != strlen(contents.c_str())) {
         Error("input file appears to be binary: " + filename, true);
       }
       if (is_schema) {
@@ -375,15 +392,19 @@ int FlatCompiler::Compile(int argc, const char **argv) {
         // so explicitly using an include.
         parser.reset(new flatbuffers::Parser(opts));
       }
-      ParseFile(*parser.get(), filename, contents, include_directories);
-      if (!is_schema && !parser->builder_.GetSize()) {
-        // If a file doesn't end in .fbs, it must be json/binary. Ensure we
-        // didn't just parse a schema with a different extension.
-        Error(
-            "input file is neither json nor a .fbs (schema) file: " + filename,
-            true);
+      if (is_binary_schema) {
+        LoadBinarySchema(*parser.get(), filename, contents);
+      } else {
+        ParseFile(*parser.get(), filename, contents, include_directories);
+        if (!is_schema && !parser->builder_.GetSize()) {
+          // If a file doesn't end in .fbs, it must be json/binary. Ensure we
+          // didn't just parse a schema with a different extension.
+          Error("input file is neither json nor a .fbs (schema) file: " +
+                    filename,
+                true);
+        }
       }
-      if (is_schema && !conform_to_schema.empty()) {
+      if ((is_schema || is_binary_schema) && !conform_to_schema.empty()) {
         auto err = parser->ConformTo(conform_parser);
         if (!err.empty()) Error("schemas don\'t conform: " + err);
       }
@@ -401,7 +422,8 @@ int FlatCompiler::Compile(int argc, const char **argv) {
       if (generator_enabled[i]) {
         if (!print_make_rules) {
           flatbuffers::EnsureDirExists(output_path);
-          if ((!params_.generators[i].schema_only || is_schema) &&
+          if ((!params_.generators[i].schema_only ||
+               (is_schema || is_binary_schema)) &&
               !params_.generators[i].generate(*parser.get(), output_path,
                                               filebase)) {
             Error(std::string("Unable to generate ") +

tests/generate_code.bat

@@ -18,10 +18,11 @@ if "%1"=="-b" set buildtype=%2
 ..\%buildtype%\flatc.exe --cpp --java --csharp --go --binary --python --lobster --lua --js --rust --ts --php --grpc --gen-mutable --reflect-names --gen-object-api --gen-compare --no-includes --cpp-ptr-type flatbuffers::unique_ptr --no-fb-import -I include_test monster_test.fbs monsterdata_test.json || goto FAIL
 ..\%buildtype%\flatc.exe --cpp --java --csharp --go --binary --python --lobster --lua --js --rust --ts --php --gen-mutable --reflect-names --no-fb-import --cpp-ptr-type flatbuffers::unique_ptr  -o namespace_test namespace_test/namespace_test1.fbs namespace_test/namespace_test2.fbs || goto FAIL
 ..\%buildtype%\flatc.exe --cpp --js --ts --php --gen-mutable --reflect-names --gen-object-api --gen-compare --cpp-ptr-type flatbuffers::unique_ptr -o union_vector ./union_vector/union_vector.fbs || goto FAIL
-..\%buildtype%\flatc.exe -b --schema --bfbs-comments -I include_test monster_test.fbs || goto FAIL
+..\%buildtype%\flatc.exe -b --schema --bfbs-comments --bfbs-builtins -I include_test monster_test.fbs || goto FAIL
 ..\%buildtype%\flatc.exe --jsonschema --schema -I include_test monster_test.fbs || goto FAIL
 cd ../samples
 ..\%buildtype%\flatc.exe --cpp --lobster --gen-mutable --reflect-names --gen-object-api --gen-compare --cpp-ptr-type flatbuffers::unique_ptr monster.fbs || goto FAIL
+..\%buildtype%\flatc.exe -b --schema --bfbs-comments --bfbs-builtins monster.fbs || goto FAIL
 cd ../reflection
 call generate_code.bat %1 %2 || goto FAIL
 

tests/generate_code.sh

@@ -18,9 +18,10 @@ set -e
 ../flatc --cpp --java --csharp --dart --go --binary --lobster --lua --python --js --ts --php --rust --grpc --gen-mutable --reflect-names --gen-object-api --gen-compare --no-includes --cpp-ptr-type flatbuffers::unique_ptr  --no-fb-import -I include_test monster_test.fbs monsterdata_test.json
 ../flatc --cpp --java --csharp --dart --go --binary --lobster --lua --python --js --ts --php --rust --gen-mutable --reflect-names --no-fb-import --cpp-ptr-type flatbuffers::unique_ptr  -o namespace_test namespace_test/namespace_test1.fbs namespace_test/namespace_test2.fbs
 ../flatc --cpp --js --ts --php --gen-mutable --reflect-names --gen-object-api --gen-compare --cpp-ptr-type flatbuffers::unique_ptr -o union_vector ./union_vector/union_vector.fbs
-../flatc -b --schema --bfbs-comments -I include_test monster_test.fbs
+../flatc -b --schema --bfbs-comments --bfbs-builtins -I include_test monster_test.fbs
 ../flatc --jsonschema --schema -I include_test monster_test.fbs
 cd ../samples
 ../flatc --cpp --lobster --gen-mutable --reflect-names --gen-object-api --gen-compare --cpp-ptr-type flatbuffers::unique_ptr monster.fbs
+../flatc -b --schema --bfbs-comments --bfbs-builtins monster.fbs
 cd ../reflection
 ./generate_code.sh

tests/test.cpp

@@ -569,25 +569,37 @@ void JsonDefaultTest() {
 
 // example of parsing text straight into a buffer, and generating
 // text back from it:
-void ParseAndGenerateTextTest() {
+void ParseAndGenerateTextTest(bool binary) {
   // load FlatBuffer schema (.fbs) and JSON from disk
   std::string schemafile;
   std::string jsonfile;
-  TEST_EQ(flatbuffers::LoadFile((test_data_path + "monster_test.fbs").c_str(),
-                                false, &schemafile),
+  TEST_EQ(flatbuffers::LoadFile(
+              (test_data_path + "monster_test." + (binary ? "bfbs" : "fbs"))
+                  .c_str(),
+              binary, &schemafile),
           true);
   TEST_EQ(flatbuffers::LoadFile(
               (test_data_path + "monsterdata_test.golden").c_str(), false,
               &jsonfile),
           true);
 
-  // parse schema first, so we can use it to parse the data after
-  flatbuffers::Parser parser;
   auto include_test_path =
-      flatbuffers::ConCatPathFileName(test_data_path, "include_test");
+    flatbuffers::ConCatPathFileName(test_data_path, "include_test");
   const char *include_directories[] = { test_data_path.c_str(),
                                         include_test_path.c_str(), nullptr };
-  TEST_EQ(parser.Parse(schemafile.c_str(), include_directories), true);
+
+  // parse schema first, so we can use it to parse the data after
+  flatbuffers::Parser parser;
+  if (binary) {
+    flatbuffers::Verifier verifier(
+        reinterpret_cast<const uint8_t *>(schemafile.c_str()),
+        schemafile.size());
+    TEST_EQ(reflection::VerifySchemaBuffer(verifier), true);
+    //auto schema = reflection::GetSchema(schemafile.c_str());
+    TEST_EQ(parser.Deserialize((const uint8_t *)schemafile.c_str(), schemafile.size()), true);
+  } else {
+    TEST_EQ(parser.Parse(schemafile.c_str(), include_directories), true);
+  }
   TEST_EQ(parser.Parse(jsonfile.c_str(), include_directories), true);
 
   // here, parser.builder_ contains a binary buffer that is the parsed data.
@@ -2467,7 +2479,8 @@ int FlatBufferTests() {
       test_data_path = FLATBUFFERS_STRING(FLATBUFFERS_TEST_PATH_PREFIX) +
                        test_data_path;
     #endif
-    ParseAndGenerateTextTest();
+    ParseAndGenerateTextTest(false);
+    ParseAndGenerateTextTest(true);
     ReflectionTest(flatbuf.data(), flatbuf.size());
     ParseProtoTest();
     UnionVectorTest();