merge

.gitignore

@@ -42,6 +42,7 @@ tests/monsterdata_go_wire.mon
 CMakeLists.txt.user
 CMakeScripts/**
 CTestTestfile.cmake
+FlatBuffers.cbp
 build/Xcode/FlatBuffers.xcodeproj/project.xcworkspace/**
 build/Xcode/FlatBuffers.xcodeproj/xcuserdata/**
 FlatBuffers.xcodeproj/

CMakeLists.txt

@@ -20,6 +20,8 @@ set(FlatBuffers_Compiler_SRCS
   include/flatbuffers/hash.h
   include/flatbuffers/idl.h
   include/flatbuffers/util.h
+  include/flatbuffers/reflection.h
+  include/flatbuffers/reflection_generated.h
   src/idl_parser.cpp
   src/idl_gen_cpp.cpp
   src/idl_gen_general.cpp
@@ -108,7 +110,16 @@ function(compile_flatbuffers_schema_to_cpp SRC_FBS)
   string(REGEX REPLACE "\\.fbs$" "_generated.h" GEN_HEADER ${SRC_FBS})
   add_custom_command(
     OUTPUT ${GEN_HEADER}
-    COMMAND flatc -c --gen-mutable -o "${SRC_FBS_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/${SRC_FBS}"
+    COMMAND flatc -c --no-includes --gen-mutable -o "${SRC_FBS_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/${SRC_FBS}"
+    DEPENDS flatc)
+endfunction()
+
+function(compile_flatbuffers_schema_to_binary SRC_FBS)
+  get_filename_component(SRC_FBS_DIR ${SRC_FBS} PATH)
+  string(REGEX REPLACE "\\.fbs$" ".bfbs" GEN_BINARY_SCHEMA ${SRC_FBS})
+  add_custom_command(
+    OUTPUT ${GEN_BINARY_SCHEMA}
+    COMMAND flatc -b --schema -o "${SRC_FBS_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/${SRC_FBS}"
     DEPENDS flatc)
 endfunction()
 

build/VS2010/flattests.vcxproj

@@ -265,6 +265,7 @@
   <ItemGroup>
     <ClInclude Include="..\..\include\flatbuffers\flatbuffers.h" />
     <ClInclude Include="..\..\include\flatbuffers\idl.h" />
+    <ClInclude Include="..\..\include\flatbuffers\reflection.h" />
     <ClInclude Include="..\..\include\flatbuffers\util.h" />
     <ClInclude Include="..\..\tests\monster_test_generated.h" />
     <ClCompile Include="..\..\src\idl_gen_fbs.cpp" />
@@ -276,4 +277,4 @@
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>
-</Project>
+</Project>
\ No newline at end of file

docs/html/index.html

@@ -54,7 +54,7 @@ $(document).ready(function(){initNavTree('index.html','');});
 </div><!--header-->
 <div class="contents">
 <div class="textblock"><p>FlatBuffers is an efficient cross platform serialization library for C++, with support for Java, C# and Go. It was created at Google specifically for game development and other performance-critical applications.</p>
-<p>It is available as open source under the Apache license, v2 (see LICENSE.txt).</p>
+<p>It is available as Open Source on <a href="http://github.com/google/flatbuffers">GitHub</a> under the Apache license, v2 (see LICENSE.txt).</p>
 <h2>Why use FlatBuffers?</h2>
 <ul>
 <li><b>Access to serialized data without parsing/unpacking</b> - What sets FlatBuffers apart is that it represents hierarchical data in a flat binary buffer in such a way that it can still be accessed directly without parsing/unpacking, while also still supporting data structure evolution (forwards/backwards compatibility).</li>
@@ -96,8 +96,8 @@ $(document).ready(function(){initNavTree('index.html','');});
 </ul>
 <h2>Online resources</h2>
 <ul>
-<li><a href="http://github.com/google/flatbuffers">github repository</a></li>
-<li><a href="http://google.github.io/flatbuffers">landing page</a></li>
+<li><a href="http://github.com/google/flatbuffers">GitHub repository</a></li>
+<li><a href="http://google.github.io/flatbuffers">Landing page</a></li>
 <li><a href="http://group.google.com/group/flatbuffers">FlatBuffers Google Group</a></li>
 <li><a href="http://github.com/google/flatbuffers/issues">FlatBuffers Issues Tracker</a> </li>
 </ul>

docs/html/md__compiler.html

@@ -68,11 +68,14 @@ $(document).ready(function(){initNavTree('md__compiler.html','');});
 <li><code>-o PATH</code> : Output all generated files to PATH (either absolute, or relative to the current directory). If omitted, PATH will be the current directory. PATH should end in your systems path separator, e.g. <code>/</code> or <code>\</code>.</li>
 <li><code>-I PATH</code> : when encountering <code>include</code> statements, attempt to load the files from this path. Paths will be tried in the order given, and if all fail (or none are specified) it will try to load relative to the path of the schema file being parsed.</li>
 <li><code>--strict-json</code> : Require &amp; generate strict JSON (field names are enclosed in quotes, no trailing commas in tables/vectors). By default, no quotes are required/generated, and trailing commas are allowed.</li>
+<li><code>--defaults-json</code> : Output fields whose value is equal to the default value when writing JSON text.</li>
 <li><code>--no-prefix</code> : Don't prefix enum values in generated C++ by their enum type.</li>
-<li><code>--gen-includes</code> : Generate include statements for included schemas the generated file depends on (C++).</li>
+<li><code>--gen-includes</code> : (deprecated), instead use:</li>
+<li><code>--no-includes</code> : Don't generate include statements for included schemas the generated file depends on (C++).</li>
 <li><code>--gen-mutable</code> : Generate additional non-const accessors for mutating FlatBuffers in-place.</li>
 <li><code>--raw-binary</code> : Allow binaries without a file_indentifier to be read. This may crash flatc given a mismatched schema.</li>
-<li><code>--proto</code>: Expect input files to be .proto files (protocol buffers). Output the corresponding .fbs file. Currently supports: <code>package</code>, <code>message</code>, <code>enum</code>. Does not support, but will skip without error: <code>import</code>, <code>option</code>. Does not support, will generate error: <code>service</code>, <code>extend</code>, <code>extensions</code>, <code>oneof</code>, <code>group</code>, custom options, nested declarations. </li>
+<li><code>--proto</code>: Expect input files to be .proto files (protocol buffers). Output the corresponding .fbs file. Currently supports: <code>package</code>, <code>message</code>, <code>enum</code>. Does not support, but will skip without error: <code>import</code>, <code>option</code>. Does not support, will generate error: <code>service</code>, <code>extend</code>, <code>extensions</code>, <code>oneof</code>, <code>group</code>, custom options, nested declarations.</li>
+<li><code>--schema</code>: Serialize schemas instead of JSON (use with -b). This will output a binary version of the specified schema that itself corresponds to the reflection/reflection.fbs schema. Loading this binary file is the basis for reflection functionality. </li>
 </ul>
 </div></div><!-- contents -->
 </div><!-- doc-content -->

docs/html/md__cpp_usage.html

@@ -114,8 +114,14 @@ $(document).ready(function(){initNavTree('md__cpp_usage.html','');});
 </div><!-- fragment --><p>We use the somewhat verbose term <code>mutate</code> instead of <code>set</code> to indicate that this is a special use case, not to be confused with the default way of constructing FlatBuffer data.</p>
 <p>After the above mutations, you can send on the FlatBuffer to a new recipient without any further work!</p>
 <p>Note that any <code>mutate_</code> functions on tables return a bool, which is false if the field we're trying to set isn't present in the buffer. Fields are not present if they weren't set, or even if they happen to be equal to the default value. For example, in the creation code above we set the <code>mana</code> field to <code>150</code>, which is the default value, so it was never stored in the buffer. Trying to call mutate_mana() on such data will return false, and the value won't actually be modified!</p>
-<p>There's two ways around this. First, you can call <code>ForceDefaults()</code> on a <code>FlatBufferBuilder</code> to force all fields you set to actually be written. This of course increases the size of the buffer somewhat, but this may be acceptable for a mutable buffer.</p>
-<p>Alternatively, you can use mutation functions that are able to insert fields and change the size of things. These functions are expensive however, since they need to resize the buffer and create new data.</p>
+<p>One way to solve this is to call <code>ForceDefaults()</code> on a <code>FlatBufferBuilder</code> to force all fields you set to actually be written. This of course increases the size of the buffer somewhat, but this may be acceptable for a mutable buffer.</p>
+<p>Alternatively, you can use the more powerful reflection functionality:</p>
+<h3>Reflection (&amp; Resizing)</h3>
+<p>If the above ways of accessing a buffer are still too static for you, there is experimental support for reflection in FlatBuffers, allowing you to read and write data even if you don't know the exact format of a buffer, and even allows you to change sizes of strings and vectors in-place.</p>
+<p>The way this works is very elegant, there is actually a FlatBuffer schema that describes schemas (!) which you can find in <code>reflection/reflection.fbs</code>. The compiler <code>flatc</code> can write out any schemas it has just parsed as a binary FlatBuffer, corresponding to this meta-schema.</p>
+<p>Loading in one of these binary schemas at runtime allows you traverse any FlatBuffer data that corresponds to it without knowing the exact format. You can query what fields are present, and then read/write them after.</p>
+<p>For convenient field manipulation, you can include the header <code>flatbuffers/reflection.h</code> which includes both the generated code from the meta schema, as well as a lot of helper functions.</p>
+<p>And example of usage for the moment you can find in <code>test.cpp/ReflectionTest()</code>.</p>
 <h3>Storing maps / dictionaries in a FlatBuffer</h3>
 <p>FlatBuffers doesn't support maps natively, but there is support to emulate their behavior with vectors and binary search, which means you can have fast lookups directly from a FlatBuffer without having to unpack your data into a <code>std::map</code> or similar.</p>
 <p>To use it:</p><ul>

docs/source/Compiler.md

@@ -51,7 +51,8 @@ be generated for each file processed:
 -   `--no-prefix` : Don't prefix enum values in generated C++ by their enum
     type.
 
--   `--gen-includes` : Generate include statements for included schemas the
+-   `--gen-includes` : (deprecated), instead use:
+-   `--no-includes` : Don't generate include statements for included schemas the
     generated file depends on (C++).
 
 -   `--gen-mutable` : Generate additional non-const accessors for mutating
@@ -68,3 +69,8 @@ be generated for each file processed:
     Does not support, but will skip without error: `import`, `option`.
     Does not support, will generate error: `service`, `extend`, `extensions`,
     `oneof`, `group`, custom options, nested declarations.
+
+-   `--schema`: Serialize schemas instead of JSON (use with -b). This will
+    output a binary version of the specified schema that itself corresponds
+    to the reflection/reflection.fbs schema. Loading this binary file is the
+    basis for reflection functionality.

docs/source/CppUsage.md

@@ -203,14 +203,34 @@ to `150`, which is the default value, so it was never stored in the buffer.
 Trying to call mutate_mana() on such data will return false, and the value won't
 actually be modified!
 
-There's two ways around this. First, you can call `ForceDefaults()` on a
+One way to solve this is to call `ForceDefaults()` on a
 `FlatBufferBuilder` to force all fields you set to actually be written. This
 of course increases the size of the buffer somewhat, but this may be
 acceptable for a mutable buffer.
 
-Alternatively, you can use mutation functions that are able to insert fields
-and change the size of things. These functions are expensive however, since
-they need to resize the buffer and create new data.
+Alternatively, you can use the more powerful reflection functionality:
+
+### Reflection (& Resizing)
+
+If the above ways of accessing a buffer are still too static for you, there is
+experimental support for reflection in FlatBuffers, allowing you to read and
+write data even if you don't know the exact format of a buffer, and even allows
+you to change sizes of strings and vectors in-place.
+
+The way this works is very elegant, there is actually a FlatBuffer schema that
+describes schemas (!) which you can find in `reflection/reflection.fbs`.
+The compiler `flatc` can write out any schemas it has just parsed as a binary
+FlatBuffer, corresponding to this meta-schema.
+
+Loading in one of these binary schemas at runtime allows you traverse any
+FlatBuffer data that corresponds to it without knowing the exact format. You
+can query what fields are present, and then read/write them after.
+
+For convenient field manipulation, you can include the header
+`flatbuffers/reflection.h` which includes both the generated code from the meta
+schema, as well as a lot of helper functions.
+
+And example of usage for the moment you can find in `test.cpp/ReflectionTest()`.
 
 ### Storing maps / dictionaries in a FlatBuffer
 

include/flatbuffers/flatbuffers.h

@@ -754,7 +754,7 @@ class FlatBufferBuilder FLATBUFFERS_FINAL_CLASS {
   }
 
   template<typename T> Offset<Vector<Offset<T>>> CreateVectorOfSortedTables(
-                                                            std::vector<T> *v) {
+                                                    std::vector<Offset<T>> *v) {
     return CreateVectorOfSortedTables(v->data(), v->size());
   }
 
@@ -1017,6 +1017,8 @@ class Table {
     return true;
   }
 
+  uint8_t *GetVTable() { return data_ - ReadScalar<soffset_t>(data_); }
+
   bool CheckField(voffset_t field) const {
     return GetOptionalFieldOffset(field) != 0;
   }

include/flatbuffers/idl.h

@@ -25,6 +25,7 @@
 
 #include "flatbuffers/flatbuffers.h"
 #include "flatbuffers/hash.h"
+#include "flatbuffers/reflection.h"
 
 // This file defines the data types representing a parsed IDL (Interface
 // Definition Language) / schema file.
@@ -126,6 +127,8 @@ struct Type {
 
   Type VectorType() const { return Type(element, struct_def, enum_def); }
 
+  Offset<reflection::Type> Serialize(FlatBufferBuilder *builder) const;
+
   BaseType base_type;
   BaseType element;       // only set if t == BASE_TYPE_VECTOR
   StructDef *struct_def;  // only set if t or element == BASE_TYPE_STRUCT
@@ -179,7 +182,8 @@ struct Namespace {
 
 // Base class for all definition types (fields, structs_, enums_).
 struct Definition {
-  Definition() : generated(false), defined_namespace(nullptr) {}
+  Definition() : generated(false), defined_namespace(nullptr),
+                 serialized_location(0), index(-1) {}
 
   std::string name;
   std::string file;
@@ -187,12 +191,19 @@ struct Definition {
   SymbolTable<Value> attributes;
   bool generated;  // did we already output code for this definition?
   Namespace *defined_namespace;  // Where it was defined.
+
+  // For use with Serialize()
+  uoffset_t serialized_location;
+  int index;  // Inside the vector it is stored.
 };
 
 struct FieldDef : public Definition {
   FieldDef() : deprecated(false), required(false), key(false), padding(0),
                used(false) {}
 
+  Offset<reflection::Field> Serialize(FlatBufferBuilder *builder, uint16_t id)
+                                                                          const;
+
   Value value;
   bool deprecated; // Field is allowed to be present in old data, but can't be
                    // written in new data nor accessed in new code.
@@ -218,6 +229,8 @@ struct StructDef : public Definition {
     if (fields.vec.size()) fields.vec.back()->padding = padding;
   }
 
+  Offset<reflection::Object> Serialize(FlatBufferBuilder *builder) const;
+
   SymbolTable<FieldDef> fields;
   bool fixed;       // If it's struct, not a table.
   bool predecl;     // If it's used before it was defined.
@@ -243,6 +256,8 @@ struct EnumVal {
   EnumVal(const std::string &_name, int64_t _val)
     : name(_name), value(_val), struct_def(nullptr) {}
 
+  Offset<reflection::EnumVal> Serialize(FlatBufferBuilder *builder) const;
+
   std::string name;
   std::vector<std::string> doc_comment;
   int64_t value;
@@ -263,6 +278,8 @@ struct EnumDef : public Definition {
     return nullptr;
   }
 
+  Offset<reflection::Enum> Serialize(FlatBufferBuilder *builder) const;
+
   SymbolTable<EnumVal> vals;
   bool is_union;
   Type underlying_type;
@@ -271,7 +288,7 @@ struct EnumDef : public Definition {
 class Parser {
  public:
   Parser(bool strict_json = false, bool proto_mode = false)
-    : root_struct_def(nullptr),
+    : root_struct_def_(nullptr),
       source_(nullptr),
       cursor_(nullptr),
       line_(1),
@@ -325,6 +342,10 @@ class Parser {
   std::set<std::string> GetIncludedFilesRecursive(
       const std::string &file_name) const;
 
+  // Fills builder_ with a binary version of the schema parsed.
+  // See reflection/reflection.fbs
+  void Serialize();
+
  private:
   int64_t ParseHexNum(int nibbles);
   void Next();
@@ -363,7 +384,7 @@ class Parser {
   std::string error_;         // User readable error_ if Parse() == false
 
   FlatBufferBuilder builder_;  // any data contained in the file
-  StructDef *root_struct_def;
+  StructDef *root_struct_def_;
   std::string file_identifier_;
   std::string file_extension_;
 
@@ -417,7 +438,7 @@ struct GeneratorOptions {
                        output_default_scalars_in_json(false),
                        indent_step(2),
                        output_enum_identifiers(true), prefixed_enums(true),
-                       include_dependence_headers(false),
+                       include_dependence_headers(true),
                        mutable_buffer(false),
                        one_file(false),
                        lang(GeneratorOptions::kJava) {}

reflection/generate_code.sh

+../flatc -c --no-prefix -o ../include/flatbuffers reflection.fbs

reflection/reflection.fbs

+// This schema defines objects that represent a parsed schema, like
+// the binary version of a .fbs file.
+// This could be used to operate on unknown FlatBuffers at runtime.
+// It can even ... represent itself (!)
+
+namespace reflection;
+
+// These must correspond to the enum in idl.h.
+enum BaseType : byte {
+    None,
+    UType,
+    Bool,
+    Byte,
+    UByte,
+    Short,
+    UShort,
+    Int,
+    UInt,
+    Long,
+    ULong,
+    Float,
+    Double,
+    String,
+    Vector,
+    Obj,     // Used for tables & structs.
+    Union    //
+}
+
+table Type {
+    base_type:BaseType;
+    element:BaseType = None;  // Only if base_type == Vector.
+    index:int = -1;  // If base_type == Object, index into "objects" below.
+                     // If base_type == Union, UnionType, or integral derived
+                     // from an enum, index into "enums" below.
+}
+
+table EnumVal {
+    name:string (required);
+    value:long (key);
+    object:Object;  // Only if part of a union.
+}
+
+table Enum {
+    name:string (required, key);
+    values:[EnumVal] (required);  // In order of their values.
+    is_union:bool = false;
+    underlying_type:Type (required);
+}
+
+table Field {
+    name:string (required, key);
+    type:Type (required);
+    id:ushort;
+    offset:ushort;  // Offset into the vtable for tables, or into the struct.
+    default_integer:long = 0;
+    default_real:double = 0.0;
+    deprecated:bool = false;
+    required:bool = false;
+    key:bool = false;
+}
+
+table Object {  // Used for both tables and structs.
+    name:string (required, key);
+    fields:[Field] (required);  // Sorted.
+    is_struct:bool = false;
+}
+
+table Schema {
+    objects:[Object] (required);  // Sorted.
+    enums:[Enum] (required);      // Sorted.
+    file_ident:string;
+    file_ext:string;
+    root_table:Object;
+}
+
+root_type Schema;
+
+file_identifier "BFBS";
+file_extension "bfbs";

src/flatc.cpp

@@ -90,13 +90,15 @@ static void Error(const std::string &err, bool usage, bool show_exe_name) {
       "  --defaults-json Output fields whose value is the default when\n"
       "                  writing JSON\n"
       "  --no-prefix     Don\'t prefix enum values with the enum type in C++.\n"
-      "  --gen-includes  Generate include statements for included schemas the\n"
-      "                  generated file depends on (C++).\n"
+      "  --gen-includes  (deprecated), instead use:\n"
+      "  --no-includes   Don\'t generate include statements for included\n"
+      "                  schemas the generated file depends on (C++).\n"
       "  --gen-mutable   Generate accessors that can mutate buffers in-place.\n"
       "  --gen-onefile   Generate single output file for C#\n"
       "  --raw-binary    Allow binaries without file_indentifier to be read.\n"
       "                  This may crash flatc given a mismatched schema.\n"
       "  --proto         Input is a .proto, translate to .fbs.\n"
+      "  --schema        Serialize schemas instead of JSON (use with -b)\n"
       "FILEs may depend on declarations in earlier files.\n"
       "FILEs after the -- must be binary flatbuffer format files.\n"
       "Output files are named using the base file name of the input,\n"
@@ -117,6 +119,7 @@ int main(int argc, const char *argv[]) {
   bool print_make_rules = false;
   bool proto_mode = false;
   bool raw_binary = false;
+  bool schema_binary = false;
   std::vector<std::string> filenames;
   std::vector<const char *> include_directories;
   size_t binary_files_from = std::numeric_limits<size_t>::max();
@@ -140,7 +143,10 @@ int main(int argc, const char *argv[]) {
       } else if(arg == "--gen-mutable") {
         opts.mutable_buffer = true;
       } else if(arg == "--gen-includes") {
-        opts.include_dependence_headers = true;
+        // Deprecated, remove this option some time in the future.
+        printf("warning: --gen-includes is deprecated (it is now default)\n");
+      } else if(arg == "--no-includes") {
+        opts.include_dependence_headers = false;
       } else if (arg == "--gen-onefile") {
         opts.one_file = true;
       } else if (arg == "--raw-binary") {
@@ -150,6 +156,8 @@ int main(int argc, const char *argv[]) {
       } else if(arg == "--proto") {
         proto_mode = true;
         any_generator = true;
+      } else if(arg == "--schema") {
+        schema_binary = true;
       } else if(arg == "-M") {
         print_make_rules = true;
       } else {
@@ -216,6 +224,10 @@ int main(int argc, const char *argv[]) {
         if (!parser.Parse(contents.c_str(), &include_directories[0],
                           file_it->c_str()))
           Error(parser.error_, false, false);
+        if (schema_binary) {
+          parser.Serialize();
+          parser.file_extension_ = reflection::SchemaExtension();
+        }
         include_directories.pop_back();
         include_directories.pop_back();
       }

src/idl_gen_cpp.cpp

@@ -701,8 +701,8 @@ std::string GenerateCPP(const Parser &parser,
     code += enum_code_post;
 
     // Generate convenient global helper functions:
-    if (parser.root_struct_def) {
-      auto &name = parser.root_struct_def->name;
+    if (parser.root_struct_def_) {
+      auto &name = parser.root_struct_def_->name;
       std::string qualified_name = parser.GetFullyQualifiedName(name);
       std::string cpp_qualified_name = TranslateNameSpace(qualified_name);
 

src/idl_gen_general.cpp

@@ -527,7 +527,7 @@ static void GenStruct(const LanguageParameters &lang, const Parser &parser,
     code += ") + _bb.";
     code += lang.get_bb_position;
     code += ", _bb)); }\n";
-    if (parser.root_struct_def == &struct_def) {
+    if (parser.root_struct_def_ == &struct_def) {
       if (parser.file_identifier_.length()) {
         // Check if a buffer has the identifier.
         code += "  public static ";
@@ -842,7 +842,7 @@ static void GenStruct(const LanguageParameters &lang, const Parser &parser,
       }
     }
     code += "    return o;\n  }\n";
-    if (parser.root_struct_def == &struct_def) {
+    if (parser.root_struct_def_ == &struct_def) {
       code += "  public static void ";
       code += FunctionStart(lang, 'F') + "inish" + struct_def.name;
       code += "Buffer(FlatBufferBuilder builder, int offset) { ";
@@ -878,11 +878,13 @@ static bool SaveClass(const LanguageParameters &lang, const Parser &parser,
   EnsureDirExists(namespace_dir);
 
   std::string code = "// automatically generated, do not modify\n\n";
-  code += lang.namespace_ident + namespace_general + lang.namespace_begin;
-  code += "\n\n";
+  if (!namespace_general.empty()) {
+	code += lang.namespace_ident + namespace_general + lang.namespace_begin;
+	code += "\n\n";
+  }
   if (needs_includes) code += lang.includes;
   code += classcode;
-  code += lang.namespace_end;
+  if (!namespace_general.empty()) code += lang.namespace_end;
   auto filename = namespace_dir + defname + lang.file_extension;
   return SaveFile(filename.c_str(), code, false);
 }
@@ -1007,7 +1009,7 @@ std::string BinaryMakeRule(const Parser &parser,
   std::string make_rule = BinaryFileName(parser, path, filebase) + ": " +
       file_name;
   auto included_files = parser.GetIncludedFilesRecursive(
-      parser.root_struct_def->file);
+      parser.root_struct_def_->file);
   for (auto it = included_files.begin();
        it != included_files.end(); ++it) {
     make_rule += " " + *it;

src/idl_gen_go.cpp

@@ -677,7 +677,7 @@ bool GenerateGo(const Parser &parser,
   for (auto it = parser.structs_.vec.begin();
        it != parser.structs_.vec.end(); ++it) {
     std::string declcode;
-    go::GenStruct(**it, &declcode, parser.root_struct_def);
+    go::GenStruct(**it, &declcode, parser.root_struct_def_);
     if (!go::SaveType(parser, **it, declcode, path, true))
       return false;
   }

src/idl_gen_python.cpp

@@ -651,7 +651,7 @@ bool GeneratePython(const Parser &parser,
   for (auto it = parser.structs_.vec.begin();
        it != parser.structs_.vec.end(); ++it) {
     std::string declcode;
-    python::GenStruct(**it, &declcode, parser.root_struct_def);
+    python::GenStruct(**it, &declcode, parser.root_struct_def_);
     if (!python::SaveType(parser, **it, declcode, path, true))
       return false;
   }

src/idl_gen_text.cpp

@@ -270,9 +270,9 @@ static void GenStruct(const StructDef &struct_def, const Table *table,
 void GenerateText(const Parser &parser, const void *flatbuffer,
                   const GeneratorOptions &opts, std::string *_text) {
   std::string &text = *_text;
-  assert(parser.root_struct_def);  // call SetRootType()
+  assert(parser.root_struct_def_);  // call SetRootType()
   text.reserve(1024);   // Reduce amount of inevitable reallocs.
-  GenStruct(*parser.root_struct_def,
+  GenStruct(*parser.root_struct_def_,
             GetRoot<Table>(flatbuffer),
             0,
             opts,
@@ -289,7 +289,7 @@ bool GenerateTextFile(const Parser &parser,
                       const std::string &path,
                       const std::string &file_name,
                       const GeneratorOptions &opts) {
-  if (!parser.builder_.GetSize() || !parser.root_struct_def) return true;
+  if (!parser.builder_.GetSize() || !parser.root_struct_def_) return true;
   std::string text;
   GenerateText(parser, parser.builder_.GetBufferPointer(), opts,
                &text);
@@ -302,12 +302,12 @@ std::string TextMakeRule(const Parser &parser,
                          const std::string &path,
                          const std::string &file_name,
                          const GeneratorOptions & /*opts*/) {
-  if (!parser.builder_.GetSize() || !parser.root_struct_def) return "";
+  if (!parser.builder_.GetSize() || !parser.root_struct_def_) return "";
   std::string filebase = flatbuffers::StripPath(
       flatbuffers::StripExtension(file_name));
   std::string make_rule = TextFileName(path, filebase) + ": " + file_name;
   auto included_files = parser.GetIncludedFilesRecursive(
-      parser.root_struct_def->file);
+      parser.root_struct_def_->file);
   for (auto it = included_files.begin();
        it != included_files.end(); ++it) {
     make_rule += " " + *it;

src/idl_parser.cpp

@@ -17,8 +17,6 @@
 #include <algorithm>
 #include <list>
 
-#include "flatbuffers/flatbuffers.h"
-#include "flatbuffers/hash.h"
 #include "flatbuffers/idl.h"
 #include "flatbuffers/util.h"
 
@@ -39,6 +37,12 @@ const char kTypeSizes[] = {
   #undef FLATBUFFERS_TD
 };
 
+// The enums in the reflection schema should match the ones we use internally.
+// Compare the last element to check if these go out of sync.
+static_assert(BASE_TYPE_UNION ==
+              static_cast<BaseType>(reflection::Union),
+              "enums don't match");
+
 static void Error(const std::string &msg) {
   throw msg;
 }
@@ -198,7 +202,7 @@ void Parser::Next() {
         }
         // fall thru
       default:
-        if (isalpha(static_cast<unsigned char>(c))) {
+        if (isalpha(static_cast<unsigned char>(c)) || c == '_') {
           // Collect all chars of an identifier:
           const char *start = cursor_ - 1;
           while (isalnum(static_cast<unsigned char>(*cursor_)) ||
@@ -981,8 +985,8 @@ void Parser::ParseDecl() {
 }
 
 bool Parser::SetRootType(const char *name) {
-  root_struct_def = structs_.Lookup(GetFullyQualifiedName(name));
-  return root_struct_def != nullptr;
+  root_struct_def_ = structs_.Lookup(GetFullyQualifiedName(name));
+  return root_struct_def_ != nullptr;
 }
 
 std::string Parser::GetFullyQualifiedName(const std::string &name) const {
@@ -1200,11 +1204,11 @@ bool Parser::Parse(const char *source, const char **include_paths,
       } else if (token_ == kTokenNameSpace) {
         ParseNamespace();
       } else if (token_ == '{') {
-        if (!root_struct_def) Error("no root type set to parse json with");
+        if (!root_struct_def_) Error("no root type set to parse json with");
         if (builder_.GetSize()) {
           Error("cannot have more than one json object in a file");
         }
-        builder_.Finish(Offset<Table>(ParseTable(*root_struct_def)),
+        builder_.Finish(Offset<Table>(ParseTable(*root_struct_def_)),
           file_identifier_.length() ? file_identifier_.c_str() : nullptr);
       } else if (token_ == kTokenEnum) {
         ParseEnum(false);
@@ -1216,7 +1220,7 @@ bool Parser::Parse(const char *source, const char **include_paths,
         Expect(kTokenIdentifier);
         if (!SetRootType(root_type.c_str()))
           Error("unknown root type: " + root_type);
-        if (root_struct_def->fixed)
+        if (root_struct_def_->fixed)
           Error("root type must be a table");
         Expect(';');
       } else if (token_ == kTokenFileIdentifier) {
@@ -1302,4 +1306,105 @@ std::set<std::string> Parser::GetIncludedFilesRecursive(
   return included_files;
 }
 
+// Schema serialization functionality:
+
+template<typename T> void AssignIndices(const std::vector<T *> &defvec) {
+  // Pre-sort these vectors, such that we can set the correct indices for them.
+  auto vec = defvec;
+  std::sort(vec.begin(), vec.end(),
+            [](const T *a, const T *b) { return a->name < b->name; });
+  for (int i = 0; i < static_cast<int>(vec.size()); i++) vec[i]->index = i;
+}
+
+void Parser::Serialize() {
+  builder_.Clear();
+  AssignIndices(structs_.vec);
+  AssignIndices(enums_.vec);
+  std::vector<Offset<reflection::Object>> object_offsets;
+  for (auto it = structs_.vec.begin(); it != structs_.vec.end(); ++it) {
+    auto offset = (*it)->Serialize(&builder_);
+    object_offsets.push_back(offset);
+    (*it)->serialized_location = offset.o;
+  }
+  std::vector<Offset<reflection::Enum>> enum_offsets;
+  for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) {
+    auto offset = (*it)->Serialize(&builder_);
+    enum_offsets.push_back(offset);
+    (*it)->serialized_location = offset.o;
+  }
+  auto schema_offset = reflection::CreateSchema(
+                         builder_,
+                         builder_.CreateVectorOfSortedTables(&object_offsets),
+                         builder_.CreateVectorOfSortedTables(&enum_offsets),
+                         builder_.CreateString(file_identifier_),
+                         builder_.CreateString(file_extension_),
+                         root_struct_def_->serialized_location);
+  builder_.Finish(schema_offset, reflection::SchemaIdentifier());
+}
+
+Offset<reflection::Object> StructDef::Serialize(FlatBufferBuilder *builder)
+                                                                         const {
+  std::vector<Offset<reflection::Field>> field_offsets;
+  for (auto it = fields.vec.begin(); it != fields.vec.end(); ++it) {
+    field_offsets.push_back(
+      (*it)->Serialize(builder,
+                       static_cast<uint16_t>(it - fields.vec.begin())));
+  }
+  return reflection::CreateObject(*builder,
+                                  builder->CreateString(name),
+                                  builder->CreateVectorOfSortedTables(
+                                    &field_offsets),
+                                  fixed);
+}
+
+Offset<reflection::Field> FieldDef::Serialize(FlatBufferBuilder *builder,
+                                              uint16_t id) const {
+  return reflection::CreateField(*builder,
+                                 builder->CreateString(name),
+                                 value.type.Serialize(builder),
+                                 id,
+                                 value.offset,
+                                 IsInteger(value.type.base_type)
+                                   ? StringToInt(value.constant.c_str())
+                                   : 0,
+                                 IsFloat(value.type.base_type)
+                                   ? strtod(value.constant.c_str(), nullptr)
+                                   : 0.0,
+                                 deprecated,
+                                 required,
+                                 key);
+  // TODO: value.constant is almost always "0", we could save quite a bit of
+  // space by sharing it. Same for common values of value.type.
+}
+
+Offset<reflection::Enum> EnumDef::Serialize(FlatBufferBuilder *builder) const {
+  std::vector<Offset<reflection::EnumVal>> enumval_offsets;
+  for (auto it = vals.vec.begin(); it != vals.vec.end(); ++it) {
+    enumval_offsets.push_back((*it)->Serialize(builder));
+  }
+  return reflection::CreateEnum(*builder,
+                                builder->CreateString(name),
+                                builder->CreateVector(enumval_offsets),
+                                is_union,
+                                underlying_type.Serialize(builder));
+}
+
+Offset<reflection::EnumVal> EnumVal::Serialize(FlatBufferBuilder *builder) const
+                                                                               {
+  return reflection::CreateEnumVal(*builder,
+                                   builder->CreateString(name),
+                                   value,
+                                   struct_def
+                                     ? struct_def->serialized_location
+                                     : 0);
+}
+
+Offset<reflection::Type> Type::Serialize(FlatBufferBuilder *builder) const {
+  return reflection::CreateType(*builder,
+                                static_cast<reflection::BaseType>(base_type),
+                                static_cast<reflection::BaseType>(element),
+                                struct_def ? struct_def->index :
+                                             (enum_def ? enum_def->index : -1));
+}
+
 }  // namespace flatbuffers

tests/generate_code.sh

+../flatc -c -j -n -g -b --gen-mutable monster_test.fbs monsterdata_test.json
+../flatc -b --schema monster_test.fbs

tests/test.cpp

@@ -14,6 +14,8 @@
  * limitations under the License.
  */
 
+#define FLATBUFFERS_DEBUG_VERIFICATION_FAILURE 1
+
 #include "flatbuffers/flatbuffers.h"
 #include "flatbuffers/idl.h"
 #include "flatbuffers/util.h"
@@ -36,20 +38,33 @@ using namespace MyGame::Example;
 
 int testing_fails = 0;
 
+void TestFail(const char *expval, const char *val, const char *exp,
+              const char *file, int line) {
+  TEST_OUTPUT_LINE("TEST FAILED: %s:%d, %s (%s) != %s", file, line,
+                   exp, expval, val);
+  assert(0);
+  testing_fails++;
+}
+
+void TestEqStr(const char *expval, const char *val, const char *exp,
+               const char *file, int line) {
+  if (strcmp(expval, val) != 0) {
+    TestFail(expval, val, exp, file, line);
+  }
+}
+
 template<typename T, typename U>
 void TestEq(T expval, U val, const char *exp, const char *file, int line) {
   if (U(expval) != val) {
-    auto expval_str = flatbuffers::NumToString(expval);
-    auto val_str = flatbuffers::NumToString(val);
-    TEST_OUTPUT_LINE("TEST FAILED: %s:%d, %s (%s) != %s", file, line,
-                     exp, expval_str.c_str(), val_str.c_str());
-    assert(0);
-    testing_fails++;
+    TestFail(flatbuffers::NumToString(expval).c_str(),
+             flatbuffers::NumToString(val).c_str(),
+             exp, file, line);
   }
 }
 
 #define TEST_EQ(exp, val) TestEq(exp,         val,   #exp, __FILE__, __LINE__)
 #define TEST_NOTNULL(exp) TestEq(exp == NULL, false, #exp, __FILE__, __LINE__)
+#define TEST_EQ_STR(exp, val) TestEqStr(exp,  val,   #exp, __FILE__, __LINE__)
 
 // Include simple random number generator to ensure results will be the
 // same cross platform.
@@ -143,7 +158,7 @@ void AccessFlatBufferTest(const uint8_t *flatbuf, size_t length) {
 
   TEST_EQ(monster->hp(), 80);
   TEST_EQ(monster->mana(), 150);  // default
-  TEST_EQ(strcmp(monster->name()->c_str(), "MyMonster"), 0);
+  TEST_EQ_STR(monster->name()->c_str(), "MyMonster");
   // Can't access the following field, it is deprecated in the schema,
   // which means accessors are not generated:
   // monster.friendly()
@@ -167,22 +182,22 @@ void AccessFlatBufferTest(const uint8_t *flatbuf, size_t length) {
   TEST_EQ(monster->test_type(), Any_Monster);  // First make sure which it is.
   auto monster2 = reinterpret_cast<const Monster *>(monster->test());
   TEST_NOTNULL(monster2);
-  TEST_EQ(strcmp(monster2->name()->c_str(), "Fred"), 0);
+  TEST_EQ_STR(monster2->name()->c_str(), "Fred");
 
   // Example of accessing a vector of strings:
   auto vecofstrings = monster->testarrayofstring();
   TEST_EQ(vecofstrings->Length(), 2U);
-  TEST_EQ(strcmp(vecofstrings->Get(0)->c_str(), "bob"), 0);
-  TEST_EQ(strcmp(vecofstrings->Get(1)->c_str(), "fred"), 0);
+  TEST_EQ_STR(vecofstrings->Get(0)->c_str(), "bob");
+  TEST_EQ_STR(vecofstrings->Get(1)->c_str(), "fred");
 
   // Example of accessing a vector of tables:
   auto vecoftables = monster->testarrayoftables();
   TEST_EQ(vecoftables->Length(), 3U);
   for (auto it = vecoftables->begin(); it != vecoftables->end(); ++it)
     TEST_EQ(strlen(it->name()->c_str()) >= 4, true);
-  TEST_EQ(strcmp(vecoftables->Get(0)->name()->c_str(), "Barney"), 0);
-  TEST_EQ(strcmp(vecoftables->Get(1)->name()->c_str(), "Fred"), 0);
-  TEST_EQ(strcmp(vecoftables->Get(2)->name()->c_str(), "Wilma"), 0);
+  TEST_EQ_STR(vecoftables->Get(0)->name()->c_str(), "Barney");
+  TEST_EQ_STR(vecoftables->Get(1)->name()->c_str(), "Fred");
+  TEST_EQ_STR(vecoftables->Get(2)->name()->c_str(), "Wilma");
   TEST_NOTNULL(vecoftables->LookupByKey("Barney"));
   TEST_NOTNULL(vecoftables->LookupByKey("Fred"));
   TEST_NOTNULL(vecoftables->LookupByKey("Wilma"));
@@ -275,6 +290,86 @@ void ParseAndGenerateTextTest() {
   }
 }
 
+void ReflectionTest(uint8_t *flatbuf, size_t length) {
+  // Load a binary schema.
+  std::string bfbsfile;
+  TEST_EQ(flatbuffers::LoadFile(
+    "tests/monster_test.bfbs", true, &bfbsfile), true);
+
+  // Verify it, just in case:
+  flatbuffers::Verifier verifier(
+    reinterpret_cast<const uint8_t *>(bfbsfile.c_str()), bfbsfile.length());
+  TEST_EQ(reflection::VerifySchemaBuffer(verifier), true);
+
+  // Make sure the schema is what we expect it to be.
+  auto schema = reflection::GetSchema(bfbsfile.c_str());
+  auto root_table = schema->root_table();
+  TEST_EQ_STR(root_table->name()->c_str(), "Monster");
+  auto fields = root_table->fields();
+  auto hp_field = fields->LookupByKey("hp");
+  TEST_NOTNULL(hp_field);
+  TEST_EQ_STR(hp_field->name()->c_str(), "hp");
+  TEST_EQ(hp_field->id(), 2);
+  TEST_EQ(hp_field->type()->base_type(), reflection::Short);
+
+  // Now use it to dynamically access a buffer.
+  auto root = flatbuffers::GetAnyRoot(flatbuf);
+  auto hp = flatbuffers::GetFieldI<uint16_t>(root, hp_field);
+  TEST_EQ(hp, 80);
+
+  // Rather than needing to know the type, we can also get the value of
+  // any field as an int64_t/double/string, regardless of what it actually is.
+  auto hp_int64 = flatbuffers::GetAnyFieldI(root, hp_field);
+  TEST_EQ(hp_int64, 80);
+  auto hp_double = flatbuffers::GetAnyFieldF(root, hp_field);
+  TEST_EQ(hp_double, 80.0);
+  auto hp_string = flatbuffers::GetAnyFieldS(root, hp_field);
+  TEST_EQ_STR(hp_string.c_str(), "80");
+
+  // We can also modify it.
+  flatbuffers::SetField<uint16_t>(root, hp_field, 200);
+  hp = flatbuffers::GetFieldI<uint16_t>(root, hp_field);
+  TEST_EQ(hp, 200);
+
+  // We can also set fields generically:
+  flatbuffers::SetAnyFieldI(root, hp_field, 300);
+  hp_int64 = flatbuffers::GetAnyFieldI(root, hp_field);
+  TEST_EQ(hp_int64, 300);
+  flatbuffers::SetAnyFieldF(root, hp_field, 300.5);
+  hp_int64 = flatbuffers::GetAnyFieldI(root, hp_field);
+  TEST_EQ(hp_int64, 300);
+  flatbuffers::SetAnyFieldS(root, hp_field, "300");
+  hp_int64 = flatbuffers::GetAnyFieldI(root, hp_field);
+  TEST_EQ(hp_int64, 300);
+
+  // Reset it, for further tests.
+  flatbuffers::SetField<uint16_t>(root, hp_field, 80);
+
+  // More advanced functionality: changing the size of items in-line!
+  // First we put the FlatBuffer inside an std::vector.
+  std::vector<uint8_t> resizingbuf(flatbuf, flatbuf + length);
+  // Find the field we want to modify.
+  auto name_field = fields->LookupByKey("name");
+  // Get the root.
+  // This time we wrap the result from GetAnyRoot in a smartpointer that
+  // will keep rroot valid as resizingbuf resizes.
+  auto rroot = flatbuffers::piv(flatbuffers::GetAnyRoot(resizingbuf.data()),
+                                resizingbuf);
+  SetString(*schema, "totally new string", GetFieldS(*rroot, name_field),
+            &resizingbuf);
+  // Here resizingbuf has changed, but rroot is still valid.
+  TEST_EQ_STR(GetFieldS(*rroot, name_field)->c_str(), "totally new string");
+  // Now lets extend a vector by 100 elements (10 -> 110).
+  auto inventory_field = fields->LookupByKey("inventory");
+  auto rinventory = flatbuffers::piv(
+                     flatbuffers::GetFieldV<uint8_t>(*rroot, inventory_field),
+                     resizingbuf);
+  flatbuffers::ResizeVector<uint8_t>(*schema, 110, 50, *rinventory,
+                                     &resizingbuf);
+  // rinventory still valid, so lets read from it.
+  TEST_EQ(rinventory->Get(10), 50);
+}
+
 // Parse a .proto schema, output as .fbs
 void ParseProtoTest() {
   // load the .proto and the golden file from disk
@@ -291,6 +386,7 @@ void ParseProtoTest() {
 
   // Generate fbs.
   flatbuffers::GeneratorOptions opts;
+  opts.include_dependence_headers = false;
   auto fbs = flatbuffers::GenerateFBS(parser, "test", opts);
 
   // Ensure generated file is parsable.
@@ -666,6 +762,7 @@ int main(int /*argc*/, const char * /*argv*/[]) {
 
   #ifndef FLATBUFFERS_NO_FILE_TESTS
   ParseAndGenerateTextTest();
+  ReflectionTest(flatbuf.get(), rawbuf.length());
   ParseProtoTest();
   #endif