Support for required fields.

Change-Id: I560c7ca11b3d665eecafb528f3737b7e139ca9b0
Tested: on Linux and Windows.

docs/html/md__schemas.html

@@ -133,6 +133,7 @@ root_type Monster;
 <ul>
 <li><code>id: n</code> (on a table field): manually set the field identifier to <code>n</code>. If you use this attribute, you must use it on ALL fields of this table, and the numbers must be a contiguous range from 0 onwards. Additionally, since a union type effectively adds two fields, its id must be that of the second field (the first field is the type field and not explicitly declared in the schema). For example, if the last field before the union field had id 6, the union field should have id 8, and the unions type field will implicitly be 7. IDs allow the fields to be placed in any order in the schema. When a new field is added to the schema is must use the next available ID.</li>
 <li><code>deprecated</code> (on a field): do not generate accessors for this field anymore, code should stop using this data.</li>
+<li><code>required</code> (on a non-scalar table field): this field must always be set. By default, all fields are optional, i.e. may be left out. This is desirable, as it helps with forwards/backwards compatibility, and flexibility of data structures. It is also a burden on the reading code, since for non-scalar fields it requires you to check against NULL and take appropriate action. By specifying this field, you force code that constructs FlatBuffers to ensure this field is initialized, so the reading code may access it directly, without checking for NULL. If the constructing code does not initialize this field, they will get an assert, and also the verifier will fail on buffers that have missing required fields.</li>
 <li><code>original_order</code> (on a table): since elements in a table do not need to be stored in any particular order, they are often optimized for space by sorting them to size. This attribute stops that from happening.</li>
 <li><code>force_align: size</code> (on a struct): force the alignment of this struct to be something higher than what it is naturally aligned to. Causes these structs to be aligned to that amount inside a buffer, IF that buffer is allocated with that alignment (which is not necessarily the case for buffers accessed directly inside a <code>FlatBufferBuilder</code>).</li>
 <li><code>bit_flags</code> (on an enum): the values of this field indicate bits, meaning that any value N specified in the schema will end up representing 1&lt;&lt;N, or if you don't specify values at all, you'll get the sequence 1, 2, 4, 8, ...</li>

docs/source/Schemas.md

@@ -146,7 +146,7 @@ packages.
 You can include other schemas files in your current one, e.g.:
 
     include "mydefinitions.fbs"
-    
+
 This makes it easier to refer to types defined elsewhere. `include`
 automatically ensures each file is parsed just once, even when referred to
 more than once.
@@ -232,6 +232,16 @@ Current understood attributes:
     When a new field is added to the schema is must use the next available ID.
 -   `deprecated` (on a field): do not generate accessors for this field
     anymore, code should stop using this data.
+-   `required` (on a non-scalar table field): this field must always be set.
+    By default, all fields are optional, i.e. may be left out. This is
+    desirable, as it helps with forwards/backwards compatibility, and
+    flexibility of data structures. It is also a burden on the reading code,
+    since for non-scalar fields it requires you to check against NULL and
+    take appropriate action. By specifying this field, you force code that
+    constructs FlatBuffers to ensure this field is initialized, so the reading
+    code may access it directly, without checking for NULL. If the constructing
+    code does not initialize this field, they will get an assert, and also
+    the verifier will fail on buffers that have missing required fields.
 -   `original_order` (on a table): since elements in a table do not need
     to be stored in any particular order, they are often optimized for
     space by sorting them to size. This attribute stops that from happening.

include/flatbuffers/flatbuffers.h

@@ -540,6 +540,17 @@ class FlatBufferBuilder {
     return vtableoffsetloc;
   }
 
+  // This checks a required field has been set in a given table that has
+  // just been constructed.
+  template<typename T> void Required(Offset<T> table, voffset_t field) {
+    auto table_ptr = buf_.data_at(table.o);
+    auto vtable_ptr = table_ptr - ReadScalar<uoffset_t>(table_ptr);
+    bool ok = ReadScalar<voffset_t>(vtable_ptr + field) != 0;
+    // If this fails, the caller will show what field needs to be set.
+    assert(ok);
+    (void)ok;
+  }
+
   uoffset_t StartStruct(size_t alignment) {
     Align(alignment);
     return GetSize();
@@ -678,15 +689,19 @@ class Verifier {
       num_tables_(0), max_tables_(_max_tables)
     {}
 
-  // Verify any range within the buffer.
-  bool Verify(const void *elem, size_t elem_len) const {
-    bool ok = elem >= buf_ && elem <= end_ - elem_len;
+  // Central location where any verification failures register.
+  bool Check(bool ok) const {
     #ifdef FLATBUFFERS_DEBUG_VERIFICATION_FAILURE
       assert(ok);
     #endif
     return ok;
   }
 
+  // Verify any range within the buffer.
+  bool Verify(const void *elem, size_t elem_len) const {
+    return Check(elem >= buf_ && elem <= end_ - elem_len);
+  }
+
   // Verify a range indicated by sizeof(T).
   template<typename T> bool Verify(const void *elem) const {
     return Verify(elem, sizeof(T));
@@ -710,8 +725,8 @@ class Verifier {
     const uint8_t *end;
     return !str ||
            (VerifyVector(reinterpret_cast<const uint8_t *>(str), 1, &end) &&
-            Verify(end, 1) &&  // Must have terminator
-            *end == '\0');  // Terminating byte must be 0.
+            Verify(end, 1) &&      // Must have terminator
+            Check(*end == '\0'));  // Terminating byte must be 0.
   }
 
   // Common code between vectors and strings.
@@ -762,11 +777,7 @@ class Verifier {
   bool VerifyComplexity() {
     depth_++;
     num_tables_++;
-    bool too_complex = depth_ > max_depth_ || num_tables_ > max_tables_;
-    #ifdef FLATBUFFERS_DEBUG_VERIFICATION_FAILURE
-      assert(!too_complex);
-    #endif
-    return !too_complex;
+    return Check(depth_ <= max_depth_ && num_tables_ <= max_tables_);
   }
 
   // Called at the end of a table to pop the depth count.
@@ -876,6 +887,14 @@ class Table {
     return !field_offset || verifier.Verify<T>(data_ + field_offset);
   }
 
+  // VerifyField for required fields.
+  template<typename T> bool VerifyFieldRequired(const Verifier &verifier,
+                                        voffset_t field) const {
+    auto field_offset = GetOptionalFieldOffset(field);
+    return verifier.Check(field_offset != 0) &&
+           verifier.Verify<T>(data_ + field_offset);
+  }
+
  private:
   // private constructor & copy constructor: you obtain instances of this
   // class by pointing to existing data only

include/flatbuffers/idl.h

@@ -184,10 +184,11 @@ struct Definition {
 };
 
 struct FieldDef : public Definition {
-  FieldDef() : deprecated(false), padding(0), used(false) {}
+  FieldDef() : deprecated(false), required(false), padding(0), used(false) {}
 
   Value value;
   bool deprecated;
+  bool required;
   size_t padding;  // Bytes to always pad after this field.
   bool used;       // Used during JSON parsing to check for repeated fields.
 };

java/com/google/flatbuffers/FlatBufferBuilder.java

@@ -242,6 +242,17 @@ public class FlatBufferBuilder {
         return vtableloc;
     }
 
+    // This checks a required field has been set in a given table that has
+    // just been constructed.
+    public void required(int table, int field) {
+        int table_start = bb.capacity() - table;
+        int vtable_start = table_start - bb.getInt(table_start);
+        boolean ok = bb.getShort(vtable_start + field) != 0;
+        // If this fails, the caller will show what field needs to be set.
+        if (!ok)
+            throw new AssertionError("FlatBuffers: field " + field + " must be set");
+    }
+
     public void finish(int root_table) {
         prep(minalign, SIZEOF_INT);
         addOffset(root_table);

net/FlatBuffers/FlatBufferBuilder.cs

@@ -335,6 +335,19 @@ namespace FlatBuffers
             return vtableloc;
         }
 
+        // This checks a required field has been set in a given table that has
+        // just been constructed.
+        public void Required(int table, int field)
+        {
+          int table_start = _bb.Length - table;
+          int vtable_start = table_start - _bb.GetInt(table_start);
+          bool ok = _bb.GetShort(vtable_start + field) != 0;
+          // If this fails, the caller will show what field needs to be set.
+          if (!ok)
+            throw new InvalidOperationException("FlatBuffers: field " + field +
+                                                " must be set");
+        }
+
         public void Finish(int rootTable)
         {
             Prep(_minAlign, sizeof(int));

src/idl_gen_cpp.cpp

@@ -225,7 +225,9 @@ static void GenTable(const Parser &parser, StructDef &struct_def,
        ++it) {
     auto &field = **it;
     if (!field.deprecated) {
-      code += prefix + "VerifyField<" + GenTypeSize(parser, field.value.type);
+      code += prefix + "VerifyField";
+      if (field.required) code += "Required";
+      code += "<" + GenTypeSize(parser, field.value.type);
       code += ">(verifier, " + NumToString(field.value.offset);
       code += " /* " + field.name + " */)";
       switch (field.value.type.base_type) {
@@ -301,9 +303,19 @@ static void GenTable(const Parser &parser, StructDef &struct_def,
   code += "  " + struct_def.name + "Builder &operator=(const ";
   code += struct_def.name + "Builder &);\n";
   code += "  flatbuffers::Offset<" + struct_def.name;
-  code += "> Finish() { return flatbuffers::Offset<" + struct_def.name;
+  code += "> Finish() {\n    auto o = flatbuffers::Offset<" + struct_def.name;
   code += ">(fbb_.EndTable(start_, ";
-  code += NumToString(struct_def.fields.vec.size()) + ")); }\n};\n\n";
+  code += NumToString(struct_def.fields.vec.size()) + "));\n";
+  for (auto it = struct_def.fields.vec.begin();
+       it != struct_def.fields.vec.end();
+       ++it) {
+    auto &field = **it;
+    if (!field.deprecated && field.required) {
+      code += "    fbb_.Required(o, " + NumToString(field.value.offset);
+      code += ");  // " + field.name + "\n";
+    }
+  }
+  code += "    return o;\n  }\n};\n\n";
 
   // Generate a convenient CreateX function that uses the above builder
   // to create a table in one go.

src/idl_gen_general.cpp

@@ -554,8 +554,19 @@ static void GenStruct(const LanguageParameters &lang, const Parser &parser,
     }
     code += "  public static int ";
     code += FunctionStart(lang, 'E') + "nd" + struct_def.name;
-    code += "(FlatBufferBuilder builder) { return builder.";
-    code += FunctionStart(lang, 'E') + "ndObject(); }\n";
+    code += "(FlatBufferBuilder builder) {\n    int o = builder.";
+    code += FunctionStart(lang, 'E') + "ndObject();\n";
+    for (auto it = struct_def.fields.vec.begin();
+         it != struct_def.fields.vec.end();
+         ++it) {
+      auto &field = **it;
+      if (!field.deprecated && field.required) {
+        code += "    builder." + FunctionStart(lang, 'R') + "equired(o, ";
+        code += NumToString(field.value.offset);
+        code += ");  // " + field.name + "\n";
+      }
+    }
+    code += "    return o;\n  }\n";
     if (parser.root_struct_def == &struct_def) {
       code += "  public static void ";
       code += FunctionStart(lang, 'F') + "inish" + struct_def.name;

src/idl_parser.cpp

@@ -376,6 +376,10 @@ void Parser::ParseField(StructDef &struct_def) {
   field.deprecated = field.attributes.Lookup("deprecated") != nullptr;
   if (field.deprecated && struct_def.fixed)
     Error("can't deprecate fields in a struct");
+  field.required = field.attributes.Lookup("required") != nullptr;
+  if (field.required && (struct_def.fixed ||
+                         IsScalar(field.value.type.base_type)))
+    Error("only non-scalar fields in tables may be 'required'");
   auto nested = field.attributes.Lookup("nested_flatbuffer");
   if (nested) {
     if (nested->type.base_type != BASE_TYPE_STRING)

tests/FlatBuffers.Test/FlatBuffers.Test.csproj

@@ -66,13 +66,13 @@
     </ProjectReference>
   </ItemGroup>
   <ItemGroup>
-    <Content Include="..\monsterdata_test.bin">
-      <Link>Resources\monsterdata_test.bin</Link>
+    <Content Include="..\monsterdata_test.mon">
+      <Link>Resources\monsterdata_test.mon</Link>
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
-  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it.
        Other similar extension points exist, see Microsoft.Common.targets.
   <Target Name="BeforeBuild">
   </Target>

tests/FlatBuffers.Test/FlatBuffersExampleTests.cs

@@ -50,8 +50,9 @@ namespace FlatBuffers.Test
             }
             var inv = fbb.EndVector();
 
+            var fred = fbb.CreateString("Fred");
             Monster.StartMonster(fbb);
-            Monster.AddHp(fbb, (short)20);
+            Monster.AddName(fbb, fred);
             var mon2 = Monster.EndMonster(fbb);
 
             Monster.StartTest4Vector(fbb, 2);
@@ -83,7 +84,7 @@ namespace FlatBuffers.Test
             using (var ms = new MemoryStream(fbb.DataBuffer().Data, fbb.DataBuffer().position(), fbb.Offset()))
             {
                 var data = ms.ToArray();
-                File.WriteAllBytes(@"Resources/monsterdata_cstest.bin",data);
+                File.WriteAllBytes(@"Resources/monsterdata_cstest.mon",data);
             }
 
             // Now assert the buffer
@@ -113,7 +114,7 @@ namespace FlatBuffers.Test
 
             var monster2 = new Monster();
             Assert.IsTrue(monster.Test(monster2) != null);
-            Assert.AreEqual(20, monster2.Hp());
+            Assert.AreEqual("Fred", monster2.Name());
 
 
             Assert.AreEqual(5, monster.InventoryLength());
@@ -130,7 +131,6 @@ namespace FlatBuffers.Test
 
             Assert.AreEqual(100, test0.A() + test0.B() + test1.A() + test1.B());
 
-
             Assert.AreEqual(2, monster.TestarrayofstringLength());
             Assert.AreEqual("test1", monster.Testarrayofstring(0));
             Assert.AreEqual("test2", monster.Testarrayofstring(1));
@@ -138,7 +138,7 @@ namespace FlatBuffers.Test
 
         public void CanReadCppGeneratedWireFile()
         {
-            var data = File.ReadAllBytes(@"Resources/monsterdata_test.bin");
+            var data = File.ReadAllBytes(@"Resources/monsterdata_test.mon");
             var bb = new ByteBuffer(data);
             TestBuffer(bb);
         }

tests/JavaTest.java

@@ -26,7 +26,7 @@ class JavaTest {
         // This file was generated from monsterdata_test.json
 
         byte[] data = null;
-        File file = new File("monsterdata_test.bin");
+        File file = new File("monsterdata_test.mon");
         RandomAccessFile f = null;
         try {
             f = new RandomAccessFile(file, "r");
@@ -55,8 +55,9 @@ class JavaTest {
 
         int inv = Monster.createInventoryVector(fbb, new byte[] { 0, 1, 2, 3, 4 });
 
+        int fred = fbb.createString("Fred");
         Monster.startMonster(fbb);
-        Monster.addHp(fbb, (short)20);
+        Monster.addName(fbb, fred);
         int mon2 = Monster.endMonster(fbb);
 
         Monster.startTest4Vector(fbb, 2);
@@ -90,7 +91,7 @@ class JavaTest {
 
         try {
             DataOutputStream os = new DataOutputStream(new FileOutputStream(
-                                           "monsterdata_java_wire.bin"));
+                                           "monsterdata_java_wire.mon"));
             os.write(fbb.dataBuffer().array(), fbb.dataBuffer().position(), fbb.offset());
             os.close();
         } catch(java.io.IOException e) {
@@ -142,7 +143,7 @@ class JavaTest {
         TestEq(monster.testType(), (byte)Any.Monster);
         Monster monster2 = new Monster();
         TestEq(monster.test(monster2) != null, true);
-        TestEq(monster2.hp(), (short)20);
+        TestEq(monster2.name(), "Fred");
 
         TestEq(monster.inventoryLength(), 5);
         int invsum = 0;

tests/MyGame/Example/Monster.cs

@@ -60,7 +60,11 @@ public class Monster : Table {
   public static int CreateTestnestedflatbufferVector(FlatBufferBuilder builder, byte[] data) { builder.StartVector(1, data.Length, 1); for (int i = data.Length - 1; i >= 0; i--) builder.AddByte(data[i]); return builder.EndVector(); }
   public static void StartTestnestedflatbufferVector(FlatBufferBuilder builder, int numElems) { builder.StartVector(1, numElems, 1); }
   public static void AddTestempty(FlatBufferBuilder builder, int testemptyOffset) { builder.AddOffset(14, testemptyOffset, 0); }
-  public static int EndMonster(FlatBufferBuilder builder) { return builder.EndObject(); }
+  public static int EndMonster(FlatBufferBuilder builder) {
+    int o = builder.EndObject();
+    builder.Required(o, 10);  // name
+    return o;
+  }
   public static void FinishMonsterBuffer(FlatBufferBuilder builder, int offset) { builder.Finish(offset, "MONS"); }
 };
 

tests/MyGame/Example/Monster.java

@@ -68,7 +68,11 @@ public class Monster extends Table {
   public static int createTestnestedflatbufferVector(FlatBufferBuilder builder, byte[] data) { builder.startVector(1, data.length, 1); for (int i = data.length - 1; i >= 0; i--) builder.addByte(data[i]); return builder.endVector(); }
   public static void startTestnestedflatbufferVector(FlatBufferBuilder builder, int numElems) { builder.startVector(1, numElems, 1); }
   public static void addTestempty(FlatBufferBuilder builder, int testemptyOffset) { builder.addOffset(14, testemptyOffset, 0); }
-  public static int endMonster(FlatBufferBuilder builder) { return builder.endObject(); }
+  public static int endMonster(FlatBufferBuilder builder) {
+    int o = builder.endObject();
+    builder.required(o, 10);  // name
+    return o;
+  }
   public static void finishMonsterBuffer(FlatBufferBuilder builder, int offset) { builder.finish(offset, "MONS"); }
 };
 

tests/monster_test.fbs

@@ -23,7 +23,7 @@ table Monster {
   pos:Vec3 (id: 0);
   hp:short = 100 (id: 2);
   mana:short = 150 (id: 1);
-  name:string (id: 3);
+  name:string (id: 3, required);
   color:Color = Blue (id: 6);
   inventory:[ubyte] (id: 5);
   friendly:bool = false (deprecated, priority: 1, id: 4);

tests/monster_test_generated.h

@@ -107,7 +107,7 @@ struct Monster : private flatbuffers::Table {
            VerifyField<Vec3>(verifier, 4 /* pos */) &&
            VerifyField<int16_t>(verifier, 6 /* mana */) &&
            VerifyField<int16_t>(verifier, 8 /* hp */) &&
-           VerifyField<flatbuffers::uoffset_t>(verifier, 10 /* name */) &&
+           VerifyFieldRequired<flatbuffers::uoffset_t>(verifier, 10 /* name */) &&
            verifier.Verify(name()) &&
            VerifyField<flatbuffers::uoffset_t>(verifier, 14 /* inventory */) &&
            verifier.Verify(inventory()) &&
@@ -152,7 +152,11 @@ struct MonsterBuilder {
   void add_testempty(flatbuffers::Offset<Monster> testempty) { fbb_.AddOffset(32, testempty); }
   MonsterBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); }
   MonsterBuilder &operator=(const MonsterBuilder &);
-  flatbuffers::Offset<Monster> Finish() { return flatbuffers::Offset<Monster>(fbb_.EndTable(start_, 15)); }
+  flatbuffers::Offset<Monster> Finish() {
+    auto o = flatbuffers::Offset<Monster>(fbb_.EndTable(start_, 15));
+    fbb_.Required(o, 10);  // name
+    return o;
+  }
 };
 
 inline flatbuffers::Offset<Monster> CreateMonster(flatbuffers::FlatBufferBuilder &_fbb,

tests/monsterdata_test.golden

@@ -21,7 +21,7 @@
   ],
   test_type: Monster,
   test: {
-    hp: 20
+    name: "Fred"
   },
   test4: [
     {
@@ -36,7 +36,5 @@
   testarrayofstring: [
     "test1",
     "test2"
-  ],
-  testempty: {
-  }
+  ]
 }

tests/monsterdata_test.json

@@ -21,7 +21,7 @@
   ],
   test_type: Monster,
   test: {
-    hp: 20
+    name: "Fred"
   },
   test4: [
     {
@@ -36,7 +36,5 @@
   testarrayofstring: [
     "test1",
     "test2"
-  ],
-  testempty: {
-  }
+  ]
 }

tests/test.cpp

@@ -75,8 +75,9 @@ std::string CreateFlatBufferTest() {
 
   // create monster with very few fields set:
   // (same functionality as CreateMonster below, but sets fields manually)
+  auto fred = builder.CreateString("Fred");
   MonsterBuilder mb(builder);
-  mb.add_hp(20);
+  mb.add_name(fred);
   auto mloc2 = mb.Finish();
 
   // Create an array of strings:
@@ -144,7 +145,7 @@ void AccessFlatBufferTest(const std::string &flatbuf) {
   TEST_EQ(monster->test_type(), Any_Monster);  // First make sure which it is.
   auto monster2 = reinterpret_cast<const Monster *>(monster->test());
   TEST_NOTNULL(monster2);
-  TEST_EQ(monster2->hp(), 20);
+  TEST_EQ(strcmp(monster2->name()->c_str(), "Fred"), 0);
 
   // Example of accessing a vector of strings:
   auto vecofstrings = monster->testarrayofstring();
@@ -156,7 +157,7 @@ void AccessFlatBufferTest(const std::string &flatbuf) {
   auto vecoftables = monster->testarrayoftables();
   TEST_EQ(vecoftables->Length(), 1U);
   for (auto it = vecoftables->begin(); it != vecoftables->end(); ++it)
-    TEST_EQ(it->hp(), 20);
+    TEST_EQ(strcmp(it->name()->c_str(), "Fred"), 0);
 
   // Since Flatbuffers uses explicit mechanisms to override the default
   // compiler alignment, double check that the compiler indeed obeys them: