External references for the object API thru a resolver function.

This allows hashed string fields to be used for lookup of any
C++ objects, a pointer to which are then stored in the object
besides the original hash for easy access.

Change-Id: I2247a13c349b905f1c54660becde2c818ad23e97
Tested: on Linux.
Bug: 30204449

docs/source/CppUsage.md

@@ -106,6 +106,25 @@ To use:
     CreateMonster(fbb, monsterobj.get());     // Serialize into new buffer.
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+# External references.
+
+An additional feature of the object API is the ability to allow you to load
+multiple independent FlatBuffers, and have them refer to eachothers objects
+using hashes which are then represented as typed pointers in the object API.
+
+To make this work have a field in the objects you want to referred to which is
+using the string hashing feature (see `hash` attribute in the
+[schema](@ref flatbuffers_guide_writing_schema) documentation). Then you have
+a similar hash in the field referring to it, along with a `cpp_type`
+attribute specifying the C++ type this will refer to (this can be any C++
+type, and will get a `*` added).
+
+Then, in JSON or however you create these buffers, make sure they use the
+same string (or hash).
+
+When you call `UnPack` (or `Create`), you'll need a function that maps from
+hash to the object (see `resolver_function_t` for details).
+
 ## Reflection (& Resizing)
 
 There is experimental support for reflection in FlatBuffers, allowing you to

docs/source/Schemas.md

@@ -304,6 +304,10 @@ Current understood attributes:
 -   `key` (on a field): this field is meant to be used as a key when sorting
     a vector of the type of table it sits in. Can be used for in-place
     binary search.
+-   `hash` (on a field). This is an (un)signed 32/64 bit integer field, whose
+    value during JSON parsing is allowed to be a string, which will then be
+    stored as its hash. The value of attribute is the hashing algorithm to
+    use, one of `fnv1_32` `fnv1_64` `fnv1a_32` `fnv1a_64`.
 
 ## JSON Parsing
 

include/flatbuffers/flatbuffers.h

@@ -1560,6 +1560,24 @@ class Table {
 struct NativeTable {
 };
 
+/// @brief Function types to be used with resolving hashes into objects and
+/// back again. The resolver gets a pointer to a field inside an object API
+/// object that is of the type specified in the schema using the attribute
+/// `cpp_type` (it is thus important whatever you write to this address
+/// matches that type). The value of this field is initially null, so you
+/// may choose to implement a delayed binding lookup using this function
+/// if you wish. The resolver does the opposite lookup, for when the object
+/// is being serialized again.
+typedef uint64_t hash_value_t;
+#ifdef FLATBUFFERS_CPP98_STL
+  typedef void (*resolver_function_t)(void **pointer_adr, hash_value_t hash);
+  typedef hash_value_t (*rehasher_function_t)(void *pointer);
+#else
+  typedef std::function<void (void **pointer_adr, hash_value_t hash)>
+          resolver_function_t;
+  typedef std::function<hash_value_t (void *pointer)> rehasher_function_t;
+#endif
+
 // Helper function to test if a field is present, using any of the field
 // enums in the generated code.
 // `table` must be a generated table type. Since this is a template parameter,

include/flatbuffers/idl.h

@@ -450,6 +450,7 @@ class Parser : public ParserState {
     known_attributes_["csharp_partial"] = true;
     known_attributes_["streaming"] = true;
     known_attributes_["idempotent"] = true;
+    known_attributes_["cpp_type"] = true;
   }
 
   ~Parser() {

samples/monster_generated.h

@@ -47,8 +47,8 @@ struct EquipmentUnion {
   EquipmentUnion &operator=(const EquipmentUnion &);
   ~EquipmentUnion();
 
-  static flatbuffers::NativeTable *UnPack(const void *union_obj, Equipment type);
-  flatbuffers::Offset<void> Pack(flatbuffers::FlatBufferBuilder &_fbb) const;
+  static flatbuffers::NativeTable *UnPack(const void *union_obj, Equipment type, const flatbuffers::resolver_function_t *resolver);
+  flatbuffers::Offset<void> Pack(flatbuffers::FlatBufferBuilder &_fbb, const flatbuffers::rehasher_function_t *rehasher = nullptr) const;
 
   WeaponT *AsWeapon() { return type == Equipment_Weapon ? reinterpret_cast<WeaponT *>(table) : nullptr; }
 };
@@ -142,7 +142,7 @@ struct Monster FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
            VerifyEquipment(verifier, equipped(), equipped_type()) &&
            verifier.EndTable();
   }
-  std::unique_ptr<MonsterT> UnPack() const;
+  std::unique_ptr<MonsterT> UnPack(const flatbuffers::resolver_function_t *resolver = nullptr) const;
 };
 
 struct MonsterBuilder {
@@ -201,7 +201,7 @@ inline flatbuffers::Offset<Monster> CreateMonsterDirect(flatbuffers::FlatBufferB
   return CreateMonster(_fbb, pos, mana, hp, name ? _fbb.CreateString(name) : 0, inventory ? _fbb.CreateVector<uint8_t>(*inventory) : 0, color, weapons ? _fbb.CreateVector<flatbuffers::Offset<Weapon>>(*weapons) : 0, equipped_type, equipped);
 }
 
-inline flatbuffers::Offset<Monster> CreateMonster(flatbuffers::FlatBufferBuilder &_fbb, const MonsterT *_o);
+inline flatbuffers::Offset<Monster> CreateMonster(flatbuffers::FlatBufferBuilder &_fbb, const MonsterT *_o, const flatbuffers::rehasher_function_t *rehasher = nullptr);
 
 struct WeaponT : public flatbuffers::NativeTable {
   std::string name;
@@ -224,7 +224,7 @@ struct Weapon FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
            VerifyField<int16_t>(verifier, VT_DAMAGE) &&
            verifier.EndTable();
   }
-  std::unique_ptr<WeaponT> UnPack() const;
+  std::unique_ptr<WeaponT> UnPack(const flatbuffers::resolver_function_t *resolver = nullptr) const;
 };
 
 struct WeaponBuilder {
@@ -255,9 +255,9 @@ inline flatbuffers::Offset<Weapon> CreateWeaponDirect(flatbuffers::FlatBufferBui
   return CreateWeapon(_fbb, name ? _fbb.CreateString(name) : 0, damage);
 }
 
-inline flatbuffers::Offset<Weapon> CreateWeapon(flatbuffers::FlatBufferBuilder &_fbb, const WeaponT *_o);
+inline flatbuffers::Offset<Weapon> CreateWeapon(flatbuffers::FlatBufferBuilder &_fbb, const WeaponT *_o, const flatbuffers::rehasher_function_t *rehasher = nullptr);
 
-inline std::unique_ptr<MonsterT> Monster::UnPack() const {
+inline std::unique_ptr<MonsterT> Monster::UnPack(const flatbuffers::resolver_function_t *resolver) const {
   auto _o = new MonsterT();
   { auto _e = pos(); if (_e) _o->pos = std::unique_ptr<Vec3>(new Vec3(*_e)); };
   { auto _e = mana(); _o->mana = _e; };
@@ -265,13 +265,13 @@ inline std::unique_ptr<MonsterT> Monster::UnPack() const {
   { auto _e = name(); if (_e) _o->name = _e->str(); };
   { auto _e = inventory(); if (_e) { for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->inventory.push_back(_e->Get(_i)); } } };
   { auto _e = color(); _o->color = _e; };
-  { auto _e = weapons(); if (_e) { for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->weapons.push_back(_e->Get(_i)->UnPack()); } } };
+  { auto _e = weapons(); if (_e) { for (flatbuffers::uoffset_t _i = 0; _i < _e->size(); _i++) { _o->weapons.push_back(_e->Get(_i)->UnPack(resolver)); } } };
   { auto _e = equipped_type(); _o->equipped.type = _e; };
-  { auto _e = equipped(); if (_e) _o->equipped.table = EquipmentUnion::UnPack(_e, equipped_type()); };
+  { auto _e = equipped(); if (_e) _o->equipped.table = EquipmentUnion::UnPack(_e, equipped_type(), resolver); };
   return std::unique_ptr<MonsterT>(_o);
 }
 
-inline flatbuffers::Offset<Monster> CreateMonster(flatbuffers::FlatBufferBuilder &_fbb, const MonsterT *_o) {
+inline flatbuffers::Offset<Monster> CreateMonster(flatbuffers::FlatBufferBuilder &_fbb, const MonsterT *_o, const flatbuffers::rehasher_function_t *rehasher) {
   return CreateMonster(_fbb,
     _o->pos ? _o->pos.get() : 0,
     _o->mana,
@@ -279,19 +279,19 @@ inline flatbuffers::Offset<Monster> CreateMonster(flatbuffers::FlatBufferBuilder
     _o->name.size() ? _fbb.CreateString(_o->name) : 0,
     _o->inventory.size() ? _fbb.CreateVector(_o->inventory) : 0,
     _o->color,
-    _o->weapons.size() ? _fbb.CreateVector<flatbuffers::Offset<Weapon>>(_o->weapons.size(), [&](size_t i) { return CreateWeapon(_fbb, _o->weapons[i].get()); }) : 0,
+    _o->weapons.size() ? _fbb.CreateVector<flatbuffers::Offset<Weapon>>(_o->weapons.size(), [&](size_t i) { return CreateWeapon(_fbb, _o->weapons[i].get(), rehasher); }) : 0,
     _o->equipped.type,
     _o->equipped.Pack(_fbb));
 }
 
-inline std::unique_ptr<WeaponT> Weapon::UnPack() const {
+inline std::unique_ptr<WeaponT> Weapon::UnPack(const flatbuffers::resolver_function_t *resolver) const {
   auto _o = new WeaponT();
   { auto _e = name(); if (_e) _o->name = _e->str(); };
   { auto _e = damage(); _o->damage = _e; };
   return std::unique_ptr<WeaponT>(_o);
 }
 
-inline flatbuffers::Offset<Weapon> CreateWeapon(flatbuffers::FlatBufferBuilder &_fbb, const WeaponT *_o) {
+inline flatbuffers::Offset<Weapon> CreateWeapon(flatbuffers::FlatBufferBuilder &_fbb, const WeaponT *_o, const flatbuffers::rehasher_function_t *rehasher) {
   return CreateWeapon(_fbb,
     _o->name.size() ? _fbb.CreateString(_o->name) : 0,
     _o->damage);
@@ -305,18 +305,18 @@ inline bool VerifyEquipment(flatbuffers::Verifier &verifier, const void *union_o
   }
 }
 
-inline flatbuffers::NativeTable *EquipmentUnion::UnPack(const void *union_obj, Equipment type) {
+inline flatbuffers::NativeTable *EquipmentUnion::UnPack(const void *union_obj, Equipment type, const flatbuffers::resolver_function_t *resolver) {
   switch (type) {
     case Equipment_NONE: return nullptr;
-    case Equipment_Weapon: return reinterpret_cast<const Weapon *>(union_obj)->UnPack().release();
+    case Equipment_Weapon: return reinterpret_cast<const Weapon *>(union_obj)->UnPack(resolver).release();
     default: return nullptr;
   }
 }
 
-inline flatbuffers::Offset<void> EquipmentUnion::Pack(flatbuffers::FlatBufferBuilder &_fbb) const {
+inline flatbuffers::Offset<void> EquipmentUnion::Pack(flatbuffers::FlatBufferBuilder &_fbb, const flatbuffers::rehasher_function_t *rehasher) const {
   switch (type) {
     case Equipment_NONE: return 0;
-    case Equipment_Weapon: return CreateWeapon(_fbb, reinterpret_cast<const WeaponT *>(table)).Union();
+    case Equipment_Weapon: return CreateWeapon(_fbb, reinterpret_cast<const WeaponT *>(table), rehasher).Union();
     default: return 0;
   }
 }

src/idl_gen_cpp.cpp

@@ -361,25 +361,32 @@ class CppGenerator : public BaseGenerator {
     return (inclass ? "static " : "") +
            std::string("flatbuffers::NativeTable *") +
            (inclass ? "" : enum_def.name + "Union::") +
-           "UnPack(const void *union_obj, " + enum_def.name + " type)";
+           "UnPack(const void *union_obj, " + enum_def.name +
+           " type, const flatbuffers::resolver_function_t *resolver)";
   }
 
   std::string UnionPackSignature(EnumDef &enum_def, bool inclass) {
     return "flatbuffers::Offset<void> " +
            (inclass ? "" : enum_def.name + "Union::") +
-           "Pack(flatbuffers::FlatBufferBuilder &_fbb) const";
+           "Pack(flatbuffers::FlatBufferBuilder &_fbb, " +
+           "const flatbuffers::rehasher_function_t *rehasher" +
+           (inclass ? " = nullptr" : "") + ") const";
   }
 
-  std::string TableCreateSignature(StructDef &struct_def) {
+  std::string TableCreateSignature(StructDef &struct_def, bool predecl) {
     return "inline flatbuffers::Offset<" + struct_def.name + "> Create" +
            struct_def.name  +
            "(flatbuffers::FlatBufferBuilder &_fbb, const " +
-           NativeName(struct_def.name) + " *_o)";
+           NativeName(struct_def.name) +
+           " *_o, const flatbuffers::rehasher_function_t *rehasher" +
+           (predecl ? " = nullptr" : "") + ")";
   }
 
   std::string TableUnPackSignature(StructDef &struct_def, bool inclass) {
     return "std::unique_ptr<" + NativeName(struct_def.name) + "> " +
-           (inclass ? "" : struct_def.name + "::") + "UnPack() const";
+           (inclass ? "" : struct_def.name + "::") +
+           "UnPack(const flatbuffers::resolver_function_t *resolver" +
+           (inclass ? " = nullptr" : "") + ") const";
   }
 
   // Generate an enum declaration and an enum string lookup table.
@@ -524,7 +531,7 @@ class CppGenerator : public BaseGenerator {
         } else {
           code += ": return reinterpret_cast<const ";
           code += WrapInNameSpace(*ev.struct_def);
-          code += " *>(union_obj)->UnPack().release();\n";
+          code += " *>(union_obj)->UnPack(resolver).release();\n";
         }
       }
       code += "    default: return nullptr;\n  }\n}\n\n";
@@ -540,7 +547,7 @@ class CppGenerator : public BaseGenerator {
           code += ": return Create" + ev.struct_def->name;
           code += "(_fbb, reinterpret_cast<const ";
           code += NativeName(WrapInNameSpace(*ev.struct_def));
-          code += " *>(table)).Union();\n";
+          code += " *>(table), rehasher).Union();\n";
         }
       }
       code += "    default: return 0;\n  }\n}\n\n";
@@ -638,8 +645,10 @@ class CppGenerator : public BaseGenerator {
         auto &field = **it;
         if (!field.deprecated &&  // Deprecated fields won't be accessible.
             field.value.type.base_type != BASE_TYPE_UTYPE) {
-          code += "  " + GenTypeNative(field.value.type, false) + " ";
-          code += field.name + ";\n";
+          auto type = GenTypeNative(field.value.type, false);
+          auto cpp_type = field.attributes.Lookup("cpp_type");
+          code += "  " + (cpp_type ? cpp_type->constant + " *" : type+ " ") +
+                  field.name + ";\n";
         }
       }
       code += "};\n\n";
@@ -945,7 +954,7 @@ class CppGenerator : public BaseGenerator {
     if (parser_.opts.generate_object_based_api) {
       // Generate a pre-declaration for a CreateX method that works with an
       // unpacked C++ object.
-      code += TableCreateSignature(struct_def) + ";\n\n";
+      code += TableCreateSignature(struct_def, true) + ";\n\n";
     }
   }
 
@@ -982,7 +991,7 @@ class CppGenerator : public BaseGenerator {
                            WrapInNameSpace (*type.struct_def) + "(*" + val + "))";
                   }
                 } else {
-                  return val + "->UnPack()";
+                  return val + "->UnPack(resolver)";
                 }
               default:
                 return val;
@@ -1009,17 +1018,29 @@ class CppGenerator : public BaseGenerator {
               code += prefix + deref + union_field.name + ".type = _e;";
               break;
             }
-            case BASE_TYPE_UNION:
+            case BASE_TYPE_UNION: {
               code += prefix + dest + ".table = ";
               code += field.value.type.enum_def->name;
               code += "Union::UnPack(_e, ";
-              code += field.name + UnionTypeFieldSuffix() + "());";
+              code += field.name + UnionTypeFieldSuffix() + "(), resolver);";
               break;
-            default:
-              code += assign + gen_unpack_val(field.value.type, "_e", false);
+            }
+            default: {
+              auto cpp_type = field.attributes.Lookup("cpp_type");
+              if (cpp_type) {
+                code += prefix;
+                code += "if (resolver) (*resolver)(reinterpret_cast<void **>(&";
+                code += dest;
+                code += "), static_cast<flatbuffers::hash_value_t>(_e)); else ";
+                code += dest + " = nullptr";
+              } else {
+                code += assign;
+                code += gen_unpack_val(field.value.type, "_e", false);
+              }
               code += ";";
               break;
             }
+          }
           code += " };\n";
         }
       }
@@ -1027,7 +1048,7 @@ class CppGenerator : public BaseGenerator {
       code += ">(_o);\n}\n\n";
 
       // Generate a CreateX method that works with an unpacked C++ object.
-      code += TableCreateSignature(struct_def) + " {\n";
+      code += TableCreateSignature(struct_def, false) + " {\n";
       auto before_return_statement = code.size();
       code += "  return Create";
       code += struct_def.name + "(_fbb";
@@ -1044,6 +1065,10 @@ class CppGenerator : public BaseGenerator {
             field_name += ".type";
           }
           auto accessor = "_o->" + field_name;
+          if (field.attributes.Lookup("cpp_type"))
+            accessor = "rehasher ? static_cast<" +
+                       GenTypeBasic(field.value.type, false) +
+                       ">((*rehasher)(" + accessor + ")) : 0";
           auto ptrprefix = accessor + " ? ";
           auto stlprefix = accessor + ".size() ? ";
           auto postfix = " : 0";
@@ -1074,7 +1099,7 @@ class CppGenerator : public BaseGenerator {
                     code += vector_type.struct_def->name + ">>(" + accessor;
                     code += ".size(), [&](size_t i) { return Create";
                     code += vector_type.struct_def->name + "(_fbb, " + accessor;
-                    code += "[i].get()); })";
+                    code += "[i].get(), rehasher); })";
                   }
                   break;
                 default:
@@ -1093,7 +1118,7 @@ class CppGenerator : public BaseGenerator {
               } else {
                 code += ptrprefix + "Create";
                 code += field.value.type.struct_def->name;
-                code += "(_fbb, " + accessor + ".get())" + postfix;
+                code += "(_fbb, " + accessor + ".get(), rehasher)" + postfix;
               }
               break;
             default:

src/idl_parser.cpp

@@ -657,6 +657,11 @@ CheckedError Parser::ParseField(StructDef &struct_def) {
               "only int, uint, long and ulong data types support hashing.");
     }
   }
+  auto cpp_type = field->attributes.Lookup("cpp_type");
+  if (cpp_type) {
+    if (!hash_name)
+      return Error("cpp_type can only be used with a hashed field");
+  }
   if (field->deprecated && struct_def.fixed)
     return Error("can't deprecate fields in a struct");
   field->required = field->attributes.Lookup("required") != nullptr;

tests/monster_test.fbs

@@ -61,7 +61,7 @@ table Monster {
   testhashs64_fnv1:long (id:18, hash:"fnv1_64");
   testhashu64_fnv1:ulong (id:19, hash:"fnv1_64");
   testhashs32_fnv1a:int (id:20, hash:"fnv1a_32");
-  testhashu32_fnv1a:uint (id:21, hash:"fnv1a_32");
+  testhashu32_fnv1a:uint (id:21, hash:"fnv1a_32", cpp_type:"Stat");
   testhashs64_fnv1a:long (id:22, hash:"fnv1a_64");
   testhashu64_fnv1a:ulong (id:23, hash:"fnv1a_64");
   testf:float = 3.14159 (id:25);

tests/test.cpp

@@ -314,17 +314,30 @@ void MutateFlatBuffersTest(uint8_t *flatbuf, std::size_t length) {
 
 // Unpack a FlatBuffer into objects.
 void ObjectFlatBuffersTest(uint8_t *flatbuf) {
+  // Optional: we can specify resolver and rehasher functions to turn hashed
+  // strings into object pointers and back, to implement remote references
+  // and such.
+  auto resolver = flatbuffers::resolver_function_t([](void **pointer_adr,
+                                               flatbuffers::hash_value_t hash) {
+    return nullptr;  // Fail the lookup.
+  });
+  auto rehasher = flatbuffers::rehasher_function_t([](void *pointer) {
+    return 0;
+  });
+
   // Turn a buffer into C++ objects.
-  auto monster1 = GetMonster(flatbuf)->UnPack();
+  auto monster1 = GetMonster(flatbuf)->UnPack(&resolver);
 
   // Re-serialize the data.
   flatbuffers::FlatBufferBuilder fbb1;
-  fbb1.Finish(CreateMonster(fbb1, monster1.get()), MonsterIdentifier());
+  fbb1.Finish(CreateMonster(fbb1, monster1.get(), &rehasher),
+              MonsterIdentifier());
 
   // Unpack again, and re-serialize again.
-  auto monster2 = GetMonster(fbb1.GetBufferPointer())->UnPack();
+  auto monster2 = GetMonster(fbb1.GetBufferPointer())->UnPack(&resolver);
   flatbuffers::FlatBufferBuilder fbb2;
-  fbb2.Finish(CreateMonster(fbb2, monster2.get()), MonsterIdentifier());
+  fbb2.Finish(CreateMonster(fbb2, monster2.get(), &rehasher),
+              MonsterIdentifier());
 
   // Now we've gone full round-trip, the two buffers should match.
   auto len1 = fbb1.GetSize();