Added the hash attribute to ints and longs.

FlatBuffer schema files can now optionally specify a hash attribute that
will allow someone writing json files to enter a string to be hashed
rather than a specific value. The hashing algorithm to use is specified
by the schema.

Currently the only algorithms are fnv1 and fnv1a. There are 32 bit and
64 variatns for each. Additionally, a hashing command line tool was
added so that you can see what a string will hash to without needing to
inspect the flatbuffer binary blob.

Change-Id: I0cb359d0e2dc7d2dc1874b446dc19a17cc77109d

.gitignore

@@ -29,6 +29,7 @@ proguard-project.txt
 linklint_results
 Makefile
 flatc
+flathash
 flattests
 flatsamplebinary
 flatsampletext

CMakeLists.txt

@@ -7,6 +7,7 @@ option(FLATBUFFERS_CODE_COVERAGE "Enable the code coverage build option." OFF)
 option(FLATBUFFERS_BUILD_TESTS "Enable the build of tests and samples." ON)
 option(FLATBUFFERS_INSTALL "Enable the installation of targets." ON)
 option(FLATBUFFERS_BUILD_FLATC "Enable the build of the flatbuffers compiler" ON)
+option(FLATBUFFERS_BUILD_FLATHASH "Enable the build of flathash" ON)
 
 if(NOT FLATBUFFERS_BUILD_FLATC AND FLATBUFFERS_BUILD_TESTS)
     message(WARNING
@@ -16,6 +17,7 @@ endif()
 
 set(FlatBuffers_Compiler_SRCS
   include/flatbuffers/flatbuffers.h
+  include/flatbuffers/hash.h
   include/flatbuffers/idl.h
   include/flatbuffers/util.h
   src/idl_parser.cpp
@@ -27,8 +29,14 @@ set(FlatBuffers_Compiler_SRCS
   src/flatc.cpp
 )
 
+set(FlatHash_SRCS
+  include/flatbuffers/hash.h
+  src/flathash.cpp
+)
+
 set(FlatBuffers_Tests_SRCS
   include/flatbuffers/flatbuffers.h
+  include/flatbuffers/hash.h
   include/flatbuffers/idl.h
   include/flatbuffers/util.h
   src/idl_parser.cpp
@@ -48,6 +56,7 @@ set(FlatBuffers_Sample_Binary_SRCS
 
 set(FlatBuffers_Sample_Text_SRCS
   include/flatbuffers/flatbuffers.h
+  include/flatbuffers/hash.h
   include/flatbuffers/idl.h
   include/flatbuffers/util.h
   src/idl_parser.cpp
@@ -83,7 +92,11 @@ endif(BIICODE)
 include_directories(include)
 
 if(FLATBUFFERS_BUILD_FLATC)
-add_executable(flatc ${FlatBuffers_Compiler_SRCS})
+  add_executable(flatc ${FlatBuffers_Compiler_SRCS})
+endif()
+
+if(FLATBUFFERS_BUILD_FLATC)
+  add_executable(flathash ${FlatHash_SRCS})
 endif()
 
 function(compile_flatbuffers_schema_to_cpp SRC_FBS)

include/flatbuffers/hash.h

+/*
+ * Copyright 2015 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FLATBUFFERS_HASH_H_
+#define FLATBUFFERS_HASH_H_
+
+#include <cstdint>
+#include <cstring>
+
+namespace flatbuffers {
+
+template <typename T>
+struct FnvTraits {
+  static const T kFnvPrime;
+  static const T kOffsetBasis;
+};
+
+template <>
+struct FnvTraits<uint32_t> {
+  static const uint32_t kFnvPrime = 0x01000193;
+  static const uint32_t kOffsetBasis = 0x811C9DC5;
+};
+
+template <>
+struct FnvTraits<uint64_t> {
+  static const uint64_t kFnvPrime = 0x00000100000001b3;
+  static const uint64_t kOffsetBasis = 0xcbf29ce484222645;
+};
+
+template <typename T>
+T HashFnv1(const char *input) {
+  T hash = FnvTraits<T>::kOffsetBasis;
+  for (const char *c = input; *c; ++c) {
+    hash *= FnvTraits<T>::kFnvPrime;
+    hash ^= static_cast<unsigned char>(*c);
+  }
+  return hash;
+}
+
+template <typename T>
+T HashFnv1a(const char *input) {
+  T hash = FnvTraits<T>::kOffsetBasis;
+  for (const char *c = input; *c; ++c) {
+    hash ^= static_cast<unsigned char>(*c);
+    hash *= FnvTraits<T>::kFnvPrime;
+  }
+  return hash;
+}
+
+template <typename T>
+struct NamedHashFunction {
+  const char *name;
+
+  typedef T (*HashFunction)(const char*);
+  HashFunction function;
+};
+
+const NamedHashFunction<uint32_t> kHashFunctions32[] = {
+  { "fnv1_32",  HashFnv1<uint32_t> },
+  { "fnv1a_32", HashFnv1a<uint32_t> },
+};
+
+const NamedHashFunction<uint64_t> kHashFunctions64[] = {
+  { "fnv1_64",  HashFnv1<uint64_t> },
+  { "fnv1a_64", HashFnv1a<uint64_t> },
+};
+
+inline NamedHashFunction<uint32_t>::HashFunction FindHashFunction32(
+    const char *name) {
+  std::size_t size = sizeof(kHashFunctions32) / sizeof(kHashFunctions32[0]);
+  for (std::size_t i = 0; i < size; ++i) {
+    if (std::strcmp(name, kHashFunctions32[i].name) == 0) {
+      return kHashFunctions32[i].function;
+    }
+  }
+  return nullptr;
+}
+
+inline NamedHashFunction<uint64_t>::HashFunction FindHashFunction64(
+    const char *name) {
+  std::size_t size = sizeof(kHashFunctions64) / sizeof(kHashFunctions64[0]);
+  for (std::size_t i = 0; i < size; ++i) {
+    if (std::strcmp(name, kHashFunctions64[i].name) == 0) {
+      return kHashFunctions64[i].function;
+    }
+  }
+  return nullptr;
+}
+
+}  // namespace flatbuffers
+
+#endif  // FLATBUFFERS_HASH_H_

include/flatbuffers/idl.h

@@ -24,6 +24,7 @@
 #include <functional>
 
 #include "flatbuffers/flatbuffers.h"
+#include "flatbuffers/hash.h"
 
 // This file defines the data types representing a parsed IDL (Interface
 // Definition Language) / schema file.
@@ -280,6 +281,7 @@ class Parser {
     known_attributes_.insert("deprecated");
     known_attributes_.insert("required");
     known_attributes_.insert("key");
+    known_attributes_.insert("hash");
     known_attributes_.insert("id");
     known_attributes_.insert("force_align");
     known_attributes_.insert("bit_flags");
@@ -334,6 +336,7 @@ class Parser {
   uoffset_t ParseVector(const Type &type);
   void ParseMetaData(Definition &def);
   bool TryTypedValue(int dtoken, bool check, Value &e, BaseType req);
+  void ParseHash(Value &e, FieldDef* field);
   void ParseSingleValue(Value &e);
   int64_t ParseIntegerFromString(Type &type);
   StructDef *LookupCreateStruct(const std::string &name);

src/flathash.cpp

+/*
+ * Copyright 2015 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <iostream>
+#include <sstream>
+#include <string>
+#include "flatbuffers/hash.h"
+
+enum OutputFormat {
+  kDecimal,
+  kHexadecimal,
+  kHexadecimal0x,
+};
+
+int main(int argc, char* argv[]) {
+  const char* name = argv[0];
+  if (argc <= 1) {
+    printf("%s HASH [OPTION]... STRING... [-- STRING...]\n", name);
+    printf("Available hashing algorithms:\n  32 bit:\n");
+    size_t size = sizeof(flatbuffers::kHashFunctions32) /
+                  sizeof(flatbuffers::kHashFunctions32[0]);
+    for (size_t i = 0; i < size; ++i) {
+      printf("    * %s\n", flatbuffers::kHashFunctions32[i].name);
+    }
+    printf("  64 bit:\n");
+    size = sizeof(flatbuffers::kHashFunctions64) /
+           sizeof(flatbuffers::kHashFunctions64[0]);
+    for (size_t i = 0; i < size; ++i) {
+      printf("    * %s\n", flatbuffers::kHashFunctions64[i].name);
+    }
+    printf(
+        "  -d         Output hash in decimal.\n"
+        "  -x         Output hash in hexadecimal.\n"
+        "  -0x        Output hash in hexadecimal and prefix with 0x.\n"
+        "  -c         Append the string to the output in a c-style comment.\n");
+    return 0;
+  }
+
+  const char* hash_algorithm = argv[1];
+
+  flatbuffers::NamedHashFunction<uint32_t>::HashFunction hash_function32 =
+      flatbuffers::FindHashFunction32(hash_algorithm);
+  flatbuffers::NamedHashFunction<uint64_t>::HashFunction hash_function64 =
+      flatbuffers::FindHashFunction64(hash_algorithm);
+
+  if (!hash_function32 && !hash_function64) {
+    printf("\"%s\" is not a known hash algorithm.\n", hash_algorithm);
+    return 0;
+  }
+
+  OutputFormat output_format = kHexadecimal;
+  bool annotate = false;
+  bool escape_dash = false;
+  for (int i = 2; i < argc; i++) {
+    const char* arg = argv[i];
+    if (!escape_dash && arg[0] == '-') {
+      std::string opt = arg;
+      if (opt == "-d")       output_format = kDecimal;
+      else if (opt == "-x")  output_format = kHexadecimal;
+      else if (opt == "-0x") output_format = kHexadecimal0x;
+      else if (opt == "-c")  annotate = true;
+      else if (opt == "--")  escape_dash = true;
+      else printf("Unrecognized argument: \"%s\"\n", arg);
+    } else {
+      std::stringstream ss;
+      if (output_format == kDecimal) {
+        ss << std::dec;
+      } else if (output_format == kHexadecimal) {
+        ss << std::hex;
+      } else if (output_format == kHexadecimal0x) {
+        ss << std::hex;
+        ss << "0x";
+      }
+      if (hash_function32)
+        ss << hash_function32(arg);
+      else if (hash_function64)
+        ss << hash_function64(arg);
+
+      if (annotate)
+        ss << " /* \"" << arg << "\" */";
+
+      ss << "\n";
+
+      std::cout << ss.str();
+    }
+  }
+  return 0;
+}
+

src/idl_parser.cpp

@@ -18,6 +18,7 @@
 #include <list>
 
 #include "flatbuffers/flatbuffers.h"
+#include "flatbuffers/hash.h"
 #include "flatbuffers/idl.h"
 #include "flatbuffers/util.h"
 
@@ -392,6 +393,27 @@ void Parser::ParseField(StructDef &struct_def) {
   field.doc_comment = dc;
   ParseMetaData(field);
   field.deprecated = field.attributes.Lookup("deprecated") != nullptr;
+  auto hash_name = field.attributes.Lookup("hash");
+  if (hash_name) {
+    switch (type.base_type) {
+      case BASE_TYPE_INT:
+      case BASE_TYPE_UINT: {
+        if (FindHashFunction32(hash_name->constant.c_str()) == nullptr)
+          Error("Unknown hashing algorithm for 32 bit types: " +
+                hash_name->constant);
+        break;
+      }
+      case BASE_TYPE_LONG:
+      case BASE_TYPE_ULONG: {
+        if (FindHashFunction64(hash_name->constant.c_str()) == nullptr)
+          Error("Unknown hashing algorithm for 64 bit types: " +
+                hash_name->constant);
+        break;
+      }
+      default:
+        Error("only int, uint, long and ulong data types support hashing.");
+    }
+  }
   if (field.deprecated && struct_def.fixed)
     Error("can't deprecate fields in a struct");
   field.required = field.attributes.Lookup("required") != nullptr;
@@ -465,6 +487,18 @@ void Parser::ParseAnyValue(Value &val, FieldDef *field) {
       val.constant = NumToString(ParseVector(val.type.VectorType()));
       break;
     }
+    case BASE_TYPE_INT:
+    case BASE_TYPE_UINT:
+    case BASE_TYPE_LONG:
+    case BASE_TYPE_ULONG: {
+      if (field && field->attributes.Lookup("hash") &&
+          (token_ == kTokenIdentifier || token_ == kTokenStringConstant)) {
+        ParseHash(val, field);
+      } else {
+        ParseSingleValue(val);
+      }
+      break;
+    }
     default:
       ParseSingleValue(val);
       break;
@@ -583,7 +617,7 @@ uoffset_t Parser::ParseVector(const Type &type) {
     if ((!strict_json_ || !count) && IsNext(']')) break;
     Value val;
     val.type = type;
-    ParseAnyValue(val, NULL);
+    ParseAnyValue(val, nullptr);
     field_stack_.push_back(std::make_pair(val, nullptr));
     count++;
     if (IsNext(']')) break;
@@ -689,6 +723,31 @@ int64_t Parser::ParseIntegerFromString(Type &type) {
   return result;
 }
 
+
+void Parser::ParseHash(Value &e, FieldDef* field) {
+  assert(field);
+  Value *hash_name = field->attributes.Lookup("hash");
+  switch (e.type.base_type) {
+    case BASE_TYPE_INT:
+    case BASE_TYPE_UINT: {
+      auto hash = FindHashFunction32(hash_name->constant.c_str());
+      uint32_t hashed_value = hash(attribute_.c_str());
+      e.constant = NumToString(hashed_value);
+      break;
+    }
+    case BASE_TYPE_LONG:
+    case BASE_TYPE_ULONG: {
+      auto hash = FindHashFunction64(hash_name->constant.c_str());
+      uint64_t hashed_value = hash(attribute_.c_str());
+      e.constant = NumToString(hashed_value);
+      break;
+    }
+    default:
+      assert(0);
+  }
+  Next();
+}
+
 void Parser::ParseSingleValue(Value &e) {
   // First check if this could be a string/identifier enum value:
   if (e.type.base_type != BASE_TYPE_STRING &&

tests/monster_test.fbs

@@ -44,6 +44,14 @@ table Monster {
   testnestedflatbuffer:[ubyte] (id:13, nested_flatbuffer: "Monster");
   testempty:Stat (id:14);
   testbool:bool (id:15);
+  testhashs32_fnv1:int (id:16, hash:"fnv1_32");
+  testhashu32_fnv1:uint (id:17, hash:"fnv1_32");
+  testhashs64_fnv1:long (id:18, hash:"fnv1_64");
+  testhashu64_fnv1:ulong (id:19, hash:"fnv1_64");
+  testhashs32_fnv1a:int (id:20, hash:"fnv1a_32");
+  testhashu32_fnv1a:uint (id:21, hash:"fnv1a_32");
+  testhashs64_fnv1a:long (id:22, hash:"fnv1a_64");
+  testhashu64_fnv1a:ulong (id:23, hash:"fnv1a_64");
 }
 
 root_type Monster;

tests/monster_test_generated.h

@@ -141,6 +141,10 @@ struct Monster FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   const Monster *testnestedflatbuffer_nested_root() { return flatbuffers::GetRoot<Monster>(testnestedflatbuffer()->Data()); }
   const Stat *testempty() const { return GetPointer<const Stat *>(32); }
   uint8_t testbool() const { return GetField<uint8_t>(34, 0); }
+  int32_t testhashs32_fnv1() const { return GetField<int32_t>(36, 0); }
+  uint32_t testhashu32_fnv1() const { return GetField<uint32_t>(38, 0); }
+  int64_t testhashs64_fnv1() const { return GetField<int64_t>(40, 0); }
+  uint64_t testhashu64_fnv1() const { return GetField<uint64_t>(42, 0); }
   bool Verify(flatbuffers::Verifier &verifier) const {
     return VerifyTableStart(verifier) &&
            VerifyField<Vec3>(verifier, 4 /* pos */) &&
@@ -169,6 +173,10 @@ struct Monster FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
            VerifyField<flatbuffers::uoffset_t>(verifier, 32 /* testempty */) &&
            verifier.VerifyTable(testempty()) &&
            VerifyField<uint8_t>(verifier, 34 /* testbool */) &&
+           VerifyField<int32_t>(verifier, 36 /* testhashs32_fnv1 */) &&
+           VerifyField<uint32_t>(verifier, 38 /* testhashu32_fnv1 */) &&
+           VerifyField<int64_t>(verifier, 40 /* testhashs64_fnv1 */) &&
+           VerifyField<uint64_t>(verifier, 42 /* testhashu64_fnv1 */) &&
            verifier.EndTable();
   }
 };
@@ -191,10 +199,14 @@ struct MonsterBuilder {
   void add_testnestedflatbuffer(flatbuffers::Offset<flatbuffers::Vector<uint8_t>> testnestedflatbuffer) { fbb_.AddOffset(30, testnestedflatbuffer); }
   void add_testempty(flatbuffers::Offset<Stat> testempty) { fbb_.AddOffset(32, testempty); }
   void add_testbool(uint8_t testbool) { fbb_.AddElement<uint8_t>(34, testbool, 0); }
+  void add_testhashs32_fnv1(int32_t testhashs32_fnv1) { fbb_.AddElement<int32_t>(36, testhashs32_fnv1, 0); }
+  void add_testhashu32_fnv1(uint32_t testhashu32_fnv1) { fbb_.AddElement<uint32_t>(38, testhashu32_fnv1, 0); }
+  void add_testhashs64_fnv1(int64_t testhashs64_fnv1) { fbb_.AddElement<int64_t>(40, testhashs64_fnv1, 0); }
+  void add_testhashu64_fnv1(uint64_t testhashu64_fnv1) { fbb_.AddElement<uint64_t>(42, testhashu64_fnv1, 0); }
   MonsterBuilder(flatbuffers::FlatBufferBuilder &_fbb) : fbb_(_fbb) { start_ = fbb_.StartTable(); }
   MonsterBuilder &operator=(const MonsterBuilder &);
   flatbuffers::Offset<Monster> Finish() {
-    auto o = flatbuffers::Offset<Monster>(fbb_.EndTable(start_, 16));
+    auto o = flatbuffers::Offset<Monster>(fbb_.EndTable(start_, 20));
     fbb_.Required(o, 10);  // name
     return o;
   }
@@ -215,8 +227,16 @@ inline flatbuffers::Offset<Monster> CreateMonster(flatbuffers::FlatBufferBuilder
    flatbuffers::Offset<Monster> enemy = 0,
    flatbuffers::Offset<flatbuffers::Vector<uint8_t>> testnestedflatbuffer = 0,
    flatbuffers::Offset<Stat> testempty = 0,
-   uint8_t testbool = 0) {
+   uint8_t testbool = 0,
+   int32_t testhashs32_fnv1 = 0,
+   uint32_t testhashu32_fnv1 = 0,
+   int64_t testhashs64_fnv1 = 0,
+   uint64_t testhashu64_fnv1 = 0) {
   MonsterBuilder builder_(_fbb);
+  builder_.add_testhashu64_fnv1(testhashu64_fnv1);
+  builder_.add_testhashs64_fnv1(testhashs64_fnv1);
+  builder_.add_testhashu32_fnv1(testhashu32_fnv1);
+  builder_.add_testhashs32_fnv1(testhashs32_fnv1);
   builder_.add_testempty(testempty);
   builder_.add_testnestedflatbuffer(testnestedflatbuffer);
   builder_.add_enemy(enemy);

tests/monsterdata_test.golden

@@ -36,5 +36,13 @@
   testarrayofstring: [
     "test1",
     "test2"
-  ]
+  ],
+  testhashs32_fnv1: -579221183,
+  testhashu32_fnv1: 3715746113,
+  testhashs64_fnv1: 7930699090847568257,
+  testhashu64_fnv1: 7930699090847568257,
+  testhashs32_fnv1a: -1904106383,
+  testhashu32_fnv1a: 2390860913,
+  testhashs64_fnv1a: 4898026182817603057,
+  testhashu64_fnv1a: 4898026182817603057
 }

tests/monsterdata_test.json

@@ -36,5 +36,13 @@
   testarrayofstring: [
     "test1",
     "test2"
-  ]
+  ],
+  testhashs32_fnv1: "This string is being hashed!",
+  testhashu32_fnv1: "This string is being hashed!",
+  testhashs64_fnv1: "This string is being hashed!",
+  testhashu64_fnv1: "This string is being hashed!",
+  testhashs32_fnv1a: "This string is being hashed!",
+  testhashu32_fnv1a: "This string is being hashed!",
+  testhashs64_fnv1a: "This string is being hashed!",
+  testhashu64_fnv1a: "This string is being hashed!",
 }