Added (nested) FlexBuffer JSON parsing and output.

FlexBuffer parser is just 40 lines of code (on top of existing parser!).

Change-Id: Idebebadafb661ca5333f5621139031f6df3c3e1a
Tested: on Linux.

include/flatbuffers/flexbuffers.h

@@ -457,25 +457,61 @@ class Reference {
   }
 
   // Unlike AsString(), this will convert any type to a std::string.
-  std::string ToString() const {
+  std::string ToString() {
+    std::string s;
+    ToString(false, false, s);
+    return s;
+  }
+
+  // Convert any type to a JSON-like string. strings_quoted determines if
+  // string values at the top level receive "" quotes (inside other values
+  // they always do). keys_quoted determines if keys are quoted, at any level.
+  // TODO(wvo): add further options to have indentation/newlines.
+  void ToString(bool strings_quoted, bool keys_quoted, std::string &s) const {
     if (type_ == TYPE_STRING) {
-      return String(Indirect(), byte_width_).c_str();
+      String str(Indirect(), byte_width_);
+      if (strings_quoted) {
+        flatbuffers::EscapeString(str.c_str(), str.length(), &s, true);
+      } else {
+        s.append(str.c_str(), str.length());
+      }
     } else if (IsKey()) {
-      return AsKey();
+      auto str = AsKey();
+      if (keys_quoted) {
+        flatbuffers::EscapeString(str, strlen(str), &s, true);
+      } else {
+        s += str;
+      }
     } else if (IsInt()) {
-      return flatbuffers::NumToString(AsInt64());
+      s += flatbuffers::NumToString(AsInt64());
     } else if (IsUInt()) {
-      return flatbuffers::NumToString(AsUInt64());
+      s += flatbuffers::NumToString(AsUInt64());
     } else if (IsFloat()) {
-      return flatbuffers::NumToString(AsDouble());
+      s += flatbuffers::NumToString(AsDouble());
     } else if (IsNull()) {
-      return "null";
+      s += "null";
     } else if (IsMap()) {
-      return "{..}";  // TODO: show elements.
+      s += "{ ";
+      auto m = AsMap();
+      auto keys = m.Keys();
+      auto vals = m.Values();
+      for (size_t i = 0; i < keys.size(); i++) {
+        keys[i].ToString(true, keys_quoted, s);
+        s += ": ";
+        vals[i].ToString(true, keys_quoted, s);
+        if (i < keys.size() - 1) s += ", ";
+      }
+      s += " }";
     } else if (IsVector()) {
-      return "[..]";  // TODO: show elements.
+      s += "[ ";
+      auto v = AsVector();
+      for (size_t i = 0; i < v.size(); i++) {
+        v[i].ToString(true, keys_quoted, s);
+        if (i < v.size() - 1) s += ", ";
+      }
+      s += " ]";
     } else {
-      return "(?)";
+      s += "(?)";
     }
   }
 
@@ -746,6 +782,17 @@ class Builder FLATBUFFERS_FINAL_CLASS {
     return buf_;
   }
 
+  // Reset all state so we can re-use the buffer.
+  void Clear() {
+    buf_.clear();
+    stack_.clear();
+    finished_ = false;
+    // flags_ remains as-is;
+    force_min_bit_width_ = BIT_WIDTH_8;
+    key_pool.clear();
+    string_pool.clear();
+  }
+
   // All value constructing functions below have two versions: one that
   // takes a key (for placement inside a map) and one that doesn't (for inside
   // vectors and elsewhere).

include/flatbuffers/idl.h

@@ -25,6 +25,7 @@
 #include "flatbuffers/flatbuffers.h"
 #include "flatbuffers/hash.h"
 #include "flatbuffers/reflection.h"
+#include "flatbuffers/flexbuffers.h"
 
 // This file defines the data types representing a parsed IDL (Interface
 // Definition Language) / schema file.
@@ -226,18 +227,20 @@ struct Definition {
 };
 
 struct FieldDef : public Definition {
-  FieldDef() : deprecated(false), required(false), key(false), padding(0) {}
+  FieldDef() : deprecated(false), required(false), key(false),
+               flexbuffer(false), padding(0) {}
 
   Offset<reflection::Field> Serialize(FlatBufferBuilder *builder, uint16_t id,
                                       const Parser &parser) const;
 
   Value value;
-  bool deprecated; // Field is allowed to be present in old data, but can't be
+  bool deprecated; // Field is allowed to be present in old data, but can't be.
                    // written in new data nor accessed in new code.
   bool required;   // Field must always be present.
   bool key;        // Field functions as a key for creating sorted vectors.
   bool native_inline;  // Field will be defined inline (instead of as a pointer)
                        // for native tables if field is a struct.
+  bool flexbuffer; // This field contains FlexBuffer data.
   size_t padding;  // Bytes to always pad after this field.
 };
 
@@ -536,6 +539,11 @@ class Parser : public ParserState {
   // of the schema provided. Returns non-empty error on any problems.
   std::string ConformTo(const Parser &base);
 
+  // Similar to Parse(), but now only accepts JSON to be parsed into a
+  // FlexBuffer.
+  bool ParseFlexBuffer(const char *source, const char *source_filename,
+                       flexbuffers::Builder *builder);
+
   FLATBUFFERS_CHECKED_ERROR CheckInRange(int64_t val, int64_t min, int64_t max);
 
 private:
@@ -559,15 +567,21 @@ private:
   FLATBUFFERS_CHECKED_ERROR ParseAnyValue(Value &val, FieldDef *field,
                                           size_t parent_fieldn,
                                           const StructDef *parent_struct_def);
+  FLATBUFFERS_CHECKED_ERROR ParseTableDelimiters(size_t &fieldn,
+                                                 const StructDef *struct_def,
+              const std::function<CheckedError(const std::string &name)> &body);
   FLATBUFFERS_CHECKED_ERROR ParseTable(const StructDef &struct_def,
                                        std::string *value, uoffset_t *ovalue);
   void SerializeStruct(const StructDef &struct_def, const Value &val);
   void AddVector(bool sortbysize, int count);
+  FLATBUFFERS_CHECKED_ERROR ParseVectorDelimiters(size_t &count,
+                                     const std::function<CheckedError()> &body);
   FLATBUFFERS_CHECKED_ERROR ParseVector(const Type &type, uoffset_t *ovalue);
   FLATBUFFERS_CHECKED_ERROR ParseMetaData(SymbolTable<Value> *attributes);
   FLATBUFFERS_CHECKED_ERROR TryTypedValue(int dtoken, bool check, Value &e,
                                           BaseType req, bool *destmatch);
   FLATBUFFERS_CHECKED_ERROR ParseHash(Value &e, FieldDef* field);
+  FLATBUFFERS_CHECKED_ERROR TokenError();
   FLATBUFFERS_CHECKED_ERROR ParseSingleValue(Value &e);
   FLATBUFFERS_CHECKED_ERROR ParseEnumFromString(Type &type, int64_t *result);
   StructDef *LookupCreateStruct(const std::string &name,
@@ -587,9 +601,9 @@ private:
   FLATBUFFERS_CHECKED_ERROR ParseProtoCurliesOrIdent();
   FLATBUFFERS_CHECKED_ERROR ParseTypeFromProtoType(Type *type);
   FLATBUFFERS_CHECKED_ERROR SkipAnyJsonValue();
-  FLATBUFFERS_CHECKED_ERROR SkipJsonObject();
-  FLATBUFFERS_CHECKED_ERROR SkipJsonArray();
-  FLATBUFFERS_CHECKED_ERROR SkipJsonString();
+  FLATBUFFERS_CHECKED_ERROR ParseFlexBufferValue(flexbuffers::Builder *builder);
+  FLATBUFFERS_CHECKED_ERROR StartParseFile(const char *source,
+                                           const char *source_filename);
   FLATBUFFERS_CHECKED_ERROR DoParse(const char *_source,
                                     const char **include_paths,
                                     const char *source_filename,

include/flatbuffers/util.h

@@ -72,9 +72,8 @@ template<> inline std::string NumToString<double>(double t) {
   // Sadly, std::fixed turns "1" into "1.00000", so here we undo that.
   auto p = s.find_last_not_of('0');
   if (p != std::string::npos) {
-    s.resize(p + 1);  // Strip trailing zeroes.
-    if (s[s.size() - 1] == '.')
-      s.erase(s.size() - 1, 1);  // Strip '.' if a whole number.
+    // Strip trailing zeroes. If it is a whole number, keep one zero.
+    s.resize(p + (s[p] == '.' ? 2 : 1));
   }
   return s;
 }
@@ -361,6 +360,72 @@ inline std::string WordWrap(const std::string in, size_t max_length,
   return wrapped;
 }
 
+inline bool EscapeString(const char *s, size_t length, std::string *_text,
+                         bool allow_non_utf8) {
+  std::string &text = *_text;
+  text += "\"";
+  for (uoffset_t i = 0; i < length; i++) {
+    char c = s[i];
+    switch (c) {
+      case '\n': text += "\\n"; break;
+      case '\t': text += "\\t"; break;
+      case '\r': text += "\\r"; break;
+      case '\b': text += "\\b"; break;
+      case '\f': text += "\\f"; break;
+      case '\"': text += "\\\""; break;
+      case '\\': text += "\\\\"; break;
+      default:
+        if (c >= ' ' && c <= '~') {
+          text += c;
+        } else {
+          // Not printable ASCII data. Let's see if it's valid UTF-8 first:
+          const char *utf8 = s + i;
+          int ucc = FromUTF8(&utf8);
+          if (ucc < 0) {
+            if (allow_non_utf8) {
+              text += "\\x";
+              text += IntToStringHex(static_cast<uint8_t>(c), 2);
+            } else {
+              // There are two cases here:
+              //
+              // 1) We reached here by parsing an IDL file. In that case,
+              // we previously checked for non-UTF-8, so we shouldn't reach
+              // here.
+              //
+              // 2) We reached here by someone calling GenerateText()
+              // on a previously-serialized flatbuffer. The data might have
+              // non-UTF-8 Strings, or might be corrupt.
+              //
+              // In both cases, we have to give up and inform the caller
+              // they have no JSON.
+              return false;
+            }
+          } else {
+            if (ucc <= 0xFFFF) {
+              // Parses as Unicode within JSON's \uXXXX range, so use that.
+              text += "\\u";
+              text += IntToStringHex(ucc, 4);
+            } else if (ucc <= 0x10FFFF) {
+              // Encode Unicode SMP values to a surrogate pair using two \u escapes.
+              uint32_t base = ucc - 0x10000;
+              auto high_surrogate = (base >> 10) + 0xD800;
+              auto low_surrogate = (base & 0x03FF) + 0xDC00;
+              text += "\\u";
+              text += IntToStringHex(high_surrogate, 4);
+              text += "\\u";
+              text += IntToStringHex(low_surrogate, 4);
+            }
+            // Skip past characters recognized.
+            i = static_cast<uoffset_t>(utf8 - s - 1);
+          }
+        }
+        break;
+    }
+  }
+  text += "\"";
+  return true;
+}
+
 }  // namespace flatbuffers
 
 #endif  // FLATBUFFERS_UTIL_H_

src/idl_gen_cpp.cpp

@@ -1343,7 +1343,7 @@ class CppGenerator : public BaseGenerator {
         code_ += "  }";
       }
 
-      if (field.attributes.Lookup("flexbuffer")) {
+      if (field.flexbuffer) {
         code_ += "  flexbuffers::Reference {{FIELD_NAME}}_flexbuffer_root()"
                                                                      " const {";
         code_ += "    auto v = {{FIELD_NAME}}();";

src/idl_gen_text.cpp

@@ -19,6 +19,7 @@
 #include "flatbuffers/flatbuffers.h"
 #include "flatbuffers/idl.h"
 #include "flatbuffers/util.h"
+#include "flatbuffers/flexbuffers.h"
 
 namespace flatbuffers {
 
@@ -101,71 +102,6 @@ template<typename T> bool PrintVector(const Vector<T> &v, Type type,
   return true;
 }
 
-static bool EscapeString(const String &s, std::string *_text, const IDLOptions& opts) {
-  std::string &text = *_text;
-  text += "\"";
-  for (uoffset_t i = 0; i < s.size(); i++) {
-    char c = s[i];
-    switch (c) {
-      case '\n': text += "\\n"; break;
-      case '\t': text += "\\t"; break;
-      case '\r': text += "\\r"; break;
-      case '\b': text += "\\b"; break;
-      case '\f': text += "\\f"; break;
-      case '\"': text += "\\\""; break;
-      case '\\': text += "\\\\"; break;
-      default:
-        if (c >= ' ' && c <= '~') {
-          text += c;
-        } else {
-          // Not printable ASCII data. Let's see if it's valid UTF-8 first:
-          const char *utf8 = s.c_str() + i;
-          int ucc = FromUTF8(&utf8);
-          if (ucc < 0) {
-            if (opts.allow_non_utf8) {
-              text += "\\x";
-              text += IntToStringHex(static_cast<uint8_t>(c), 2);
-            } else {
-              // There are two cases here:
-              //
-              // 1) We reached here by parsing an IDL file. In that case,
-              // we previously checked for non-UTF-8, so we shouldn't reach
-              // here.
-              //
-              // 2) We reached here by someone calling GenerateText()
-              // on a previously-serialized flatbuffer. The data might have
-              // non-UTF-8 Strings, or might be corrupt.
-              //
-              // In both cases, we have to give up and inform the caller
-              // they have no JSON.
-              return false;
-            }
-          } else {
-            if (ucc <= 0xFFFF) {
-              // Parses as Unicode within JSON's \uXXXX range, so use that.
-              text += "\\u";
-              text += IntToStringHex(ucc, 4);
-            } else if (ucc <= 0x10FFFF) {
-              // Encode Unicode SMP values to a surrogate pair using two \u escapes.
-              uint32_t base = ucc - 0x10000;
-              auto high_surrogate = (base >> 10) + 0xD800;
-              auto low_surrogate = (base & 0x03FF) + 0xDC00;
-              text += "\\u";
-              text += IntToStringHex(high_surrogate, 4);
-              text += "\\u";
-              text += IntToStringHex(low_surrogate, 4);
-            }
-            // Skip past characters recognized.
-            i = static_cast<uoffset_t>(utf8 - s.c_str() - 1);
-          }
-        }
-        break;
-    }
-  }
-  text += "\"";
-  return true;
-}
-
 // Specialization of Print above for pointer types.
 template<> bool Print<const void *>(const void *val,
                                     Type type, int indent,
@@ -189,7 +125,8 @@ template<> bool Print<const void *>(const void *val,
       }
       break;
     case BASE_TYPE_STRING: {
-      if (!EscapeString(*reinterpret_cast<const String *>(val), _text, opts)) {
+      auto s = reinterpret_cast<const String *>(val);
+      if (!EscapeString(s->c_str(), s->Length(), _text, opts.allow_non_utf8)) {
         return false;
       }
       break;
@@ -238,6 +175,11 @@ static bool GenFieldOffset(const FieldDef &fd, const Table *table, bool fixed,
     assert(IsStruct(fd.value.type));
     val = reinterpret_cast<const Struct *>(table)->
             GetStruct<const void *>(fd.value.offset);
+  } else if (fd.flexbuffer) {
+    auto vec = table->GetPointer<const Vector<uint8_t> *>(fd.value.offset);
+    auto root = flexbuffers::GetRoot(vec->data(), vec->size());
+    root.ToString(true, false, *_text);
+    return true;
   } else {
     val = IsStruct(fd.value.type)
       ? table->GetStruct<const void *>(fd.value.offset)

src/reflection.cpp

@@ -87,8 +87,11 @@ std::string GetAnyValueS(reflection::BaseType type, const uint8_t *data,
             auto &fielddef = **it;
             if (!table_field->CheckField(fielddef.offset())) continue;
             auto val = GetAnyFieldS(*table_field, fielddef, schema);
-            if (fielddef.type()->base_type() == reflection::String)
-              val = "\"" + val + "\"";  // Doesn't deal with escape codes etc.
+            if (fielddef.type()->base_type() == reflection::String) {
+              std::string esc;
+              flatbuffers::EscapeString(val.c_str(), val.length(), &esc, true);
+              val = esc;
+            }
             s += fielddef.name()->str();
             s += ": ";
             s += val;

tests/monsterdata_test.golden

 {
   pos: {
-    x: 1,
-    y: 2,
-    z: 3,
-    test1: 3,
+    x: 1.0,
+    y: 2.0,
+    z: 3.0,
+    test1: 3.0,
     test2: Green,
     test3: {
       a: 10,
@@ -63,5 +63,6 @@
   testhashs32_fnv1a: -1904106383,
   testhashu32_fnv1a: 2390860913,
   testhashs64_fnv1a: 4898026182817603057,
-  testhashu64_fnv1a: 4898026182817603057
+  testhashu64_fnv1a: 4898026182817603057,
+  flex: 1234
 }

tests/test.cpp

@@ -309,12 +309,11 @@ void AccessFlatBufferTest(const uint8_t *flatbuf, size_t length,
 
   // Test flexbuffer if available:
   auto flex = monster->flex();
-  if (flex) {
-    // flex is a vector of bytes you can memcpy. However, if you
-    // actually want to access the nested data, this is a convenient
-    // accessor that directly gives you the root value:
-    TEST_EQ(monster->flex_flexbuffer_root().AsInt16(), 1234);
-  }
+  // flex is a vector of bytes you can memcpy etc.
+  TEST_EQ(flex->size(), 4);  // Encoded FlexBuffer bytes.
+  // However, if you actually want to access the nested data, this is a
+  // convenient accessor that directly gives you the root value:
+  TEST_EQ(monster->flex_flexbuffer_root().AsInt16(), 1234);
 
   // Since Flatbuffers uses explicit mechanisms to override the default
   // compiler alignment, double check that the compiler indeed obeys them:
@@ -919,7 +918,8 @@ void FuzzTest2() {
       AddToSchemaAndInstances(("  " + field_name + ":").c_str(),
                               deprecated ? "" : (field_name + ": ").c_str());
       // Pick random type:
-      int base_type = lcg_rand() % (flatbuffers::BASE_TYPE_UNION + 1);
+      auto base_type = static_cast<flatbuffers::BaseType>(
+                         lcg_rand() % (flatbuffers::BASE_TYPE_UNION + 1));
       switch (base_type) {
         case flatbuffers::BASE_TYPE_STRING:
           if (is_struct) {
@@ -970,7 +970,9 @@ void FuzzTest2() {
             // We want each instance to use its own random value.
             for (int inst = 0; inst < instances_per_definition; inst++)
               definitions[definition].instances[inst] +=
-              flatbuffers::NumToString(lcg_rand() % 128).c_str();
+              flatbuffers::IsFloat(base_type)
+                  ? flatbuffers::NumToString<double>(lcg_rand() % 128).c_str()
+                  : flatbuffers::NumToString<int>(lcg_rand() % 128).c_str();
           }
       }
       AddToSchemaAndInstances(
@@ -1548,7 +1550,7 @@ void FlexBuffersTest() {
   TEST_EQ(vec[2].AsDouble(), 4.0);
   TEST_EQ(vec[2].AsString().IsTheEmptyString(), true);  // Wrong Type.
   TEST_EQ_STR(vec[2].AsString().c_str(), "");  // This still works though.
-  TEST_EQ_STR(vec[2].ToString().c_str(), "4");  // Or have it converted.
+  TEST_EQ_STR(vec[2].ToString().c_str(), "4.0");  // Or have it converted.
   auto tvec = map["bar"].AsTypedVector();
   TEST_EQ(tvec.size(), 3);
   TEST_EQ(tvec[2].AsInt8(), 3);
@@ -1570,6 +1572,22 @@ void FlexBuffersTest() {
   TEST_EQ(vec[2].MutateFloat(2.0f), true);
   TEST_EQ(vec[2].AsFloat(), 2.0f);
   TEST_EQ(vec[2].MutateFloat(3.14159), false);  // Double does not fit in float.
+
+  // Parse from JSON:
+  flatbuffers::Parser parser;
+  slb.Clear();
+  auto jsontest = "{ a: [ 123, 456.0 ], b: \"hello\" }";
+  TEST_EQ(parser.ParseFlexBuffer(jsontest, nullptr, &slb),
+          true);
+  auto jroot = flexbuffers::GetRoot(slb.GetBuffer());
+  auto jmap = jroot.AsMap();
+  auto jvec = jmap["a"].AsVector();
+  TEST_EQ(jvec[0].AsInt64(), 123);
+  TEST_EQ(jvec[1].AsDouble(), 456.0);
+  TEST_EQ_STR(jmap["b"].AsString().c_str(), "hello");
+  // And from FlexBuffer back to JSON:
+  auto jsonback = jroot.ToString();
+  TEST_EQ_STR(jsontest, jsonback.c_str());
 }
 
 int main(int /*argc*/, const char * /*argv*/[]) {