Merge branch 'master' of https://github.com/google/flatbuffers

include/flatbuffers/idl.h

@@ -348,6 +348,7 @@ struct IDLOptions {
   bool escape_proto_identifiers;
   bool generate_object_based_api;
   bool union_value_namespacing;
+  bool allow_non_utf8;
 
   // Possible options for the more general generator below.
   enum Language { kJava, kCSharp, kGo, kMAX };
@@ -370,6 +371,7 @@ struct IDLOptions {
       escape_proto_identifiers(false),
       generate_object_based_api(false),
       union_value_namespacing(true),
+      allow_non_utf8(false),
       lang(IDLOptions::kJava) {}
 };
 

include/flatbuffers/util.h

@@ -276,6 +276,10 @@ inline int FromUTF8(const char **in) {
   }
   if ((**in << len) & 0x80) return -1;  // Bit after leading 1's must be 0.
   if (!len) return *(*in)++;
+  // UTF-8 encoded values with a length are between 2 and 4 bytes.
+  if (len < 2 || len > 4) {
+    return -1;
+  }
   // Grab initial bits of the code.
   int ucc = *(*in)++ & ((1 << (7 - len)) - 1);
   for (int i = 0; i < len - 1; i++) {
@@ -283,6 +287,32 @@ inline int FromUTF8(const char **in) {
     ucc <<= 6;
     ucc |= *(*in)++ & 0x3F;  // Grab 6 more bits of the code.
   }
+  // UTF-8 cannot encode values between 0xD800 and 0xDFFF (reserved for
+  // UTF-16 surrogate pairs).
+  if (ucc >= 0xD800 && ucc <= 0xDFFF) {
+    return -1;
+  }
+  // UTF-8 must represent code points in their shortest possible encoding.
+  switch (len) {
+    case 2:
+      // Two bytes of UTF-8 can represent code points from U+0080 to U+07FF.
+      if (ucc < 0x0080 || ucc > 0x07FF) {
+        return -1;
+      }
+      break;
+    case 3:
+      // Three bytes of UTF-8 can represent code points from U+0800 to U+FFFF.
+      if (ucc < 0x0800 || ucc > 0xFFFF) {
+        return -1;
+      }
+      break;
+    case 4:
+      // Four bytes of UTF-8 can represent code points from U+10000 to U+10FFFF.
+      if (ucc < 0x10000 || ucc > 0x10FFFF) {
+        return -1;
+      }
+      break;
+  }
   return ucc;
 }
 

src/flatc.cpp

@@ -106,6 +106,9 @@ static void Error(const std::string &err, bool usage, bool show_exe_name) {
       "  --version          Print the version number of flatc and exit.\n"
       "  --strict-json      Strict JSON: field names must be / will be quoted,\n"
       "                     no trailing commas in tables/vectors.\n"
+      "  --allow-non-utf8   Pass non-UTF-8 input through parser and emit nonstandard\n"
+      "                     \\x escapes in JSON. (Default is to raise parse error on\n"
+      "                     non-UTF-8 input.)\n"
       "  --defaults-json    Output fields whose value is the default when\n"
       "                     writing JSON\n"
       "  --unknown-json     Allow fields in JSON that are not defined in the\n"
@@ -184,6 +187,8 @@ int main(int argc, const char *argv[]) {
         conform_to_schema = argv[argi];
       } else if(arg == "--strict-json") {
         opts.strict_json = true;
+      } else if(arg == "--allow-non-utf8") {
+        opts.allow_non_utf8 = true;
       } else if(arg == "--no-js-exports") {
         opts.skip_js_exports = true;
       } else if(arg == "--defaults-json") {

src/idl_gen_go.cpp

@@ -126,7 +126,7 @@ static void NewRootTypeFromBuffer(const StructDef &struct_def,
   code += " {\n";
   code += "\tn := flatbuffers.GetUOffsetT(buf[offset:])\n";
   code += "\tx := &" + struct_def.name + "{}\n";
-  code += "\tx.Init(buf, n + offset)\n";
+  code += "\tx.Init(buf, n+offset)\n";
   code += "\treturn x\n";
   code += "}\n\n";
 }
@@ -178,9 +178,10 @@ static void GetScalarFieldOfStruct(const StructDef &struct_def,
   std::string getter = GenGetter(field.value.type);
   GenReceiver(struct_def, code_ptr);
   code += " " + MakeCamel(field.name);
-  code += "() " + TypeName(field) + " { return " + getter;
+  code += "() " + TypeName(field) + " {\n";
+  code +="\treturn " + getter;
   code += "(rcv._tab.Pos + flatbuffers.UOffsetT(";
-  code += NumToString(field.value.offset) + ")) }\n";
+  code += NumToString(field.value.offset) + "))\n}\n";
 }
 
 // Get the value of a table's scalar.
@@ -212,7 +213,7 @@ static void GetStructFieldOfStruct(const StructDef &struct_def,
   code += "\tif obj == nil {\n";
   code += "\t\tobj = new(" + TypeName(field) + ")\n";
   code += "\t}\n";
-  code += "\tobj.Init(rcv._tab.Bytes, rcv._tab.Pos + ";
+  code += "\tobj.Init(rcv._tab.Bytes, rcv._tab.Pos+";
   code += NumToString(field.value.offset) + ")";
   code += "\n\treturn obj\n";
   code += "}\n";
@@ -287,9 +288,9 @@ static void GetMemberOfVectorOfStruct(const StructDef &struct_def,
   if (!(vectortype.struct_def->fixed)) {
     code += "\t\tx = rcv._tab.Indirect(x)\n";
   }
-  code += "\tif obj == nil {\n";
-  code += "\t\tobj = new(" + TypeName(field) + ")\n";
-  code += "\t}\n";
+  code += "\t\tif obj == nil {\n";
+  code += "\t\t\tobj = new(" + TypeName(field) + ")\n";
+  code += "\t\t}\n";
   code += "\t\tobj.Init(rcv._tab.Bytes, x)\n";
   code += "\t\treturn true\n\t}\n";
   code += "\treturn false\n";
@@ -310,7 +311,7 @@ static void GetMemberOfVectorOfNonStruct(const StructDef &struct_def,
   code += OffsetPrefix(field);
   code += "\t\ta := rcv._tab.Vector(o)\n";
   code += "\t\treturn " + GenGetter(field.value.type) + "(";
-  code += "a + flatbuffers.UOffsetT(j * ";
+  code += "a + flatbuffers.UOffsetT(j*";
   code += NumToString(InlineSize(vectortype)) + "))\n";
   code += "\t}\n";
   if (vectortype.base_type == BASE_TYPE_STRING) {
@@ -326,7 +327,10 @@ static void BeginBuilderArgs(const StructDef &struct_def,
                              std::string *code_ptr) {
   std::string &code = *code_ptr;
 
-  code += "\n";
+  if (code.substr(code.length() - 2) != "\n\n") {
+      // a previous mutate has not put an extra new line
+      code += "\n";
+  }
   code += "func Create" + struct_def.name;
   code += "(builder *flatbuffers.Builder";
 }
@@ -368,20 +372,20 @@ static void StructBuilderBody(const StructDef &struct_def,
                               const char *nameprefix,
                               std::string *code_ptr) {
   std::string &code = *code_ptr;
-  code += "    builder.Prep(" + NumToString(struct_def.minalign) + ", ";
+  code += "\tbuilder.Prep(" + NumToString(struct_def.minalign) + ", ";
   code += NumToString(struct_def.bytesize) + ")\n";
   for (auto it = struct_def.fields.vec.rbegin();
        it != struct_def.fields.vec.rend();
        ++it) {
     auto &field = **it;
     if (field.padding)
-      code += "    builder.Pad(" + NumToString(field.padding) + ")\n";
+      code += "\tbuilder.Pad(" + NumToString(field.padding) + ")\n";
     if (IsStruct(field.value.type)) {
       StructBuilderBody(*field.value.type.struct_def,
                         (nameprefix + (field.name + "_")).c_str(),
                         code_ptr);
     } else {
-      code += "    builder.Prepend" + GenMethod(field) + "(";
+      code += "\tbuilder.Prepend" + GenMethod(field) + "(";
       code += nameprefix + MakeCamel(field.name, false) + ")\n";
     }
   }
@@ -389,7 +393,7 @@ static void StructBuilderBody(const StructDef &struct_def,
 
 static void EndBuilderBody(std::string *code_ptr) {
   std::string &code = *code_ptr;
-  code += "    return builder.Offset()\n";
+  code += "\treturn builder.Offset()\n";
   code += "}\n";
 }
 
@@ -398,10 +402,10 @@ static void GetStartOfTable(const StructDef &struct_def,
                             std::string *code_ptr) {
   std::string &code = *code_ptr;
   code += "func " + struct_def.name + "Start";
-  code += "(builder *flatbuffers.Builder) { ";
-  code += "builder.StartObject(";
+  code += "(builder *flatbuffers.Builder) {\n";
+  code += "\tbuilder.StartObject(";
   code += NumToString(struct_def.fields.vec.size());
-  code += ") }\n";
+  code += ")\n}\n";
 }
 
 // Set the value of a table's field.
@@ -418,8 +422,8 @@ static void BuildFieldOfTable(const StructDef &struct_def,
   } else {
     code += GenTypeBasic(field.value.type);
   }
-  code += ") ";
-  code += "{ builder.Prepend";
+  code += ") {\n";
+  code += "\tbuilder.Prepend";
   code += GenMethod(field) + "Slot(";
   code += NumToString(offset) + ", ";
   if (!IsScalar(field.value.type.base_type) && (!struct_def.fixed)) {
@@ -430,7 +434,7 @@ static void BuildFieldOfTable(const StructDef &struct_def,
     code += MakeCamel(field.name, false);
   }
   code += ", " + field.value.constant;
-  code += ") }\n";
+  code += ")\n}\n";
 }
 
 // Set the value of one of the members of a table's vector.
@@ -441,7 +445,7 @@ static void BuildVectorOfTable(const StructDef &struct_def,
   code += "func " + struct_def.name + "Start";
   code += MakeCamel(field.name);
   code += "Vector(builder *flatbuffers.Builder, numElems int) ";
-  code += "flatbuffers.UOffsetT { return builder.StartVector(";
+  code += "flatbuffers.UOffsetT {\n\treturn builder.StartVector(";
   auto vector_type = field.value.type.VectorType();
   auto alignment = InlineAlignment(vector_type);
   auto elem_size = InlineSize(vector_type);
@@ -456,7 +460,7 @@ static void GetEndOffsetOnTable(const StructDef &struct_def,
   std::string &code = *code_ptr;
   code += "func " + struct_def.name + "End";
   code += "(builder *flatbuffers.Builder) flatbuffers.UOffsetT ";
-  code += "{ return builder.EndObject() }\n";
+  code += "{\n\treturn builder.EndObject()\n}\n";
 }
 
 // Generate the receiver for function signatures.
@@ -521,9 +525,9 @@ static void MutateScalarFieldOfStruct(const StructDef &struct_def,
   std::string setter = "rcv._tab.Mutate" + type;
   GenReceiver(struct_def, code_ptr);
   code += " Mutate" + MakeCamel(field.name);
-  code += "(n " + TypeName(field) + ") bool { return " + setter;
-  code += "(rcv._tab.Pos + flatbuffers.UOffsetT(";
-  code += NumToString(field.value.offset) + "), n) }\n\n";
+  code += "(n " + TypeName(field) + ") bool {\n\treturn " + setter;
+  code += "(rcv._tab.Pos+flatbuffers.UOffsetT(";
+  code += NumToString(field.value.offset) + "), n)\n}\n\n";
 }
 
 // Mutate the value of a table's scalar.
@@ -732,7 +736,7 @@ class GoGenerator : public BaseGenerator {
     if (needs_imports) {
       code += "import (\n";
       code += "\tflatbuffers \"github.com/google/flatbuffers/go\"\n";
-      code += ")\n";
+      code += ")\n\n";
     }
   }
 

src/idl_gen_text.cpp

@@ -93,7 +93,7 @@ template<typename T> void PrintVector(const Vector<T> &v, Type type,
   text += "]";
 }
 
-static void EscapeString(const String &s, std::string *_text) {
+static void EscapeString(const String &s, std::string *_text, const IDLOptions& opts) {
   std::string &text = *_text;
   text += "\"";
   for (uoffset_t i = 0; i < s.size(); i++) {
@@ -113,17 +113,32 @@ static void EscapeString(const String &s, std::string *_text) {
           // Not printable ASCII data. Let's see if it's valid UTF-8 first:
           const char *utf8 = s.c_str() + i;
           int ucc = FromUTF8(&utf8);
-          if (ucc >= 0x80 && ucc <= 0xFFFF) {
-            // Parses as Unicode within JSON's \uXXXX range, so use that.
-            text += "\\u";
-            text += IntToStringHex(ucc, 4);
+          if (ucc < 0) {
+            if (opts.allow_non_utf8) {
+              text += "\\x";
+              text += IntToStringHex(static_cast<uint8_t>(c), 2);
+            } else {
+              // We previously checked for non-UTF-8 and returned a parse error,
+              // so we shouldn't reach here.
+              assert(0);
+            }
+          } else {
+            if (ucc <= 0xFFFF) {
+              // Parses as Unicode within JSON's \uXXXX range, so use that.
+              text += "\\u";
+              text += IntToStringHex(ucc, 4);
+            } else if (ucc <= 0x10FFFF) {
+              // Encode Unicode SMP values to a surrogate pair using two \u escapes.
+              uint32_t base = ucc - 0x10000;
+              uint16_t highSurrogate = (base >> 10) + 0xD800;
+              uint16_t lowSurrogate = (base & 0x03FF) + 0xDC00;
+              text += "\\u";
+              text += IntToStringHex(highSurrogate, 4);
+              text += "\\u";
+              text += IntToStringHex(lowSurrogate, 4);
+            }
             // Skip past characters recognized.
             i = static_cast<uoffset_t>(utf8 - s.c_str() - 1);
-          } else {
-            // It's either unprintable ASCII, arbitrary binary, or Unicode data
-            // that doesn't fit \uXXXX, so use \xXX escape code instead.
-            text += "\\x";
-            text += IntToStringHex(static_cast<uint8_t>(c), 2);
           }
         }
         break;
@@ -157,7 +172,7 @@ template<> void Print<const void *>(const void *val,
                 _text);
       break;
     case BASE_TYPE_STRING: {
-      EscapeString(*reinterpret_cast<const String *>(val), _text);
+      EscapeString(*reinterpret_cast<const String *>(val), _text, opts);
       break;
     }
     case BASE_TYPE_VECTOR:

src/idl_parser.cpp

@@ -61,6 +61,17 @@ static_assert(BASE_TYPE_UNION ==
 #define NEXT() ECHECK(Next())
 #define EXPECT(tok) ECHECK(Expect(tok))
 
+static bool ValidateUTF8(const std::string &str) {
+  const char *s = &str[0];
+  const char * const sEnd = s + str.length();
+  while (s < sEnd) {
+    if (FromUTF8(&s) < 0) {
+      return false;
+    }
+  }
+  return true;
+}
+
 CheckedError Parser::Error(const std::string &msg) {
   error_ = file_being_parsed_.length() ? AbsolutePath(file_being_parsed_) : "";
   #ifdef _WIN32
@@ -320,6 +331,9 @@ CheckedError Parser::Next() {
             "illegal Unicode sequence (unpaired high surrogate)");
         }
         cursor_++;
+        if (!opts.allow_non_utf8 && !ValidateUTF8(attribute_)) {
+          return Error("illegal UTF-8 sequence");
+        }
         token_ = kTokenStringConstant;
         return NoError();
       }

tests/GoTest.sh

@@ -29,9 +29,9 @@ mkdir -p ${go_src}/MyGame/Example
 mkdir -p ${go_src}/github.com/google/flatbuffers/go
 mkdir -p ${go_src}/flatbuffers_test
 
-cp -u MyGame/Example/*.go ./go_gen/src/MyGame/Example/
-cp -u ../go/* ./go_gen/src/github.com/google/flatbuffers/go
-cp -u ./go_test.go ./go_gen/src/flatbuffers_test/
+cp -a MyGame/Example/*.go ./go_gen/src/MyGame/Example/
+cp -a ../go/* ./go_gen/src/github.com/google/flatbuffers/go
+cp -a ./go_test.go ./go_gen/src/flatbuffers_test/
 
 # Run tests with necessary flags.
 # Developers may wish to see more detail by appending the verbosity flag
@@ -50,6 +50,18 @@ GOPATH=${go_path} go test flatbuffers_test \
                      --fuzz_fields=4 \
                      --fuzz_objects=10000
 
+GO_TEST_RESULT=$?
 rm -rf ${go_path}/{pkg,src}
+if [[ $GO_TEST_RESULT  == 0 ]]; then
+    echo "OK: Go tests passed."
+else
+    echo "KO: Go tests failed."
+    exit 1
+fi
 
-echo "OK: Go tests passed."
+NOT_FMT_FILES=$(gofmt -l MyGame)
+if [[ ${NOT_FMT_FILES} != "" ]]; then
+    echo "These files are not well gofmt'ed:\n\n${NOT_FMT_FILES}"
+    # enable this when enums are properly formated
+    # exit 1
+fi

tests/MyGame/Example/Stat.go

@@ -5,6 +5,7 @@ package Example
 import (
 	flatbuffers "github.com/google/flatbuffers/go"
 )
+
 type Stat struct {
 	_tab flatbuffers.Table
 }
@@ -12,7 +13,7 @@ type Stat struct {
 func GetRootAsStat(buf []byte, offset flatbuffers.UOffsetT) *Stat {
 	n := flatbuffers.GetUOffsetT(buf[offset:])
 	x := &Stat{}
-	x.Init(buf, n + offset)
+	x.Init(buf, n+offset)
 	return x
 }
 
@@ -53,8 +54,18 @@ func (rcv *Stat) MutateCount(n uint16) bool {
 	return rcv._tab.MutateUint16Slot(8, n)
 }
 
-func StatStart(builder *flatbuffers.Builder) { builder.StartObject(3) }
-func StatAddId(builder *flatbuffers.Builder, id flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(id), 0) }
-func StatAddVal(builder *flatbuffers.Builder, val int64) { builder.PrependInt64Slot(1, val, 0) }
-func StatAddCount(builder *flatbuffers.Builder, count uint16) { builder.PrependUint16Slot(2, count, 0) }
-func StatEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() }
+func StatStart(builder *flatbuffers.Builder) {
+	builder.StartObject(3)
+}
+func StatAddId(builder *flatbuffers.Builder, id flatbuffers.UOffsetT) {
+	builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(id), 0)
+}
+func StatAddVal(builder *flatbuffers.Builder, val int64) {
+	builder.PrependInt64Slot(1, val, 0)
+}
+func StatAddCount(builder *flatbuffers.Builder, count uint16) {
+	builder.PrependUint16Slot(2, count, 0)
+}
+func StatEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+	return builder.EndObject()
+}

tests/MyGame/Example/Test.go

@@ -5,6 +5,7 @@ package Example
 import (
 	flatbuffers "github.com/google/flatbuffers/go"
 )
+
 type Test struct {
 	_tab flatbuffers.Struct
 }
@@ -14,17 +15,24 @@ func (rcv *Test) Init(buf []byte, i flatbuffers.UOffsetT) {
 	rcv._tab.Pos = i
 }
 
-func (rcv *Test) A() int16 { return rcv._tab.GetInt16(rcv._tab.Pos + flatbuffers.UOffsetT(0)) }
-func (rcv *Test) MutateA(n int16) bool { return rcv._tab.MutateInt16(rcv._tab.Pos + flatbuffers.UOffsetT(0), n) }
-
-func (rcv *Test) B() int8 { return rcv._tab.GetInt8(rcv._tab.Pos + flatbuffers.UOffsetT(2)) }
-func (rcv *Test) MutateB(n int8) bool { return rcv._tab.MutateInt8(rcv._tab.Pos + flatbuffers.UOffsetT(2), n) }
+func (rcv *Test) A() int16 {
+	return rcv._tab.GetInt16(rcv._tab.Pos + flatbuffers.UOffsetT(0))
+}
+func (rcv *Test) MutateA(n int16) bool {
+	return rcv._tab.MutateInt16(rcv._tab.Pos+flatbuffers.UOffsetT(0), n)
+}
 
+func (rcv *Test) B() int8 {
+	return rcv._tab.GetInt8(rcv._tab.Pos + flatbuffers.UOffsetT(2))
+}
+func (rcv *Test) MutateB(n int8) bool {
+	return rcv._tab.MutateInt8(rcv._tab.Pos+flatbuffers.UOffsetT(2), n)
+}
 
 func CreateTest(builder *flatbuffers.Builder, a int16, b int8) flatbuffers.UOffsetT {
-    builder.Prep(2, 4)
-    builder.Pad(1)
-    builder.PrependInt8(b)
-    builder.PrependInt16(a)
-    return builder.Offset()
+	builder.Prep(2, 4)
+	builder.Pad(1)
+	builder.PrependInt8(b)
+	builder.PrependInt16(a)
+	return builder.Offset()
 }

tests/MyGame/Example/TestSimpleTableWithEnum.go

@@ -5,6 +5,7 @@ package Example
 import (
 	flatbuffers "github.com/google/flatbuffers/go"
 )
+
 type TestSimpleTableWithEnum struct {
 	_tab flatbuffers.Table
 }
@@ -12,7 +13,7 @@ type TestSimpleTableWithEnum struct {
 func GetRootAsTestSimpleTableWithEnum(buf []byte, offset flatbuffers.UOffsetT) *TestSimpleTableWithEnum {
 	n := flatbuffers.GetUOffsetT(buf[offset:])
 	x := &TestSimpleTableWithEnum{}
-	x.Init(buf, n + offset)
+	x.Init(buf, n+offset)
 	return x
 }
 
@@ -33,6 +34,12 @@ func (rcv *TestSimpleTableWithEnum) MutateColor(n int8) bool {
 	return rcv._tab.MutateInt8Slot(4, n)
 }
 
-func TestSimpleTableWithEnumStart(builder *flatbuffers.Builder) { builder.StartObject(1) }
-func TestSimpleTableWithEnumAddColor(builder *flatbuffers.Builder, color int8) { builder.PrependInt8Slot(0, color, 2) }
-func TestSimpleTableWithEnumEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() }
+func TestSimpleTableWithEnumStart(builder *flatbuffers.Builder) {
+	builder.StartObject(1)
+}
+func TestSimpleTableWithEnumAddColor(builder *flatbuffers.Builder, color int8) {
+	builder.PrependInt8Slot(0, color, 2)
+}
+func TestSimpleTableWithEnumEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+	return builder.EndObject()
+}

tests/MyGame/Example/Vec3.go

@@ -5,6 +5,7 @@ package Example
 import (
 	flatbuffers "github.com/google/flatbuffers/go"
 )
+
 type Vec3 struct {
 	_tab flatbuffers.Struct
 }
@@ -14,42 +15,62 @@ func (rcv *Vec3) Init(buf []byte, i flatbuffers.UOffsetT) {
 	rcv._tab.Pos = i
 }
 
-func (rcv *Vec3) X() float32 { return rcv._tab.GetFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(0)) }
-func (rcv *Vec3) MutateX(n float32) bool { return rcv._tab.MutateFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(0), n) }
+func (rcv *Vec3) X() float32 {
+	return rcv._tab.GetFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(0))
+}
+func (rcv *Vec3) MutateX(n float32) bool {
+	return rcv._tab.MutateFloat32(rcv._tab.Pos+flatbuffers.UOffsetT(0), n)
+}
 
-func (rcv *Vec3) Y() float32 { return rcv._tab.GetFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(4)) }
-func (rcv *Vec3) MutateY(n float32) bool { return rcv._tab.MutateFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(4), n) }
+func (rcv *Vec3) Y() float32 {
+	return rcv._tab.GetFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(4))
+}
+func (rcv *Vec3) MutateY(n float32) bool {
+	return rcv._tab.MutateFloat32(rcv._tab.Pos+flatbuffers.UOffsetT(4), n)
+}
 
-func (rcv *Vec3) Z() float32 { return rcv._tab.GetFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(8)) }
-func (rcv *Vec3) MutateZ(n float32) bool { return rcv._tab.MutateFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(8), n) }
+func (rcv *Vec3) Z() float32 {
+	return rcv._tab.GetFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(8))
+}
+func (rcv *Vec3) MutateZ(n float32) bool {
+	return rcv._tab.MutateFloat32(rcv._tab.Pos+flatbuffers.UOffsetT(8), n)
+}
 
-func (rcv *Vec3) Test1() float64 { return rcv._tab.GetFloat64(rcv._tab.Pos + flatbuffers.UOffsetT(16)) }
-func (rcv *Vec3) MutateTest1(n float64) bool { return rcv._tab.MutateFloat64(rcv._tab.Pos + flatbuffers.UOffsetT(16), n) }
+func (rcv *Vec3) Test1() float64 {
+	return rcv._tab.GetFloat64(rcv._tab.Pos + flatbuffers.UOffsetT(16))
+}
+func (rcv *Vec3) MutateTest1(n float64) bool {
+	return rcv._tab.MutateFloat64(rcv._tab.Pos+flatbuffers.UOffsetT(16), n)
+}
 
-func (rcv *Vec3) Test2() int8 { return rcv._tab.GetInt8(rcv._tab.Pos + flatbuffers.UOffsetT(24)) }
-func (rcv *Vec3) MutateTest2(n int8) bool { return rcv._tab.MutateInt8(rcv._tab.Pos + flatbuffers.UOffsetT(24), n) }
+func (rcv *Vec3) Test2() int8 {
+	return rcv._tab.GetInt8(rcv._tab.Pos + flatbuffers.UOffsetT(24))
+}
+func (rcv *Vec3) MutateTest2(n int8) bool {
+	return rcv._tab.MutateInt8(rcv._tab.Pos+flatbuffers.UOffsetT(24), n)
+}
 
 func (rcv *Vec3) Test3(obj *Test) *Test {
 	if obj == nil {
 		obj = new(Test)
 	}
-	obj.Init(rcv._tab.Bytes, rcv._tab.Pos + 26)
+	obj.Init(rcv._tab.Bytes, rcv._tab.Pos+26)
 	return obj
 }
 
 func CreateVec3(builder *flatbuffers.Builder, x float32, y float32, z float32, test1 float64, test2 int8, test3_a int16, test3_b int8) flatbuffers.UOffsetT {
-    builder.Prep(16, 32)
-    builder.Pad(2)
-    builder.Prep(2, 4)
-    builder.Pad(1)
-    builder.PrependInt8(test3_b)
-    builder.PrependInt16(test3_a)
-    builder.Pad(1)
-    builder.PrependInt8(test2)
-    builder.PrependFloat64(test1)
-    builder.Pad(4)
-    builder.PrependFloat32(z)
-    builder.PrependFloat32(y)
-    builder.PrependFloat32(x)
-    return builder.Offset()
+	builder.Prep(16, 32)
+	builder.Pad(2)
+	builder.Prep(2, 4)
+	builder.Pad(1)
+	builder.PrependInt8(test3_b)
+	builder.PrependInt16(test3_a)
+	builder.Pad(1)
+	builder.PrependInt8(test2)
+	builder.PrependFloat64(test1)
+	builder.Pad(4)
+	builder.PrependFloat32(z)
+	builder.PrependFloat32(y)
+	builder.PrependFloat32(x)
+	return builder.Offset()
 }

tests/MyGame/Example2/Monster.go

@@ -5,6 +5,7 @@ package Example2
 import (
 	flatbuffers "github.com/google/flatbuffers/go"
 )
+
 type Monster struct {
 	_tab flatbuffers.Table
 }
@@ -12,7 +13,7 @@ type Monster struct {
 func GetRootAsMonster(buf []byte, offset flatbuffers.UOffsetT) *Monster {
 	n := flatbuffers.GetUOffsetT(buf[offset:])
 	x := &Monster{}
-	x.Init(buf, n + offset)
+	x.Init(buf, n+offset)
 	return x
 }
 
@@ -21,5 +22,9 @@ func (rcv *Monster) Init(buf []byte, i flatbuffers.UOffsetT) {
 	rcv._tab.Pos = i
 }
 
-func MonsterStart(builder *flatbuffers.Builder) { builder.StartObject(0) }
-func MonsterEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() }
+func MonsterStart(builder *flatbuffers.Builder) {
+	builder.StartObject(0)
+}
+func MonsterEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
+	return builder.EndObject()
+}

tests/test.cpp

@@ -978,15 +978,36 @@ void IntegerOutOfRangeTest() {
 
 void UnicodeTest() {
   flatbuffers::Parser parser;
+  // Without setting allow_non_utf8 = true, we treat \x sequences as byte sequences
+  // which are then validated as UTF-8.
   TEST_EQ(parser.Parse("table T { F:string; }"
                        "root_type T;"
                        "{ F:\"\\u20AC\\u00A2\\u30E6\\u30FC\\u30B6\\u30FC"
-                       "\\u5225\\u30B5\\u30A4\\u30C8\\x01\\x80\" }"), true);
+                       "\\u5225\\u30B5\\u30A4\\u30C8\\xE2\\x82\\xAC\\u0080\\uD83D\\uDE0E\" }"),
+          true);
   std::string jsongen;
   parser.opts.indent_step = -1;
   GenerateText(parser, parser.builder_.GetBufferPointer(), &jsongen);
-  TEST_EQ(jsongen == "{F: \"\\u20AC\\u00A2\\u30E6\\u30FC\\u30B6\\u30FC"
-                     "\\u5225\\u30B5\\u30A4\\u30C8\\x01\\x80\"}", true);
+  TEST_EQ(jsongen,
+          std::string(
+            "{F: \"\\u20AC\\u00A2\\u30E6\\u30FC\\u30B6\\u30FC"
+            "\\u5225\\u30B5\\u30A4\\u30C8\\u20AC\\u0080\\uD83D\\uDE0E\"}"));
+}
+
+void UnicodeTestAllowNonUTF8() {
+  flatbuffers::Parser parser;
+  parser.opts.allow_non_utf8 = true;
+  TEST_EQ(parser.Parse("table T { F:string; }"
+                       "root_type T;"
+                       "{ F:\"\\u20AC\\u00A2\\u30E6\\u30FC\\u30B6\\u30FC"
+                       "\\u5225\\u30B5\\u30A4\\u30C8\\x01\\x80\\u0080\\uD83D\\uDE0E\" }"), true);
+  std::string jsongen;
+  parser.opts.indent_step = -1;
+  GenerateText(parser, parser.builder_.GetBufferPointer(), &jsongen);
+  TEST_EQ(jsongen,
+          std::string(
+            "{F: \"\\u20AC\\u00A2\\u30E6\\u30FC\\u30B6\\u30FC"
+            "\\u5225\\u30B5\\u30A4\\u30C8\\u0001\\x80\\u0080\\uD83D\\uDE0E\"}"));
 }
 
 void UnicodeSurrogatesTest() {
@@ -1027,6 +1048,96 @@ void UnicodeInvalidSurrogatesTest() {
     "{ F:\"\\uDC00\"}", "unpaired low surrogate");
 }
 
+void InvalidUTF8Test() {
+  // "1 byte" pattern, under min length of 2 bytes
+  TestError(
+    "table T { F:string; }"
+    "root_type T;"
+    "{ F:\"\x80\"}", "illegal UTF-8 sequence");
+  // 2 byte pattern, string too short
+  TestError(
+    "table T { F:string; }"
+    "root_type T;"
+    "{ F:\"\xDF\"}", "illegal UTF-8 sequence");
+  // 3 byte pattern, string too short
+  TestError(
+    "table T { F:string; }"
+    "root_type T;"
+    "{ F:\"\xEF\xBF\"}", "illegal UTF-8 sequence");
+  // 4 byte pattern, string too short
+  TestError(
+    "table T { F:string; }"
+    "root_type T;"
+    "{ F:\"\xF7\xBF\xBF\"}", "illegal UTF-8 sequence");
+  // "5 byte" pattern, string too short
+  TestError(
+    "table T { F:string; }"
+    "root_type T;"
+    "{ F:\"\xFB\xBF\xBF\xBF\"}", "illegal UTF-8 sequence");
+  // "6 byte" pattern, string too short
+  TestError(
+    "table T { F:string; }"
+    "root_type T;"
+    "{ F:\"\xFD\xBF\xBF\xBF\xBF\"}", "illegal UTF-8 sequence");
+  // "7 byte" pattern, string too short
+  TestError(
+    "table T { F:string; }"
+    "root_type T;"
+    "{ F:\"\xFE\xBF\xBF\xBF\xBF\xBF\"}", "illegal UTF-8 sequence");
+  // "5 byte" pattern, over max length of 4 bytes
+  TestError(
+    "table T { F:string; }"
+    "root_type T;"
+    "{ F:\"\xFB\xBF\xBF\xBF\xBF\"}", "illegal UTF-8 sequence");
+  // "6 byte" pattern, over max length of 4 bytes
+  TestError(
+    "table T { F:string; }"
+    "root_type T;"
+    "{ F:\"\xFD\xBF\xBF\xBF\xBF\xBF\"}", "illegal UTF-8 sequence");
+  // "7 byte" pattern, over max length of 4 bytes
+  TestError(
+    "table T { F:string; }"
+    "root_type T;"
+    "{ F:\"\xFE\xBF\xBF\xBF\xBF\xBF\xBF\"}", "illegal UTF-8 sequence");
+
+  // Three invalid encodings for U+000A (\n, aka NEWLINE)
+  TestError(
+    "table T { F:string; }"
+    "root_type T;"
+    "{ F:\"\xC0\x8A\"}", "illegal UTF-8 sequence");
+  TestError(
+    "table T { F:string; }"
+    "root_type T;"
+    "{ F:\"\xE0\x80\x8A\"}", "illegal UTF-8 sequence");
+  TestError(
+    "table T { F:string; }"
+    "root_type T;"
+    "{ F:\"\xF0\x80\x80\x8A\"}", "illegal UTF-8 sequence");
+
+  // Two invalid encodings for U+00A9 (COPYRIGHT SYMBOL)
+  TestError(
+    "table T { F:string; }"
+    "root_type T;"
+    "{ F:\"\xE0\x81\xA9\"}", "illegal UTF-8 sequence");
+  TestError(
+    "table T { F:string; }"
+    "root_type T;"
+    "{ F:\"\xF0\x80\x81\xA9\"}", "illegal UTF-8 sequence");
+
+  // Invalid encoding for U+20AC (EURO SYMBOL)
+  TestError(
+    "table T { F:string; }"
+    "root_type T;"
+    "{ F:\"\xF0\x82\x82\xAC\"}", "illegal UTF-8 sequence");
+
+  // UTF-16 surrogate values between U+D800 and U+DFFF cannot be encoded in UTF-8
+  TestError(
+    "table T { F:string; }"
+    "root_type T;"
+    // U+10400 "encoded" as U+D801 U+DC00
+    "{ F:\"\xED\xA0\x81\xED\xB0\x80\"}", "illegal UTF-8 sequence");
+}
+
 void UnknownFieldsTest() {
   flatbuffers::IDLOptions opts;
   opts.skip_unexpected_fields_in_json = true;
@@ -1105,8 +1216,10 @@ int main(int /*argc*/, const char * /*argv*/[]) {
   EnumStringsTest();
   IntegerOutOfRangeTest();
   UnicodeTest();
+  UnicodeTestAllowNonUTF8();
   UnicodeSurrogatesTest();
   UnicodeInvalidSurrogatesTest();
+  InvalidUTF8Test();
   UnknownFieldsTest();
   ParseUnionTest();
   ConformTest();