Merge branch 'master' of https://github.com/google/flatbuffers into fuzzer2

include/flatbuffers/code_generators.h

@@ -21,17 +21,70 @@ namespace flatbuffers {
 
 class BaseGenerator {
  public:
-  BaseGenerator(const Parser &parser, const std::string &path,
-                const std::string &file_name)
-      : parser_(parser), path_(path), file_name_(file_name){};
   virtual bool generate() = 0;
 
+  static const std::string NamespaceDir(const Parser &parser,
+                                        const std::string &path) {
+    EnsureDirExists(path.c_str());
+    if (parser.opts.one_file) return path;
+    std::string namespace_dir = path;  // Either empty or ends in separator.
+    auto &namespaces = parser.namespaces_.back()->components;
+    for (auto it = namespaces.begin(); it != namespaces.end(); ++it) {
+      namespace_dir += *it + kPathSeparator;
+      EnsureDirExists(namespace_dir.c_str());
+    }
+    return namespace_dir;
+  }
+
  protected:
+  BaseGenerator(const Parser &parser, const std::string &path,
+                const std::string &file_name)
+      : parser_(parser),
+        path_(path),
+        file_name_(file_name),
+        namespace_dir_(BaseGenerator::NamespaceDir(parser, path)){};
   virtual ~BaseGenerator(){};
 
+  // No copy/assign.
+  BaseGenerator &operator=(const BaseGenerator &);
+  BaseGenerator(const BaseGenerator &);
+
+  const char *FlatBuffersGeneratedWarning() {
+    return "automatically generated by the FlatBuffers compiler,"
+           " do not modify\n\n";
+  }
+
+  bool IsEverythingGenerated() {
+    for (auto it = parser_.enums_.vec.begin(); it != parser_.enums_.vec.end();
+         ++it) {
+      if (!(*it)->generated) return false;
+    }
+    for (auto it = parser_.structs_.vec.begin();
+         it != parser_.structs_.vec.end(); ++it) {
+      if (!(*it)->generated) return false;
+    }
+    return true;
+  }
+
+  std::string FullNamespace(const char *separator) {
+    std::string namespace_name;
+    auto &namespaces = parser_.namespaces_.back()->components;
+    for (auto it = namespaces.begin(); it != namespaces.end(); ++it) {
+      if (namespace_name.length()) namespace_name += separator;
+      namespace_name += *it;
+    }
+    return namespace_name;
+  }
+
+  const std::string LastNamespacePart() {
+    auto &namespaces = parser_.namespaces_.back()->components;
+    if (namespaces.size()) return *(namespaces.end() - 1); else return std::string("");
+  }
+
   const Parser &parser_;
   const std::string &path_;
   const std::string &file_name_;
+  const std::string namespace_dir_;
 };
 
 }  // namespace flatbuffers

include/flatbuffers/flatbuffers.h

@@ -337,7 +337,7 @@ public:
 
   // Change elements if you have a non-const pointer to this object.
   // Scalars only. See reflection.h, and the documentation.
-  void Mutate(uoffset_t i, T val) {
+  void Mutate(uoffset_t i, const T& val) {
     assert(i < size());
     WriteScalar(data() + i, val);
   }
@@ -1161,6 +1161,17 @@ template<typename T> const T *GetRoot(const void *buf) {
   return GetMutableRoot<T>(const_cast<void *>(buf));
 }
 
+/// Helpers to get a typed pointer to objects that are currently beeing built.
+/// @warning Creating new objects will lead to reallocations and invalidates the pointer!
+template<typename T> T *GetMutableTemporaryPointer(FlatBufferBuilder &fbb, Offset<T> offset) {
+  return reinterpret_cast<T *>(fbb.GetCurrentBufferPointer() +
+    fbb.GetSize() - offset.o);
+}
+
+template<typename T> const T *GetTemporaryPointer(FlatBufferBuilder &fbb, Offset<T> offset) {
+  return GetMutableTemporaryPointer<T>(fbb, offset);
+}
+
 // Helper to see if the identifier in a buffer has the expected value.
 inline bool BufferHasIdentifier(const void *buf, const char *identifier) {
   return strncmp(reinterpret_cast<const char *>(buf) + sizeof(uoffset_t),
@@ -1495,6 +1506,35 @@ volatile __attribute__((weak)) const char *flatbuffer_version_string =
 
 #endif  // !defined(_WIN32) && !defined(__CYGWIN__)
 
+#define DEFINE_BITMASK_OPERATORS(E, T)\
+    inline E operator | (E lhs, E rhs){\
+        return E(T(lhs) | T(rhs));\
+    }\
+    inline E operator & (E lhs, E rhs){\
+        return E(T(lhs) & T(rhs));\
+    }\
+    inline E operator ^ (E lhs, E rhs){\
+        return E(T(lhs) ^ T(rhs));\
+    }\
+    inline E operator ~ (E lhs){\
+        return E(~T(lhs));\
+    }\
+    inline E operator |= (E &lhs, E rhs){\
+        lhs = lhs | rhs;\
+        return lhs;\
+    }\
+    inline E operator &= (E &lhs, E rhs){\
+        lhs = lhs & rhs;\
+        return lhs;\
+    }\
+    inline E operator ^= (E &lhs, E rhs){\
+        lhs = lhs ^ rhs;\
+        return lhs;\
+    }\
+    inline bool operator !(E rhs) \
+    {\
+        return !bool(T(rhs)); \
+    }
 /// @endcond
 }  // namespace flatbuffers
 

src/idl_gen_cpp.cpp

@@ -166,6 +166,7 @@ static void GenEnum(const Parser &parser, EnumDef &enum_def,
   if (parser.opts.scoped_enums)
     code += " : " + GenTypeBasic(enum_def.underlying_type, false);
   code += " {\n";
+  int64_t anyv = 0;
   EnumVal *minv = nullptr, *maxv = nullptr;
   for (auto it = enum_def.vals.vec.begin();
        it != enum_def.vals.vec.end();
@@ -176,13 +177,26 @@ static void GenEnum(const Parser &parser, EnumDef &enum_def,
     code += NumToString(ev.value) + ",\n";
     minv = !minv || minv->value > ev.value ? &ev : minv;
     maxv = !maxv || maxv->value < ev.value ? &ev : maxv;
+    anyv |= ev.value;
   }
-  assert(minv && maxv);
-  code += "  " + GenEnumVal(enum_def, "MIN", parser.opts) + " = ";
-  code += GenEnumVal(enum_def, minv->name, parser.opts) + ",\n";
-  code += "  " + GenEnumVal(enum_def, "MAX", parser.opts) + " = ";
-  code += GenEnumVal(enum_def, maxv->name, parser.opts) + "\n";
-  code += "};\n\n";
+  if (parser.opts.scoped_enums || parser.opts.prefixed_enums) {
+    assert(minv && maxv);
+    if (enum_def.attributes.Lookup("bit_flags")) {
+      if (minv->value != 0) // If the user didn't defined NONE value
+        code += "  " + GenEnumVal(enum_def, "NONE", parser.opts) + " = 0,\n";
+      if (maxv->value != anyv) // If the user didn't defined ANY value
+        code += "  " + GenEnumVal(enum_def, "ANY", parser.opts) + " = " + NumToString(anyv) + "\n";
+    } else { // MIN & MAX are useless for bit_flags
+      code += "  " + GenEnumVal(enum_def, "MIN", parser.opts) + " = ";
+      code += GenEnumVal(enum_def, minv->name, parser.opts) + ",\n";
+      code += "  " + GenEnumVal(enum_def, "MAX", parser.opts) + " = ";
+      code += GenEnumVal(enum_def, maxv->name, parser.opts) + "\n";
+    }
+  }
+  code += "};\n";
+  if (parser.opts.scoped_enums && enum_def.attributes.Lookup("bit_flags"))
+    code += "DEFINE_BITMASK_OPERATORS(" + enum_def.name + ", " +  GenTypeBasic(enum_def.underlying_type, false) + ")\n";
+  code += "\n";
 
   // Generate a generate string table for enum values.
   // Problem is, if values are very sparse that could generate really big
@@ -730,26 +744,10 @@ class CppGenerator : public BaseGenerator {
   // structs,
   // and tables) and output them to a single file.
   bool generate() {
-    // Check if we have any code to generate at all, to avoid an empty header.
-    for (auto it = parser_.enums_.vec.begin(); it != parser_.enums_.vec.end();
-         ++it) {
-      if (!(*it)->generated) goto generate_code;
-    }
-    for (auto it = parser_.structs_.vec.begin(); it != parser_.structs_.vec.end();
-         ++it) {
-      if (!(*it)->generated) goto generate_code;
-    }
-    // No code to generate, exit:
-    return true;
-
-  generate_code:
-
-    using namespace cpp;
+    if (IsEverythingGenerated()) return true;
 
     std::string code;
-    code =
-        "// automatically generated by the FlatBuffers compiler,"
-        " do not modify\n\n";
+    code = code + "// " + FlatBuffersGeneratedWarning();
 
     // Generate include guard.
     std::string include_guard_ident = file_name_;

src/idl_gen_general.cpp

@@ -1115,40 +1115,6 @@ static void GenStruct(const LanguageParameters &lang, const Parser &parser,
   code += "};\n\n";
 }
 
-// Save out the generated code for a single class while adding
-// declaration boilerplate.
-static bool SaveClass(const LanguageParameters &lang, const Parser &parser,
-                      const std::string &defname, const std::string &classcode,
-                      const std::string &path, bool needs_includes, bool onefile) {
-  if (!classcode.length()) return true;
-
-  std::string namespace_general;
-  std::string namespace_dir = path;  // Either empty or ends in separator.
-  auto &namespaces = parser.namespaces_.back()->components;
-  for (auto it = namespaces.begin(); it != namespaces.end(); ++it) {
-    if (namespace_general.length()) {
-      namespace_general += ".";
-    }
-    namespace_general += *it;
-    if (!onefile) {
-      namespace_dir += *it + kPathSeparator;
-    }
-
-  }
-  EnsureDirExists(namespace_dir);
-
-  std::string code = "// automatically generated, do not modify\n\n";
-  if (!namespace_general.empty()) {
-    code += lang.namespace_ident + namespace_general + lang.namespace_begin;
-    code += "\n\n";
-  }
-  if (needs_includes) code += lang.includes;
-  code += classcode;
-  if (!namespace_general.empty()) code += lang.namespace_end;
-  auto filename = namespace_dir + defname + lang.file_extension;
-  return SaveFile(filename.c_str(), code, false);
-}
-
 namespace general {
 class GeneralGenerator : public BaseGenerator {
  public:
@@ -1167,9 +1133,7 @@ class GeneralGenerator : public BaseGenerator {
       if (parser_.opts.one_file) {
         one_file_code += enumcode;
       } else {
-        if (!SaveClass(lang, parser_, (**it).name, enumcode, path_, false,
-                       false))
-          return false;
+        if (!SaveType(lang, (**it).name, enumcode, false)) return false;
       }
     }
 
@@ -1180,18 +1144,35 @@ class GeneralGenerator : public BaseGenerator {
       if (parser_.opts.one_file) {
         one_file_code += declcode;
       } else {
-        if (!SaveClass(lang, parser_, (**it).name, declcode, path_, true,
-                       false))
-          return false;
+        if (!SaveType(lang, (**it).name, declcode, true)) return false;
       }
     }
 
     if (parser_.opts.one_file) {
-      return SaveClass(lang, parser_, file_name_, one_file_code, path_, true,
-                       true);
+      return SaveType(lang, file_name_, one_file_code, true);
     }
     return true;
   }
+
+  // Save out the generated code for a single class while adding
+  // declaration boilerplate.
+  bool SaveType(const LanguageParameters &lang, const std::string &defname,
+                const std::string &classcode, bool needs_includes) {
+    if (!classcode.length()) return true;
+
+    std::string code;
+    code = code + "// " + FlatBuffersGeneratedWarning();
+    std::string namespace_name = FullNamespace(".");
+    if (!namespace_name.empty()) {
+      code += lang.namespace_ident + namespace_name + lang.namespace_begin;
+      code += "\n\n";
+    }
+    if (needs_includes) code += lang.includes;
+    code += classcode;
+    if (!namespace_name.empty()) code += lang.namespace_end;
+    auto filename = namespace_dir_ + defname + lang.file_extension;
+    return SaveFile(filename.c_str(), code, false);
+  }
 };
 }  // namespace general
 
@@ -1201,50 +1182,30 @@ bool GenerateGeneral(const Parser &parser, const std::string &path,
   return generator.generate();
 }
 
-static std::string ClassFileName(const LanguageParameters &lang,
-                                 const Parser &parser, const Definition &def,
-                                 const std::string &path) {
-  std::string namespace_general;
-  std::string namespace_dir = path;
-  auto &namespaces = parser.namespaces_.back()->components;
-  for (auto it = namespaces.begin(); it != namespaces.end(); ++it) {
-    if (namespace_general.length()) {
-      namespace_general += ".";
-      namespace_dir += kPathSeparator;
-    }
-    namespace_general += *it;
-    namespace_dir += *it;
-  }
-
-  return namespace_dir + kPathSeparator + def.name + lang.file_extension;
-}
-
-std::string GeneralMakeRule(const Parser &parser,
-                            const std::string &path,
+std::string GeneralMakeRule(const Parser &parser, const std::string &path,
                             const std::string &file_name) {
   assert(parser.opts.lang <= IDLOptions::kMAX);
   auto lang = language_parameters[parser.opts.lang];
 
   std::string make_rule;
+  std::string directory =
+      BaseGenerator::NamespaceDir(parser, path) + kPathSeparator;
 
-  for (auto it = parser.enums_.vec.begin();
-       it != parser.enums_.vec.end(); ++it) {
-    if (make_rule != "")
-      make_rule += " ";
-    make_rule += ClassFileName(lang, parser, **it, path);
+  for (auto it = parser.enums_.vec.begin(); it != parser.enums_.vec.end();
+       ++it) {
+    if (make_rule != "") make_rule += " ";
+    make_rule += directory + (**it).name + lang.file_extension;
   }
 
-  for (auto it = parser.structs_.vec.begin();
-       it != parser.structs_.vec.end(); ++it) {
-    if (make_rule != "")
-      make_rule += " ";
-    make_rule += ClassFileName(lang, parser, **it, path);
+  for (auto it = parser.structs_.vec.begin(); it != parser.structs_.vec.end();
+       ++it) {
+    if (make_rule != "") make_rule += " ";
+    make_rule += directory + (**it).name + lang.file_extension;
   }
 
   make_rule += ": ";
   auto included_files = parser.GetIncludedFilesRecursive(file_name);
-  for (auto it = included_files.begin();
-       it != included_files.end(); ++it) {
+  for (auto it = included_files.begin(); it != included_files.end(); ++it) {
     make_rule += " " + *it;
   }
   return make_rule;

src/idl_gen_go.cpp

@@ -53,20 +53,6 @@ std::string OffsetPrefix(const FieldDef &field) {
          "))\n\tif o != 0 {\n";
 }
 
-// Begin by declaring namespace and imports.
-static void BeginFile(const std::string name_space_name,
-                      const bool needs_imports,
-                      std::string *code_ptr) {
-  std::string &code = *code_ptr;
-  code += "// automatically generated, do not modify\n\n";
-  code += "package " + name_space_name + "\n\n";
-  if (needs_imports) {
-    code += "import (\n";
-    code += "\tflatbuffers \"github.com/google/flatbuffers/go\"\n";
-    code += ")\n";
-  }
-}
-
 // Begin a class declaration.
 static void BeginClass(const StructDef &struct_def, std::string *code_ptr) {
   std::string &code = *code_ptr;
@@ -589,32 +575,6 @@ static std::string GenMethod(const FieldDef &field) {
     : (IsStruct(field.value.type) ? "Struct" : "UOffsetT");
 }
 
-
-// Save out the generated code for a Go Table type.
-static bool SaveType(const Parser &parser, const Definition &def,
-                     const std::string &classcode, const std::string &path,
-                     bool needs_imports) {
-  if (!classcode.length()) return true;
-
-  std::string namespace_name;
-  std::string namespace_dir = path;  // Either empty or ends in separator.
-  auto &namespaces = parser.namespaces_.back()->components;
-  for (auto it = namespaces.begin(); it != namespaces.end(); ++it) {
-    if (namespace_name.length()) {
-      namespace_name += ".";
-    }
-    namespace_name = *it;
-    namespace_dir += *it + kPathSeparator;
-  }
-  EnsureDirExists(namespace_dir);
-
-  std::string code = "";
-  BeginFile(namespace_name, needs_imports, &code);
-  code += classcode;
-  std::string filename = namespace_dir + def.name + ".go";
-  return SaveFile(filename.c_str(), code, false);
-}
-
 static std::string GenTypeBasic(const Type &type) {
   static const char *ctypename[] = {
     #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \
@@ -671,18 +631,44 @@ class GoGenerator : public BaseGenerator {
          ++it) {
       std::string enumcode;
       go::GenEnum(**it, &enumcode);
-      if (!go::SaveType(parser_, **it, enumcode, path_, false)) return false;
+      if (!SaveType(**it, enumcode, false)) return false;
     }
 
     for (auto it = parser_.structs_.vec.begin();
          it != parser_.structs_.vec.end(); ++it) {
       std::string declcode;
       go::GenStruct(**it, &declcode, parser_.root_struct_def_);
-      if (!go::SaveType(parser_, **it, declcode, path_, true)) return false;
+      if (!SaveType(**it, declcode, true)) return false;
     }
 
     return true;
   }
+
+ private:
+  // Begin by declaring namespace and imports.
+  void BeginFile(const std::string name_space_name, const bool needs_imports,
+                 std::string *code_ptr) {
+    std::string &code = *code_ptr;
+    code = code + "// " + FlatBuffersGeneratedWarning();
+    code += "package " + name_space_name + "\n\n";
+    if (needs_imports) {
+      code += "import (\n";
+      code += "\tflatbuffers \"github.com/google/flatbuffers/go\"\n";
+      code += ")\n";
+    }
+  }
+
+  // Save out the generated code for a Go Table type.
+  bool SaveType(const Definition &def, const std::string &classcode,
+                bool needs_imports) {
+    if (!classcode.length()) return true;
+
+    std::string code = "";
+    BeginFile(LastNamespacePart(), needs_imports, &code);
+    code += classcode;
+    std::string filename = namespace_dir_ + def.name + ".go";
+    return SaveFile(filename.c_str(), code, false);
+  }
 };
 }  // namespace go
 

src/idl_gen_js.cpp

@@ -678,31 +678,27 @@ class JsGenerator : public BaseGenerator {
   // Iterate through all definitions we haven't generate code for (enums,
   // structs, and tables) and output them to a single file.
   bool generate() {
+    if (IsEverythingGenerated()) return true;
+
     std::string enum_code, struct_code, exports_code, code;
     generateEnums(&enum_code, &exports_code);
     generateStructs(&struct_code, &exports_code);
 
-    // Only output file-level code if there were any declarations.
-    if (enum_code.length() || struct_code.length()) {
-      code +=
-          "// automatically generated by the FlatBuffers compiler, do not "
-          "modify\n\n";
+    code = code + "// " + FlatBuffersGeneratedWarning();
 
-      // Generate code for all the namespace declarations.
-      GenNamespaces(parser_, &code, &exports_code);
+    // Generate code for all the namespace declarations.
+    GenNamespaces(parser_, &code, &exports_code);
 
-      // Output the main declaration code from above.
-      code += enum_code;
-      code += struct_code;
+    // Output the main declaration code from above.
+    code += enum_code;
+    code += struct_code;
 
-      if (!exports_code.empty() && !parser_.opts.skip_js_exports) {
-        code += "// Exports for Node.js and RequireJS\n";
-        code += exports_code;
-      }
+    if (!exports_code.empty() && !parser_.opts.skip_js_exports) {
+      code += "// Exports for Node.js and RequireJS\n";
+      code += exports_code;
     }
 
-    return !code.length() ||
-           SaveFile(GeneratedFileName(path_, file_name_).c_str(), code, false);
+    return SaveFile(GeneratedFileName(path_, file_name_).c_str(), code, false);
   }
 
  private:

src/idl_gen_php.cpp

@@ -54,24 +54,6 @@ namespace php {
     // Hardcode spaces per indentation.
     const std::string Indent = "    ";
 
-    // Begin by declaring namespace and imports.
-    static void BeginFile(const std::string name_space_name,
-      const bool needs_imports,
-      std::string *code_ptr) {
-      std::string &code = *code_ptr;
-      code += "<?php\n";
-      code += "// automatically generated, do not modify\n\n";
-      code += "namespace " + name_space_name + ";\n\n";
-
-      if (needs_imports) {
-        code += "use \\Google\\FlatBuffers\\Struct;\n";
-        code += "use \\Google\\FlatBuffers\\Table;\n";
-        code += "use \\Google\\FlatBuffers\\ByteBuffer;\n";
-        code += "use \\Google\\FlatBuffers\\FlatBufferBuilder;\n";
-        code += "\n";
-      }
-    }
-
     // Begin a class declaration.
     static void BeginClass(const StructDef &struct_def, std::string *code_ptr) {
       std::string &code = *code_ptr;
@@ -867,35 +849,6 @@ namespace php {
         : (IsStruct(field.value.type) ? "Struct" : "Offset");
     }
 
-
-    // Save out the generated code for a Php Table type.
-    static bool SaveType(const Parser &parser, const Definition &def,
-      const std::string &classcode, const std::string &path,
-      bool needs_imports) {
-      if (!classcode.length()) return true;
-
-      std::string namespace_name;
-      std::string namespace_dir = path;
-
-      auto &namespaces = parser.namespaces_.back()->components;
-      for (auto it = namespaces.begin(); it != namespaces.end(); ++it) {
-        if (namespace_name.length()) {
-          namespace_name += "\\";
-          namespace_dir += kPathSeparator;
-        }
-        namespace_name += *it;
-        namespace_dir += *it;
-        EnsureDirExists(namespace_dir.c_str());
-      }
-
-      std::string code = "";
-      BeginFile(namespace_name, needs_imports, &code);
-      code += classcode;
-
-      std::string filename = namespace_dir + kPathSeparator + def.name + ".php";
-      return SaveFile(filename.c_str(), code, false);
-    }
-
     static std::string GenTypeBasic(const Type &type) {
       static const char *ctypename[] = {
 #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \
@@ -993,8 +946,7 @@ namespace php {
           auto &enum_def = **it;
           std::string enumcode;
           GenEnum(enum_def, &enumcode);
-          if (!SaveType(parser_, enum_def, enumcode, path_, false))
-            return false;
+          if (!SaveType(enum_def, enumcode, false)) return false;
         }
         return true;
       }
@@ -1005,11 +957,41 @@ namespace php {
           auto &struct_def = **it;
           std::string declcode;
           GenStruct(parser_, struct_def, &declcode);
-          if (!SaveType(parser_, struct_def, declcode, path_, true))
-            return false;
+          if (!SaveType(struct_def, declcode, true)) return false;
         }
         return true;
       }
+
+      // Begin by declaring namespace and imports.
+      void BeginFile(const std::string name_space_name,
+                     const bool needs_imports, std::string *code_ptr) {
+        std::string &code = *code_ptr;
+        code += "<?php\n";
+        code = code + "// " + FlatBuffersGeneratedWarning();
+        code += "namespace " + name_space_name + ";\n\n";
+
+        if (needs_imports) {
+          code += "use \\Google\\FlatBuffers\\Struct;\n";
+          code += "use \\Google\\FlatBuffers\\Table;\n";
+          code += "use \\Google\\FlatBuffers\\ByteBuffer;\n";
+          code += "use \\Google\\FlatBuffers\\FlatBufferBuilder;\n";
+          code += "\n";
+        }
+      }
+
+      // Save out the generated code for a Php Table type.
+      bool SaveType(const Definition &def, const std::string &classcode,
+                    bool needs_imports) {
+        if (!classcode.length()) return true;
+
+        std::string code = "";
+        BeginFile(FullNamespace("\\"), needs_imports, &code);
+        code += classcode;
+
+        std::string filename =
+            namespace_dir_ + kPathSeparator + def.name + ".php";
+        return SaveFile(filename.c_str(), code, false);
+      }
     };
     }  // namespace php
 

src/idl_gen_python.cpp

@@ -49,18 +49,6 @@ std::string OffsetPrefix(const FieldDef &field) {
          "))\n" + Indent + Indent + "if o != 0:\n";
 }
 
-// Begin by declaring namespace and imports.
-static void BeginFile(const std::string name_space_name,
-                      const bool needs_imports,
-                      std::string *code_ptr) {
-  std::string &code = *code_ptr;
-  code += "# automatically generated, do not modify\n\n";
-  code += "# namespace: " + name_space_name + "\n\n";
-  if (needs_imports) {
-    code += "import flatbuffers\n\n";
-  }
-}
-
 // Begin a class declaration.
 static void BeginClass(const StructDef &struct_def, std::string *code_ptr) {
   std::string &code = *code_ptr;
@@ -558,37 +546,6 @@ static std::string GenMethod(const FieldDef &field) {
     : (IsStruct(field.value.type) ? "Struct" : "UOffsetTRelative");
 }
 
-
-// Save out the generated code for a Python Table type.
-static bool SaveType(const Parser &parser, const Definition &def,
-                     const std::string &classcode, const std::string &path,
-                     bool needs_imports) {
-  if (!classcode.length()) return true;
-
-  std::string namespace_name;
-  std::string namespace_dir = path;
-  auto &namespaces = parser.namespaces_.back()->components;
-  for (auto it = namespaces.begin(); it != namespaces.end(); ++it) {
-    if (namespace_name.length()) {
-      namespace_name += ".";
-      namespace_dir += kPathSeparator;
-    }
-    namespace_name = *it;
-    namespace_dir += *it;
-    EnsureDirExists(namespace_dir.c_str());
-
-    std::string init_py_filename = namespace_dir + "/__init__.py";
-    SaveFile(init_py_filename.c_str(), "", false);
-  }
-
-
-  std::string code = "";
-  BeginFile(namespace_name, needs_imports, &code);
-  code += classcode;
-  std::string filename = namespace_dir + kPathSeparator + def.name + ".py";
-  return SaveFile(filename.c_str(), code, false);
-}
-
 static std::string GenTypeBasic(const Type &type) {
   static const char *ctypename[] = {
     #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE, PTYPE) \
@@ -653,7 +610,7 @@ class PythonGenerator : public BaseGenerator {
       auto &enum_def = **it;
       std::string enumcode;
       GenEnum(enum_def, &enumcode);
-      if (!SaveType(parser_, enum_def, enumcode, path_, false)) return false;
+      if (!SaveType(enum_def, enumcode, false)) return false;
     }
     return true;
   }
@@ -664,10 +621,42 @@ class PythonGenerator : public BaseGenerator {
       auto &struct_def = **it;
       std::string declcode;
       GenStruct(struct_def, &declcode, parser_.root_struct_def_);
-      if (!SaveType(parser_, struct_def, declcode, path_, true)) return false;
+      if (!SaveType(struct_def, declcode, true)) return false;
     }
     return true;
   }
+
+  // Begin by declaring namespace and imports.
+  void BeginFile(const std::string name_space_name, const bool needs_imports,
+                 std::string *code_ptr) {
+    std::string &code = *code_ptr;
+    code = code + "# " + FlatBuffersGeneratedWarning();
+    code += "# namespace: " + name_space_name + "\n\n";
+    if (needs_imports) {
+      code += "import flatbuffers\n\n";
+    }
+  }
+
+  // Save out the generated code for a Python Table type.
+  bool SaveType(const Definition &def, const std::string &classcode,
+                bool needs_imports) {
+    if (!classcode.length()) return true;
+
+    std::string namespace_dir = path_;
+    auto &namespaces = parser_.namespaces_.back()->components;
+    for (auto it = namespaces.begin(); it != namespaces.end(); ++it) {
+      if (it != namespaces.begin()) namespace_dir += kPathSeparator;
+      namespace_dir += *it;
+      std::string init_py_filename = namespace_dir + "/__init__.py";
+      SaveFile(init_py_filename.c_str(), "", false);
+    }
+
+    std::string code = "";
+    BeginFile(LastNamespacePart(), needs_imports, &code);
+    code += classcode;
+    std::string filename = namespace_dir_ + kPathSeparator + def.name + ".py";
+    return SaveFile(filename.c_str(), code, false);
+  }
 };
 
 }  // namespace python

src/idl_parser.cpp

@@ -236,12 +236,19 @@ CheckedError Parser::Next() {
         if(!isdigit(static_cast<const unsigned char>(*cursor_))) return NoError();
         return Error("floating point constant can\'t start with \".\"");
       case '\"':
-      case '\'':
+      case '\'': {
+        int unicode_high_surrogate = -1;
+
         while (*cursor_ != c) {
           if (*cursor_ < ' ' && *cursor_ >= 0)
             return Error("illegal character in string constant");
           if (*cursor_ == '\\') {
             cursor_++;
+            if (unicode_high_surrogate != -1 &&
+                *cursor_ != 'u') {
+              return Error(
+                "illegal Unicode sequence (unpaired high surrogate)");
+            }
             switch (*cursor_) {
               case 'n':  attribute_ += '\n'; cursor_++; break;
               case 't':  attribute_ += '\t'; cursor_++; break;
@@ -263,18 +270,51 @@ CheckedError Parser::Next() {
                 cursor_++;
                 int64_t val;
                 ECHECK(ParseHexNum(4, &val));
-                ToUTF8(static_cast<int>(val), &attribute_);
+                if (val >= 0xD800 && val <= 0xDBFF) {
+                  if (unicode_high_surrogate != -1) {
+                    return Error(
+                      "illegal Unicode sequence (multiple high surrogates)");
+                  } else {
+                    unicode_high_surrogate = static_cast<int>(val);
+                  }
+                } else if (val >= 0xDC00 && val <= 0xDFFF) {
+                  if (unicode_high_surrogate == -1) {
+                    return Error(
+                      "illegal Unicode sequence (unpaired low surrogate)");
+                  } else {
+                    int code_point = 0x10000 +
+                      ((unicode_high_surrogate & 0x03FF) << 10) +
+                      (val & 0x03FF);
+                    ToUTF8(code_point, &attribute_);
+                    unicode_high_surrogate = -1;
+                  }
+                } else {
+                  if (unicode_high_surrogate != -1) {
+                    return Error(
+                      "illegal Unicode sequence (unpaired high surrogate)");
+                  }
+                  ToUTF8(static_cast<int>(val), &attribute_);
+                }
                 break;
               }
               default: return Error("unknown escape code in string constant");
             }
           } else { // printable chars + UTF-8 bytes
+            if (unicode_high_surrogate != -1) {
+              return Error(
+                "illegal Unicode sequence (unpaired high surrogate)");
+            }
             attribute_ += *cursor_++;
           }
         }
+        if (unicode_high_surrogate != -1) {
+          return Error(
+            "illegal Unicode sequence (unpaired high surrogate)");
+        }
         cursor_++;
         token_ = kTokenStringConstant;
         return NoError();
+      }
       case '/':
         if (*cursor_ == '/') {
           const char *start = ++cursor_;
@@ -1961,14 +2001,14 @@ flatbuffers::Offset<flatbuffers::Vector<flatbuffers::Offset<
     Definition::SerializeAttributes(FlatBufferBuilder *builder,
                                     const Parser &parser) const {
   std::vector<flatbuffers::Offset<reflection::KeyValue>> attrs;
-  for (auto kv : attributes.dict) {
-    auto it = parser.known_attributes_.find(kv.first);
+  for (auto kv = attributes.dict.begin(); kv != attributes.dict.end(); ++kv) {
+    auto it = parser.known_attributes_.find(kv->first);
     assert(it != parser.known_attributes_.end());
     if (!it->second) {  // Custom attribute.
       attrs.push_back(
-          reflection::CreateKeyValue(*builder, builder->CreateString(kv.first),
+          reflection::CreateKeyValue(*builder, builder->CreateString(kv->first),
                                      builder->CreateString(
-                                         kv.second->constant)));
+                                         kv->second->constant)));
     }
   }
   if (attrs.size()) {

tests/MyGame/Example/Any.cs

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace MyGame.Example
 {

tests/MyGame/Example/Any.go

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package Example
 

tests/MyGame/Example/Any.java

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package MyGame.Example;
 

tests/MyGame/Example/Any.php

 <?php
-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace MyGame\Example;
 

tests/MyGame/Example/Any.py

-# automatically generated, do not modify
+# automatically generated by the FlatBuffers compiler, do not modify
 
 # namespace: Example
 

tests/MyGame/Example/Color.cs

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace MyGame.Example
 {

tests/MyGame/Example/Color.go

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package Example
 

tests/MyGame/Example/Color.java

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package MyGame.Example;
 

tests/MyGame/Example/Color.php

 <?php
-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace MyGame\Example;
 

tests/MyGame/Example/Color.py

-# automatically generated, do not modify
+# automatically generated by the FlatBuffers compiler, do not modify
 
 # namespace: Example
 

tests/MyGame/Example/Monster.cs

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace MyGame.Example
 {

tests/MyGame/Example/Monster.go

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package Example
 

tests/MyGame/Example/Monster.java

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package MyGame.Example;
 

tests/MyGame/Example/Monster.php

 <?php
-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace MyGame\Example;
 

tests/MyGame/Example/Monster.py

-# automatically generated, do not modify
+# automatically generated by the FlatBuffers compiler, do not modify
 
 # namespace: Example
 

tests/MyGame/Example/Stat.cs

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace MyGame.Example
 {

tests/MyGame/Example/Stat.go

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package Example
 

tests/MyGame/Example/Stat.java

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package MyGame.Example;
 

tests/MyGame/Example/Stat.php

 <?php
-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace MyGame\Example;
 

tests/MyGame/Example/Stat.py

-# automatically generated, do not modify
+# automatically generated by the FlatBuffers compiler, do not modify
 
 # namespace: Example
 

tests/MyGame/Example/Test.cs

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace MyGame.Example
 {

tests/MyGame/Example/Test.go

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package Example
 

tests/MyGame/Example/Test.java

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package MyGame.Example;
 

tests/MyGame/Example/Test.php

 <?php
-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace MyGame\Example;
 

tests/MyGame/Example/Test.py

-# automatically generated, do not modify
+# automatically generated by the FlatBuffers compiler, do not modify
 
 # namespace: Example
 

tests/MyGame/Example/TestSimpleTableWithEnum.cs

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace MyGame.Example
 {

tests/MyGame/Example/TestSimpleTableWithEnum.go

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package Example
 

tests/MyGame/Example/TestSimpleTableWithEnum.java

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package MyGame.Example;
 

tests/MyGame/Example/TestSimpleTableWithEnum.php

 <?php
-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace MyGame\Example;
 

tests/MyGame/Example/TestSimpleTableWithEnum.py

-# automatically generated, do not modify
+# automatically generated by the FlatBuffers compiler, do not modify
 
 # namespace: Example
 

tests/MyGame/Example/Vec3.cs

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace MyGame.Example
 {

tests/MyGame/Example/Vec3.go

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package Example
 

tests/MyGame/Example/Vec3.java

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package MyGame.Example;
 

tests/MyGame/Example/Vec3.php

 <?php
-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace MyGame\Example;
 

tests/MyGame/Example/Vec3.py

-# automatically generated, do not modify
+# automatically generated by the FlatBuffers compiler, do not modify
 
 # namespace: Example
 

tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.cs

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace NamespaceA.NamespaceB
 {

tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.go

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package NamespaceB
 

tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.java

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package NamespaceA.NamespaceB;
 

tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.php

 <?php
-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace NamespaceA\NamespaceB;
 

tests/namespace_test/NamespaceA/NamespaceB/EnumInNestedNS.py

-# automatically generated, do not modify
+# automatically generated by the FlatBuffers compiler, do not modify
 
 # namespace: NamespaceB
 

tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.cs

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace NamespaceA.NamespaceB
 {

tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.go

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package NamespaceB
 

tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.java

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package NamespaceA.NamespaceB;
 

tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.php

 <?php
-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace NamespaceA\NamespaceB;
 

tests/namespace_test/NamespaceA/NamespaceB/StructInNestedNS.py

-# automatically generated, do not modify
+# automatically generated by the FlatBuffers compiler, do not modify
 
 # namespace: NamespaceB
 

tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.cs

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace NamespaceA.NamespaceB
 {

tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.go

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package NamespaceB
 

tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.java

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package NamespaceA.NamespaceB;
 

tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.php

 <?php
-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace NamespaceA\NamespaceB;
 

tests/namespace_test/NamespaceA/NamespaceB/TableInNestedNS.py

-# automatically generated, do not modify
+# automatically generated by the FlatBuffers compiler, do not modify
 
 # namespace: NamespaceB
 

tests/namespace_test/NamespaceA/SecondTableInA.cs

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace NamespaceA
 {

tests/namespace_test/NamespaceA/SecondTableInA.go

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package NamespaceA
 

tests/namespace_test/NamespaceA/SecondTableInA.java

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package NamespaceA;
 

tests/namespace_test/NamespaceA/SecondTableInA.php

 <?php
-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace NamespaceA;
 

tests/namespace_test/NamespaceA/SecondTableInA.py

-# automatically generated, do not modify
+# automatically generated by the FlatBuffers compiler, do not modify
 
 # namespace: NamespaceA
 

tests/namespace_test/NamespaceA/TableInC.cs

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace NamespaceA
 {

tests/namespace_test/NamespaceA/TableInC.go

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package NamespaceA
 

tests/namespace_test/NamespaceA/TableInC.java

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package NamespaceA;
 

tests/namespace_test/NamespaceA/TableInC.php

 <?php
-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace NamespaceA;
 

tests/namespace_test/NamespaceA/TableInC.py

-# automatically generated, do not modify
+# automatically generated by the FlatBuffers compiler, do not modify
 
 # namespace: NamespaceA
 

tests/namespace_test/NamespaceA/TableInFirstNS.cs

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace NamespaceA
 {

tests/namespace_test/NamespaceA/TableInFirstNS.go

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package NamespaceA
 

tests/namespace_test/NamespaceA/TableInFirstNS.java

-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 package NamespaceA;
 

tests/namespace_test/NamespaceA/TableInFirstNS.php

 <?php
-// automatically generated, do not modify
+// automatically generated by the FlatBuffers compiler, do not modify
 
 namespace NamespaceA;
 

tests/namespace_test/NamespaceA/TableInFirstNS.py

-# automatically generated, do not modify
+# automatically generated by the FlatBuffers compiler, do not modify
 
 # namespace: NamespaceA
 

tests/test.cpp

@@ -868,6 +868,44 @@ void UnicodeTest() {
                      "\\u5225\\u30B5\\u30A4\\u30C8\\x01\\x80\"}", true);
 }
 
+void UnicodeSurrogatesTest() {
+  flatbuffers::Parser parser;
+
+  TEST_EQ(
+    parser.Parse(
+      "table T { F:string (id: 0); }"
+      "root_type T;"
+      "{ F:\"\\uD83D\\uDCA9\"}"), true);
+  auto root = flatbuffers::GetRoot<flatbuffers::Table>(
+    parser.builder_.GetBufferPointer());
+  auto string = root->GetPointer<flatbuffers::String *>(
+    flatbuffers::FieldIndexToOffset(0));
+  TEST_EQ(strcmp(string->c_str(), "\xF0\x9F\x92\xA9"), 0);
+}
+
+void UnicodeInvalidSurrogatesTest() {
+  TestError(
+    "table T { F:string; }"
+    "root_type T;"
+    "{ F:\"\\uD800\"}", "unpaired high surrogate");
+  TestError(
+    "table T { F:string; }"
+    "root_type T;"
+    "{ F:\"\\uD800abcd\"}", "unpaired high surrogate");
+  TestError(
+    "table T { F:string; }"
+    "root_type T;"
+    "{ F:\"\\uD800\\n\"}", "unpaired high surrogate");
+  TestError(
+    "table T { F:string; }"
+    "root_type T;"
+    "{ F:\"\\uD800\\uD800\"}", "multiple high surrogates");
+  TestError(
+    "table T { F:string; }"
+    "root_type T;"
+    "{ F:\"\\uDC00\"}", "unpaired low surrogate");
+}
+
 void UnknownFieldsTest() {
   flatbuffers::IDLOptions opts;
   opts.skip_unexpected_fields_in_json = true;
@@ -916,6 +954,8 @@ int main(int /*argc*/, const char * /*argv*/[]) {
   EnumStringsTest();
   IntegerOutOfRangeTest();
   UnicodeTest();
+  UnicodeSurrogatesTest();
+  UnicodeInvalidSurrogatesTest();
   UnknownFieldsTest();
 
   if (!testing_fails) {