Added support for imports and many other .proto features.

Change-Id: I6600021b7ec8c486794349511232c3e604421c5b
Tested: on Linux.

docs/html/md__compiler.html

@@ -84,7 +84,7 @@ $(document).ready(function(){initNavTree('md__compiler.html','');});
 <li><code>--gen-mutable</code> : Generate additional non-const accessors for mutating FlatBuffers in-place.</li>
 <li><code>--gen-onefile</code> : Generate single output file (useful for C#)</li>
 <li><code>--raw-binary</code> : Allow binaries without a file_indentifier to be read. This may crash flatc given a mismatched schema.</li>
-<li><code>--proto</code>: Expect input files to be .proto files (protocol buffers). Output the corresponding .fbs file. Currently supports: <code>package</code>, <code>message</code>, <code>enum</code>, nested declarations. Does not support, but will skip without error: <code>import</code>, <code>option</code>. Does not support, will generate error: <code>service</code>, <code>extend</code>, <code>extensions</code>, <code>oneof</code>, <code>group</code>, custom options.</li>
+<li><code>--proto</code>: Expect input files to be .proto files (protocol buffers). Output the corresponding .fbs file. Currently supports: <code>package</code>, <code>message</code>, <code>enum</code>, nested declarations, <code>import</code> (use <code>-I</code> for paths), <code>extend</code>, <code>oneof</code>, <code>group</code>. Does not support, but will skip without error: <code>option</code>, <code>service</code>, <code>extensions</code>, and most everything else.</li>
 <li><code>--schema</code>: Serialize schemas instead of JSON (use with -b). This will output a binary version of the specified schema that itself corresponds to the reflection/reflection.fbs schema. Loading this binary file is the basis for reflection functionality. </li>
 </ul>
 </div></div><!-- contents -->

docs/source/Compiler.md

@@ -71,10 +71,10 @@ be generated for each file processed:
 
 -   `--proto`: Expect input files to be .proto files (protocol buffers).
     Output the corresponding .fbs file.
-    Currently supports: `package`, `message`, `enum`, nested declarations.
-    Does not support, but will skip without error: `import`, `option`.
-    Does not support, will generate error: `service`, `extend`, `extensions`,
-    `oneof`, `group`, custom options.
+    Currently supports: `package`, `message`, `enum`, nested declarations,
+    `import` (use `-I` for paths), `extend`, `oneof`, `group`.
+    Does not support, but will skip without error: `option`, `service`,
+    `extensions`, and most everything else.
 
 -   `--schema`: Serialize schemas instead of JSON (use with -b). This will
     output a binary version of the specified schema that itself corresponds

include/flatbuffers/idl.h

@@ -125,6 +125,11 @@ struct Type {
       enum_def(_ed)
   {}
 
+  bool operator==(const Type &o) {
+    return base_type == o.base_type && element == o.element &&
+           struct_def == o.struct_def && enum_def == o.enum_def;
+  }
+
   Type VectorType() const { return Type(element, struct_def, enum_def); }
 
   Offset<reflection::Type> Serialize(FlatBufferBuilder *builder) const;
@@ -163,6 +168,17 @@ template<typename T> class SymbolTable {
     return false;
   }
 
+  void Move(const std::string &oldname, const std::string &newname) {
+    auto it = dict.find(oldname);
+    if (it != dict.end()) {
+      auto obj = it->second;
+      dict.erase(it);
+      dict[newname] = obj;
+    } else {
+      assert(false);
+    }
+  }
+
   T *Lookup(const std::string &name) const {
     auto it = dict.find(name);
     return it == dict.end() ? nullptr : it->second;
@@ -178,6 +194,13 @@ template<typename T> class SymbolTable {
 // A name space, as set in the schema.
 struct Namespace {
   std::vector<std::string> components;
+
+  // Given a (potentally unqualified) name, return the "fully qualified" name
+  // which has a full namespaced descriptor.
+  // With max_components you can request less than the number of components
+  // the current namespace has.
+  std::string GetFullyQualifiedName(const std::string &name,
+                                    size_t max_components = 1000) const;
 };
 
 // Base class for all definition types (fields, structs_, enums_).
@@ -293,7 +316,8 @@ class Parser {
       cursor_(nullptr),
       line_(1),
       proto_mode_(proto_mode),
-      strict_json_(strict_json) {
+      strict_json_(strict_json),
+      anonymous_counter(0) {
     // Just in case none are declared:
     namespaces_.push_back(new Namespace());
     known_attributes_.insert("deprecated");
@@ -331,12 +355,6 @@ class Parser {
   // Mark all definitions as already having code generated.
   void MarkGenerated();
 
-  // Given a (potentally unqualified) name, return the "fully qualified" name
-  // which has a full namespaced descriptor. If the parser has no current
-  // namespace context, or if the name passed is partially qualified the input
-  // is simply returned.
-  std::string GetFullyQualifiedName(const std::string &name) const;
-
   // Get the files recursively included by the given file. The returned
   // container will have at least the given file.
   std::set<std::string> GetIncludedFilesRecursive(
@@ -351,6 +369,7 @@ class Parser {
   void Next();
   bool IsNext(int t);
   void Expect(int t);
+  std::string TokenToStringId(int t);
   EnumDef *LookupEnum(const std::string &id);
   void ParseNamespacing(std::string *id, std::string *last);
   void ParseTypeIdent(Type &type);
@@ -369,12 +388,19 @@ class Parser {
   void ParseHash(Value &e, FieldDef* field);
   void ParseSingleValue(Value &e);
   int64_t ParseIntegerFromString(Type &type);
-  StructDef *LookupCreateStruct(const std::string &name);
-  void ParseEnum(bool is_union);
+  StructDef *LookupCreateStruct(const std::string &name,
+                                bool create_if_new = true,
+                                bool definition = false);
+  EnumDef &ParseEnum(bool is_union);
   void ParseNamespace();
-  StructDef &StartStruct();
+  StructDef &StartStruct(const std::string &name);
   void ParseDecl();
+  void ParseProtoFields(StructDef *struct_def, bool isextend,
+                        bool inside_oneof);
+  void ParseProtoOption();
+  void ParseProtoKey();
   void ParseProtoDecl();
+  void ParseProtoCurliesOrIdent();
   Type ParseTypeFromProtoType();
 
  public:
@@ -405,6 +431,8 @@ class Parser {
   std::vector<uint8_t> struct_stack_;
 
   std::set<std::string> known_attributes_;
+
+  int anonymous_counter;
 };
 
 // Utility functions for multiple generators:

src/idl_gen_cpp.cpp

@@ -287,8 +287,8 @@ static void GenTable(const Parser &parser, StructDef &struct_def,
       }
       auto nested = field.attributes.Lookup("nested_flatbuffer");
       if (nested) {
-        std::string qualified_name = parser.GetFullyQualifiedName(
-            nested->constant);
+        std::string qualified_name =
+            parser.namespaces_.back()->GetFullyQualifiedName(nested->constant);
         auto nested_root = parser.structs_.Lookup(qualified_name);
         assert(nested_root);  // Guaranteed to exist by parser.
         (void)nested_root;
@@ -719,7 +719,8 @@ std::string GenerateCPP(const Parser &parser,
     // Generate convenient global helper functions:
     if (parser.root_struct_def_) {
       auto &name = parser.root_struct_def_->name;
-      std::string qualified_name = parser.GetFullyQualifiedName(name);
+      std::string qualified_name =
+          parser.namespaces_.back()->GetFullyQualifiedName(name);
       std::string cpp_qualified_name = TranslateNameSpace(qualified_name);
 
       // The root datatype accessor:

src/idl_gen_fbs.cpp

@@ -24,19 +24,49 @@ namespace flatbuffers {
 
 static std::string GenType(const Type &type) {
   switch (type.base_type) {
-    case BASE_TYPE_STRUCT: return type.struct_def->name;
-    case BASE_TYPE_UNION:  return type.enum_def->name;
-    case BASE_TYPE_VECTOR: return "[" + GenType(type.VectorType()) + "]";
-    default: return kTypeNames[type.base_type];
+    case BASE_TYPE_STRUCT:
+      return type.struct_def->defined_namespace->GetFullyQualifiedName(
+               type.struct_def->name);
+    case BASE_TYPE_UNION:
+      return type.enum_def->defined_namespace->GetFullyQualifiedName(
+               type.enum_def->name);
+    case BASE_TYPE_VECTOR:
+        return "[" + GenType(type.VectorType()) + "]";
+    default:
+        return kTypeNames[type.base_type];
   }
 }
 
+static void GenNameSpace(const Namespace &name_space, std::string *_schema,
+                         const Namespace **last_namespace) {
+  if (*last_namespace == &name_space) return;
+  *last_namespace = &name_space;
+  auto &schema = *_schema;
+  schema += "namespace ";
+  for (auto it = name_space.components.begin();
+           it != name_space.components.end(); ++it) {
+    if (it != name_space.components.begin()) schema += ".";
+    schema += *it;
+  }
+  schema += ";\n\n";
+}
+
 // Generate a flatbuffer schema from the Parser's internal representation.
 std::string GenerateFBS(const Parser &parser, const std::string &file_name,
                         const GeneratorOptions &opts) {
+  // Proto namespaces may clash with table names, so we have to prefix all:
+  for (auto it = parser.namespaces_.begin(); it != parser.namespaces_.end();
+       ++it) {
+    for (auto comp = (*it)->components.begin(); comp != (*it)->components.end();
+         ++comp) {
+      (*comp) = "_" + (*comp);
+    }
+  }
+
   std::string schema;
   schema += "// Generated from " + file_name + ".proto\n\n";
   if (opts.include_dependence_headers) {
+    #ifdef FBS_GEN_INCLUDES  // TODO: currently all in one file.
     int num_includes = 0;
     for (auto it = parser.included_files_.begin();
          it != parser.included_files_.end(); ++it) {
@@ -48,19 +78,14 @@ std::string GenerateFBS(const Parser &parser, const std::string &file_name,
       }
     }
     if (num_includes) schema += "\n";
+    #endif
   }
-  schema += "namespace ";
-  auto name_space = parser.namespaces_.back();
-  for (auto it = name_space->components.begin();
-           it != name_space->components.end(); ++it) {
-    if (it != name_space->components.begin()) schema += ".";
-    schema += *it;
-  }
-  schema += ";\n\n";
   // Generate code for all the enum declarations.
+  const Namespace *last_namespace = nullptr;
   for (auto enum_def_it = parser.enums_.vec.begin();
            enum_def_it != parser.enums_.vec.end(); ++enum_def_it) {
     EnumDef &enum_def = **enum_def_it;
+    GenNameSpace(*enum_def.defined_namespace, &schema, &last_namespace);
     GenComment(enum_def.doc_comment, &schema, nullptr);
     schema += "enum " + enum_def.name + " : ";
     schema += GenType(enum_def.underlying_type) + " {\n";
@@ -76,6 +101,7 @@ std::string GenerateFBS(const Parser &parser, const std::string &file_name,
   for (auto it = parser.structs_.vec.begin();
            it != parser.structs_.vec.end(); ++it) {
     StructDef &struct_def = **it;
+    GenNameSpace(*struct_def.defined_namespace, &schema, &last_namespace);
     GenComment(struct_def.doc_comment, &schema, nullptr);
     schema += "table " + struct_def.name + " {\n";
     for (auto field_it = struct_def.fields.vec.begin();

tests/MyGame/Example/Monster.java

@@ -7,6 +7,7 @@ import java.lang.*;
 import java.util.*;
 import com.google.flatbuffers.*;
 
+@SuppressWarnings("unused")
 public final class Monster extends Table {
   public static Monster getRootAsMonster(ByteBuffer _bb) { return getRootAsMonster(_bb, new Monster()); }
   public static Monster getRootAsMonster(ByteBuffer _bb, Monster obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__init(_bb.getInt(_bb.position()) + _bb.position(), _bb)); }

tests/MyGame/Example/Stat.java

@@ -7,6 +7,7 @@ import java.lang.*;
 import java.util.*;
 import com.google.flatbuffers.*;
 
+@SuppressWarnings("unused")
 public final class Stat extends Table {
   public static Stat getRootAsStat(ByteBuffer _bb) { return getRootAsStat(_bb, new Stat()); }
   public static Stat getRootAsStat(ByteBuffer _bb, Stat obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__init(_bb.getInt(_bb.position()) + _bb.position(), _bb)); }

tests/MyGame/Example/Test.java

@@ -7,6 +7,7 @@ import java.lang.*;
 import java.util.*;
 import com.google.flatbuffers.*;
 
+@SuppressWarnings("unused")
 public final class Test extends Struct {
   public Test __init(int _i, ByteBuffer _bb) { bb_pos = _i; bb = _bb; return this; }
 

tests/MyGame/Example/TestSimpleTableWithEnum.java

@@ -7,6 +7,7 @@ import java.lang.*;
 import java.util.*;
 import com.google.flatbuffers.*;
 
+@SuppressWarnings("unused")
 public final class TestSimpleTableWithEnum extends Table {
   public static TestSimpleTableWithEnum getRootAsTestSimpleTableWithEnum(ByteBuffer _bb) { return getRootAsTestSimpleTableWithEnum(_bb, new TestSimpleTableWithEnum()); }
   public static TestSimpleTableWithEnum getRootAsTestSimpleTableWithEnum(ByteBuffer _bb, TestSimpleTableWithEnum obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__init(_bb.getInt(_bb.position()) + _bb.position(), _bb)); }

tests/MyGame/Example/Vec3.java

@@ -7,6 +7,7 @@ import java.lang.*;
 import java.util.*;
 import com.google.flatbuffers.*;
 
+@SuppressWarnings("unused")
 public final class Vec3 extends Struct {
   public Vec3 __init(int _i, ByteBuffer _bb) { bb_pos = _i; bb = _bb; return this; }
 

tests/prototest/imported.proto

+package proto.test;
+
+message ImportedMessage {
+    optional int32 a = 26;
+}

tests/prototest/test.golden

 // Generated from test.proto
 
-namespace proto.test;
+namespace _proto._test;
 
 /// Enum doc comment.
-enum ProtoEnum : short {
+enum ProtoEnum : int {
   FOO = 1,
   /// Enum 2nd value doc comment misaligned.
   BAR = 5,
 }
 
+namespace _proto._test;
+
+table ImportedMessage {
+  a:int;
+}
+
+namespace _proto._test;
+
 /// 2nd table doc comment with
 /// many lines.
 table ProtoMessage {
@@ -29,10 +37,13 @@ table ProtoMessage {
   /// lines
   l:string (required);
   m:string;
-  n:OtherMessage;
+  n:_proto._test._ProtoMessage.OtherMessage;
   o:[string];
+  z:_proto._test.ImportedMessage;
 }
 
+namespace _proto._test._ProtoMessage;
+
 table OtherMessage {
   a:double;
   /// doc comment for b.

tests/prototest/test.proto

 // Sample .proto file that we can translate to the corresponding .fbs.
 
-package proto.test;
-
 option some_option = is_ignored;
-import "some_other_schema.proto";
+import "imported.proto";
+
+package proto.test;
 
 /// Enum doc comment.
 enum ProtoEnum {
@@ -41,4 +41,5 @@ message ProtoMessage {
   optional bytes m = 11;
   optional OtherMessage n = 12;
   repeated string o = 14;
+  optional ImportedMessage z = 16;
 }

tests/test.cpp

@@ -428,7 +428,8 @@ void ParseProtoTest() {
 
   // Parse proto.
   flatbuffers::Parser parser(false, true);
-  TEST_EQ(parser.Parse(protofile.c_str(), nullptr), true);
+  const char *include_directories[] = { "tests/prototest", nullptr };
+  TEST_EQ(parser.Parse(protofile.c_str(), include_directories), true);
 
   // Generate fbs.
   flatbuffers::GeneratorOptions opts;