The parser and flatc now allow include directories to be specified.

Bug: 17139854
Change-Id: I0eac65d054951e00a8811ad1d80ba8c37012dbf0
Tested: on Linux.

docs/html/md__compiler.html

@@ -53,7 +53,7 @@ $(document).ready(function(){initNavTree('md__compiler.html','');});
 <div class="title">Using the schema compiler </div>  </div>
 </div><!--header-->
 <div class="contents">
-<div class="textblock"><p>Usage: </p><pre class="fragment">flatc [ -c ] [ -j ] [ -b ] [ -t ] [ -o PATH ] [ -S ] FILES...
+<div class="textblock"><p>Usage: </p><pre class="fragment">flatc [ -c ] [ -j ] [ -b ] [ -t ] [ -o PATH ] [ -I PATH ] [ -S ] FILES...
       [ -- FILES...]
 </pre><p>The files are read and parsed in order, and can contain either schemas or data (see below). Later files can make use of definitions in earlier files.</p>
 <p><code>--</code> indicates that the following files are binary files in FlatBuffer format conforming to the schema(s) indicated before it. Incompatible binary files currently will give unpredictable results (!)</p>
@@ -64,6 +64,7 @@ $(document).ready(function(){initNavTree('md__compiler.html','');});
 <li><code>-b</code> : If data is contained in this file, generate a <code>filename.bin</code> containing the binary flatbuffer.</li>
 <li><code>-t</code> : If data is contained in this file, generate a <code>filename.json</code> representing the data in the flatbuffer.</li>
 <li><code>-o PATH</code> : Output all generated files to PATH (either absolute, or relative to the current directory). If omitted, PATH will be the current directory. PATH should end in your systems path separator, e.g. <code>/</code> or <code>\</code>.</li>
+<li><code>-I PATH</code> : when encountering <code>include</code> statements, attempt to load the files from this path. Paths will be tried in the order given, and if all fail (or none are specified) it will try to load relative to the path of the schema file being parsed.</li>
 <li><code>-S</code> : Generate strict JSON (field names are enclosed in quotes). By default, no quotes are generated.</li>
 <li><code>-P</code> : Don't prefix enum values in generated C++ by their enum type. </li>
 </ul>

docs/html/md__cpp_usage.html

@@ -120,6 +120,7 @@ assert(inv->Get(9) == 9);
 <p>Load text (either a schema or json) into an in-memory buffer (there is a convenient <code>LoadFile()</code> utility function in <code>flatbuffers/util.h</code> if you wish). Construct a parser: </p><pre class="fragment">flatbuffers::Parser parser;
 </pre><p>Now you can parse any number of text files in sequence: </p><pre class="fragment">parser.Parse(text_file.c_str());
 </pre><p>This works similarly to how the command-line compiler works: a sequence of files parsed by the same <code>Parser</code> object allow later files to reference definitions in earlier files. Typically this means you first load a schema file (which populates <code>Parser</code> with definitions), followed by one or more JSON files.</p>
+<p>As optional argument to <code>Parse</code>, you may specify a null-terminated list of include paths. If not specified, any include statements try to resolve from the current directory.</p>
 <p>If there were any parsing errors, <code>Parse</code> will return <code>false</code>, and <code>Parser::err</code> contains a human readable error string with a line number etc, which you should present to the creator of that file.</p>
 <p>After each JSON file, the <code>Parser::fbb</code> member variable is the <code>FlatBufferBuilder</code> that contains the binary buffer version of that file, that you can access as described above.</p>
 <p><code>samples/sample_text.cpp</code> is a code sample showing the above operations.</p>

docs/source/Compiler.md

@@ -2,7 +2,7 @@
 
 Usage:
 
-    flatc [ -c ] [ -j ] [ -b ] [ -t ] [ -o PATH ] [ -S ] FILES...
+    flatc [ -c ] [ -j ] [ -b ] [ -t ] [ -o PATH ] [ -I PATH ] [ -S ] FILES...
           [ -- FILES...]
 
 The files are read and parsed in order, and can contain either schemas
@@ -32,6 +32,11 @@ be generated for each file processed:
     current directory. PATH should end in your systems path separator,
     e.g. `/` or `\`.
 
+-   `-I PATH` : when encountering `include` statements, attempt to load the
+    files from this path. Paths will be tried in the order given, and if all
+    fail (or none are specified) it will try to load relative to the path of
+    the schema file being parsed.
+
 -   `-S` : Generate strict JSON (field names are enclosed in quotes).
     By default, no quotes are generated.
 

docs/source/CppUsage.md

@@ -249,6 +249,10 @@ reference definitions in earlier files. Typically this means you first
 load a schema file (which populates `Parser` with definitions), followed
 by one or more JSON files.
 
+As optional argument to `Parse`, you may specify a null-terminated list of
+include paths. If not specified, any include statements try to resolve from
+the current directory.
+
 If there were any parsing errors, `Parse` will return `false`, and
 `Parser::err` contains a human readable error string with a line number
 etc, which you should present to the creator of that file.

include/flatbuffers/idl.h

@@ -268,9 +268,12 @@ class Parser {
 
   // Parse the string containing either schema or JSON data, which will
   // populate the SymbolTable's or the FlatBufferBuilder above.
-  // filepath indicates the file that _source was loaded from, it is
-  // used to resolve any include statements.
-  bool Parse(const char *_source, const char *filepath);
+  // include_paths is used to resolve any include statements, and typically
+  // should at least include the project path (where you loaded source_ from).
+  // include_paths must be nullptr terminated if specified.
+  // If include_paths is nullptr, it will attempt to load from the current
+  // directory.
+  bool Parse(const char *_source, const char **include_paths = nullptr);
 
   // Set the root type. May override the one set in the schema.
   bool SetRootType(const char *name);
@@ -320,6 +323,7 @@ class Parser {
 
   std::vector<std::pair<Value, FieldDef *>> field_stack_;
   std::vector<uint8_t> struct_stack_;
+
   std::map<std::string, bool> included_files_;
 };
 

include/flatbuffers/util.h

@@ -132,6 +132,18 @@ inline std::string StripFileName(const std::string &filepath) {
   return i != std::string::npos ? filepath.substr(0, i) : "";
 }
 
+// Concatenates a path with a filename, regardless of wether the path
+// ends in a separator or not.
+inline std::string ConCatPathFileName(const std::string &path,
+                                      const std::string &filename) {
+  std::string filepath = path;
+  if (path.length() && path.back() != kPathSeparator &&
+                       path.back() != kPosixPathSeparator)
+    filepath += kPathSeparator;
+  filepath += filename;
+  return filepath;
+}
+
 // This function ensure a directory exists, by recursively
 // creating dirs for any parts of the path that don't exist yet.
 inline void EnsureDirExists(const std::string &filepath) {

samples/sample_text.cpp

@@ -37,8 +37,9 @@ int main(int /*argc*/, const char * /*argv*/[]) {
 
   // parse schema first, so we can use it to parse the data after
   flatbuffers::Parser parser;
-  ok = parser.Parse(schemafile.c_str(), "samples/") &&
-       parser.Parse(jsonfile.c_str(), "samples/");
+  const char *include_directories[] = { "samples", nullptr };
+  ok = parser.Parse(schemafile.c_str(), include_directories) &&
+       parser.Parse(jsonfile.c_str(), include_directories);
   assert(ok);
 
   // here, parser.builder_ contains a binary buffer that is the parsed data.

src/flatc.cpp

@@ -91,6 +91,7 @@ static void Error(const char *err, const char *obj, bool usage) {
     for (size_t i = 0; i < sizeof(generators) / sizeof(generators[0]); ++i)
       printf("  -%s      %s.\n", generators[i].extension, generators[i].help);
     printf("  -o PATH Prefix PATH to all generated files.\n"
+           "  -I PATH Search for includes in the specified path.\n"
            "  -S      Strict JSON: add quotes to field names.\n"
            "  -P      Don\'t prefix enum values with the enum name in C++.\n"
            "FILEs may depend on declarations in earlier files.\n"
@@ -112,6 +113,7 @@ int main(int argc, const char *argv[]) {
   bool generator_enabled[num_generators] = { false };
   bool any_generator = false;
   std::vector<std::string> filenames;
+  std::vector<const char *> include_directories;
   size_t binary_files_from = std::numeric_limits<size_t>::max();
   for (int i = 1; i < argc; i++) {
     const char *arg = argv[i];
@@ -123,11 +125,11 @@ int main(int argc, const char *argv[]) {
       switch (arg[1]) {
         case 'o':
           if (++i >= argc) Error("missing path following", arg, true);
-          output_path = argv[i];
-          if (!(output_path.back() == flatbuffers::kPathSeparator ||
-                output_path.back() == flatbuffers::kPosixPathSeparator)) {
-            output_path += flatbuffers::kPathSeparator;
-          }
+          output_path = flatbuffers::ConCatPathFileName(argv[i], "");
+          break;
+        case 'I':
+          if (++i >= argc) Error("missing path following", arg, true);
+          include_directories.push_back(argv[i]);
           break;
         case 'S':
           opts.strict_json = true;
@@ -177,8 +179,13 @@ int main(int argc, const char *argv[]) {
           reinterpret_cast<const uint8_t *>(contents.c_str()),
           contents.length());
       } else {
-        if (!parser.Parse(contents.c_str(), file_it->c_str()))
+        auto local_include_directory = flatbuffers::StripFileName(*file_it);
+        include_directories.push_back(local_include_directory.c_str());
+        include_directories.push_back(nullptr);
+        if (!parser.Parse(contents.c_str(), &include_directories[0]))
           Error((*file_it + ": " + parser.error_).c_str());
+        include_directories.pop_back();
+        include_directories.pop_back();
       }
 
       std::string filebase = flatbuffers::StripPath(

src/idl_parser.cpp

@@ -860,11 +860,7 @@ void Parser::MarkGenerated() {
   }
 }
 
-bool Parser::Parse(const char *source, const char *filepath) {
-  included_files_[filepath] = true;
-  // This is the starting point to reset to if we interrupted our parsing
-  // to deal with an include:
-  restart_parse_after_include:
+bool Parser::Parse(const char *source, const char **include_paths) {
   source_ = cursor_ = source;
   line_ = 1;
   error_.clear();
@@ -875,17 +871,25 @@ bool Parser::Parse(const char *source, const char *filepath) {
     while (IsNext(kTokenInclude)) {
       auto name = attribute_;
       Expect(kTokenStringConstant);
-      auto path = StripFileName(filepath);
-      if (path.length()) name = path + kPathSeparator + name;
       if (included_files_.find(name) == included_files_.end()) {
         // We found an include file that we have not parsed yet.
         // Load it and parse it.
         std::string contents;
-        if (!LoadFile(name.c_str(), true, &contents))
+        if (!include_paths) {
+          const char *current_directory[] = { "", nullptr };
+          include_paths = current_directory;
+        }
+        for (auto paths = include_paths; paths && *paths; paths++) {
+          auto filepath = flatbuffers::ConCatPathFileName(*paths, name);
+          if(LoadFile(filepath.c_str(), true, &contents)) break;
+        }
+        if (contents.empty())
           Error("unable to load include file: " + name);
-        Parse(contents.c_str(), name.c_str());
+        included_files_[name] = true;
+        if (!Parse(contents.c_str(), include_paths)) {
           // Any errors, we're done.
-        if (error_.length()) return false;
+          return false;
+        }
         // We do not want to output code for any included files:
         MarkGenerated();
         // This is the easiest way to continue this file after an include:
@@ -893,7 +897,9 @@ bool Parser::Parse(const char *source, const char *filepath) {
         // file anew. This will cause it to encounter the same include statement
         // again, but this time it will skip it, because it was entered into
         // included_files_.
-        goto restart_parse_after_include;
+        // This is recursive, but only go as deep as the number of include
+        // statements.
+        return Parse(source, include_paths);
       }
       Expect(';');
     }

tests/test.cpp

@@ -192,8 +192,9 @@ void ParseAndGenerateTextTest() {
 
   // parse schema first, so we can use it to parse the data after
   flatbuffers::Parser parser;
-  TEST_EQ(parser.Parse(schemafile.c_str(), "tests/"), true);
-  TEST_EQ(parser.Parse(jsonfile.c_str(), "tests/"), true);
+  const char *include_directories[] = { "tests", nullptr };
+  TEST_EQ(parser.Parse(schemafile.c_str(), include_directories), true);
+  TEST_EQ(parser.Parse(jsonfile.c_str(), include_directories), true);
 
   // here, parser.builder_ contains a binary buffer that is the parsed data.
 
@@ -406,12 +407,12 @@ void FuzzTest2() {
 
   // Parse the schema, parse the generated data, then generate text back
   // from the binary and compare against the original.
-  TEST_EQ(parser.Parse(schema.c_str(), ""), true);
+  TEST_EQ(parser.Parse(schema.c_str()), true);
 
   const std::string &json =
     definitions[num_definitions - 1].instances[0] + "\n";
 
-  TEST_EQ(parser.Parse(json.c_str(), ""), true);
+  TEST_EQ(parser.Parse(json.c_str()), true);
 
   std::string jsongen;
   flatbuffers::GeneratorOptions opts;
@@ -443,7 +444,7 @@ void FuzzTest2() {
 // Test that parser errors are actually generated.
 void TestError(const char *src, const char *error_substr) {
   flatbuffers::Parser parser;
-  TEST_EQ(parser.Parse(src, ""), false);  // Must signal error
+  TEST_EQ(parser.Parse(src), false);  // Must signal error
   // Must be the error we're expecting
   TEST_NOTNULL(strstr(parser.error_.c_str(), error_substr));
 }
@@ -498,10 +499,10 @@ void ScientificTest() {
   flatbuffers::Parser parser;
 
   // Simple schema.
-  TEST_EQ(parser.Parse("table X { Y:float; } root_type X;", ""), true);
+  TEST_EQ(parser.Parse("table X { Y:float; } root_type X;"), true);
 
   // Test scientific notation numbers.
-  TEST_EQ(parser.Parse("{ Y:0.0314159e+2 }", ""), true);
+  TEST_EQ(parser.Parse("{ Y:0.0314159e+2 }"), true);
   auto root = flatbuffers::GetRoot<float>(parser.builder_.GetBufferPointer());
   // root will point to the table, which is a 32bit vtable offset followed
   // by a float:
@@ -513,11 +514,11 @@ void EnumStringsTest() {
   flatbuffers::Parser parser1;
   TEST_EQ(parser1.Parse("enum E:byte { A, B, C } table T { F:[E]; }"
                         "root_type T;"
-                        "{ F:[ A, B, \"C\", \"A B C\" ] }", ""), true);
+                        "{ F:[ A, B, \"C\", \"A B C\" ] }"), true);
   flatbuffers::Parser parser2;
   TEST_EQ(parser2.Parse("enum E:byte { A, B, C } table T { F:[int]; }"
                         "root_type T;"
-                        "{ F:[ \"E.C\", \"E.A E.B E.C\" ] }", ""), true);
+                        "{ F:[ \"E.C\", \"E.A E.B E.C\" ] }"), true);
 }
 
 void UnicodeTest() {
@@ -525,7 +526,7 @@ void UnicodeTest() {
   TEST_EQ(parser.Parse("table T { F:string; }"
                        "root_type T;"
                        "{ F:\"\\u20AC\\u00A2\\u30E6\\u30FC\\u30B6\\u30FC"
-                       "\\u5225\\u30B5\\u30A4\\u30C8\\x01\\x80\" }", ""), true);
+                       "\\u5225\\u30B5\\u30A4\\u30C8\\x01\\x80\" }"), true);
   std::string jsongen;
   flatbuffers::GeneratorOptions opts;
   opts.indent_step = -1;