Add options to print build rule dependencies

Tested: on Linux

Bug: 16465909
Change-Id: I2f1a6def13e47716110426b00990c2c625c03251

include/flatbuffers/idl.h

@@ -19,6 +19,7 @@
 
 #include <map>
 #include <set>
+#include <stack>
 #include <memory>
 #include <functional>
 
@@ -179,6 +180,7 @@ struct Definition {
   Definition() : generated(false), defined_namespace(nullptr) {}
 
   std::string name;
+  std::string file;
   std::vector<std::string> doc_comment;
   SymbolTable<Value> attributes;
   bool generated;  // did we already output code for this definition?
@@ -309,6 +311,11 @@ class Parser {
   // Mark all definitions as already having code generated.
   void MarkGenerated();
 
+  // Get the files recursively included by the given file. The returned
+  // container will have at least the given file.
+  std::set<std::string> GetIncludedFilesRecursive(
+      const std::string &file_name) const;
+
  private:
   int64_t ParseHexNum(int nibbles);
   void Next();
@@ -349,11 +356,13 @@ class Parser {
   std::string file_extension_;
 
   std::map<std::string, bool> included_files_;
+  std::map<std::string, std::set<std::string>> files_included_per_file_;
 
  private:
   const char *source_, *cursor_;
   int line_;  // the current line being parsed
   int token_;
+  std::stack<std::string> files_being_parsed_;
   bool proto_mode_;
   bool strict_json_;
   std::string attribute_;
@@ -381,7 +390,7 @@ struct GeneratorOptions {
   bool include_dependence_headers;
 
   // Possible options for the more general generator below.
-  enum Language { kJava, kCSharp, kMAX };
+  enum Language { kJava, kCSharp, kGo, kMAX };
 
   Language lang;
 
@@ -401,6 +410,18 @@ extern void GenerateText(const Parser &parser,
                          const void *flatbuffer,
                          const GeneratorOptions &opts,
                          std::string *text);
+extern bool GenerateTextFile(const Parser &parser,
+                             const std::string &path,
+                             const std::string &file_name,
+                             const GeneratorOptions &opts);
+
+// Generate binary files from a given FlatBuffer, and a given Parser
+// object that has been populated with the corresponding schema.
+// See idl_gen_general.cpp.
+extern bool GenerateBinary(const Parser &parser,
+                           const std::string &path,
+                           const std::string &file_name,
+                           const GeneratorOptions &opts);
 
 // Generate a C++ header from the definitions in the Parser object.
 // See idl_gen_cpp.
@@ -450,6 +471,34 @@ extern bool GenerateFBS(const Parser &parser,
                         const std::string &file_name,
                         const GeneratorOptions &opts);
 
+// Generate a make rule for the generated C++ header.
+// See idl_gen_cpp.cpp.
+extern std::string CPPMakeRule(const Parser &parser,
+                               const std::string &path,
+                               const std::string &file_name,
+                               const GeneratorOptions &opts);
+
+// Generate a make rule for the generated Java/C#/... files.
+// See idl_gen_general.cpp.
+extern std::string GeneralMakeRule(const Parser &parser,
+                                   const std::string &path,
+                                   const std::string &file_name,
+                                   const GeneratorOptions &opts);
+
+// Generate a make rule for the generated text (JSON) files.
+// See idl_gen_text.cpp.
+extern std::string TextMakeRule(const Parser &parser,
+                                const std::string &path,
+                                const std::string &file_name,
+                                const GeneratorOptions &opts);
+
+// Generate a make rule for the generated binary files.
+// See idl_gen_general.cpp.
+extern std::string BinaryMakeRule(const Parser &parser,
+                                  const std::string &path,
+                                  const std::string &file_name,
+                                  const GeneratorOptions &opts);
+
 }  // namespace flatbuffers
 
 #endif  // FLATBUFFERS_IDL_H_

include/flatbuffers/util.h

@@ -80,6 +80,12 @@ inline int64_t StringToInt(const char *str, int base = 10) {
   #endif
 }
 
+// Check if file "name" exists.
+inline bool FileExists(const char *name) {
+  std::ifstream ifs(name);
+  return ifs.good();
+}
+
 // Load file "name" into "buf" returning true if successful
 // false otherwise.  If "binary" is false data is read
 // using ifstream's text mode, otherwise data is read with
@@ -234,6 +240,33 @@ inline int FromUTF8(const char **in) {
   return ucc;
 }
 
+// Wraps a string to a maximum length, inserting new lines where necessary. Any
+// existing whitespace will be collapsed down to a single space. A prefix or
+// suffix can be provided, which will be inserted before or after a wrapped
+// line, respectively.
+inline std::string WordWrap(const std::string in, size_t max_length,
+                            const std::string wrapped_line_prefix,
+                            const std::string wrapped_line_suffix) {
+  std::istringstream in_stream(in);
+  std::string wrapped, line, word;
+
+  in_stream >> word;
+  line = word;
+
+  while (in_stream >> word) {
+    if ((line.length() + 1 + word.length() + wrapped_line_suffix.length()) <
+        max_length) {
+      line += " " + word;
+    } else {
+      wrapped += line + wrapped_line_suffix + "\n";
+      line = wrapped_line_prefix + word;
+    }
+  }
+  wrapped += line;
+
+  return wrapped;
+}
+
 }  // namespace flatbuffers
 
 #endif  // FLATBUFFERS_UTIL_H_

src/flatc.cpp

@@ -21,38 +21,6 @@
 static void Error(const char *err, const char *obj = nullptr,
                   bool usage = false, bool show_exe_name = true);
 
-namespace flatbuffers {
-
-bool GenerateBinary(const Parser &parser,
-                    const std::string &path,
-                    const std::string &file_name,
-                    const GeneratorOptions & /*opts*/) {
-  auto ext = parser.file_extension_.length() ? parser.file_extension_ : "bin";
-  return !parser.builder_.GetSize() ||
-         flatbuffers::SaveFile(
-           (path + file_name + "." + ext).c_str(),
-           reinterpret_cast<char *>(parser.builder_.GetBufferPointer()),
-           parser.builder_.GetSize(),
-           true);
-}
-
-bool GenerateTextFile(const Parser &parser,
-                      const std::string &path,
-                      const std::string &file_name,
-                      const GeneratorOptions &opts) {
-  if (!parser.builder_.GetSize()) return true;
-  if (!parser.root_struct_def) Error("root_type not set");
-  std::string text;
-  GenerateText(parser, parser.builder_.GetBufferPointer(), opts,
-               &text);
-  return flatbuffers::SaveFile((path + file_name + ".json").c_str(),
-                               text,
-                               false);
-
-}
-
-}
-
 // This struct allows us to create a table of all possible output generators
 // for the various programming languages and formats we support.
 struct Generator {
@@ -60,31 +28,42 @@ struct Generator {
                    const std::string &path,
                    const std::string &file_name,
                    const flatbuffers::GeneratorOptions &opts);
-  const char *opt;
-  const char *name;
+  const char *generator_opt;
+  const char *lang_name;
   flatbuffers::GeneratorOptions::Language lang;
-  const char *help;
+  const char *generator_help;
+
+  std::string (*make_rule)(const flatbuffers::Parser &parser,
+                           const std::string &path,
+                           const std::string &file_name,
+                           const flatbuffers::GeneratorOptions &opts);
 };
 
 const Generator generators[] = {
   { flatbuffers::GenerateBinary,   "-b", "binary",
     flatbuffers::GeneratorOptions::kMAX,
-    "Generate wire format binaries for any data definitions" },
+    "Generate wire format binaries for any data definitions",
+    flatbuffers::BinaryMakeRule },
   { flatbuffers::GenerateTextFile, "-t", "text",
     flatbuffers::GeneratorOptions::kMAX,
-    "Generate text output for any data definitions" },
+    "Generate text output for any data definitions",
+    flatbuffers::TextMakeRule },
   { flatbuffers::GenerateCPP,      "-c", "C++",
     flatbuffers::GeneratorOptions::kMAX,
-    "Generate C++ headers for tables/structs" },
+    "Generate C++ headers for tables/structs",
+    flatbuffers::CPPMakeRule },
   { flatbuffers::GenerateGo,       "-g", "Go",
-    flatbuffers::GeneratorOptions::kMAX,
-    "Generate Go files for tables/structs" },
+    flatbuffers::GeneratorOptions::kGo,
+    "Generate Go files for tables/structs",
+    flatbuffers::GeneralMakeRule },
   { flatbuffers::GenerateGeneral,  "-j", "Java",
     flatbuffers::GeneratorOptions::kJava,
-    "Generate Java classes for tables/structs" },
+    "Generate Java classes for tables/structs",
+    flatbuffers::GeneralMakeRule },
   { flatbuffers::GenerateGeneral,  "-n", "C#",
     flatbuffers::GeneratorOptions::kCSharp,
-    "Generate C# classes for tables/structs" }
+    "Generate C# classes for tables/structs",
+    flatbuffers::GeneralMakeRule },
 };
 
 const char *program_name = NULL;
@@ -98,10 +77,13 @@ static void Error(const char *err, const char *obj, bool usage,
   if (usage) {
     printf("usage: %s [OPTION]... FILE... [-- FILE...]\n", program_name);
     for (size_t i = 0; i < sizeof(generators) / sizeof(generators[0]); ++i)
-      printf("  %s             %s.\n", generators[i].opt, generators[i].help);
+      printf("  %s              %s.\n",
+             generators[i].generator_opt,
+             generators[i].generator_help);
     printf(
       "  -o PATH         Prefix PATH to all generated files.\n"
       "  -I PATH         Search for includes in the specified path.\n"
+      "  -M              Print make rules for generated files.\n"
       "  --strict-json   Strict JSON: field names must be / will be quoted,\n"
       "                  no trailing commas in tables/vectors.\n"
       "  --no-prefix     Don\'t prefix enum values with the enum type in C++.\n"
@@ -110,7 +92,7 @@ static void Error(const char *err, const char *obj, bool usage,
       "  --proto         Input is a .proto, translate to .fbs.\n"
       "FILEs may depend on declarations in earlier files.\n"
       "FILEs after the -- must be binary flatbuffer format files.\n"
-      "Output files are named using the base file name of the input,"
+      "Output files are named using the base file name of the input,\n"
       "and written to the current directory or the path given by -o.\n"
       "example: %s -c -b schema1.fbs schema2.fbs data.json\n",
       program_name);
@@ -125,6 +107,7 @@ int main(int argc, const char *argv[]) {
   const size_t num_generators = sizeof(generators) / sizeof(generators[0]);
   bool generator_enabled[num_generators] = { false };
   bool any_generator = false;
+  bool print_make_rules = false;
   bool proto_mode = false;
   std::vector<std::string> filenames;
   std::vector<const char *> include_directories;
@@ -152,9 +135,11 @@ int main(int argc, const char *argv[]) {
       } else if(opt == "--proto") {
         proto_mode = true;
         any_generator = true;
+      } else if(opt == "-M") {
+        print_make_rules = true;
       } else {
         for (size_t i = 0; i < num_generators; ++i) {
-          if(opt == generators[i].opt) {
+          if (opt == generators[i].generator_opt) {
             generator_enabled[i] = true;
             any_generator = true;
             goto found;
@@ -171,8 +156,7 @@ int main(int argc, const char *argv[]) {
   if (!filenames.size()) Error("missing input files", nullptr, true);
 
   if (!any_generator)
-    Error("no options: no output files generated.",
-          "specify one of -c -g -j -t -b etc.", true);
+    Error("no options", "specify one of -c -g -j -t -b etc.", true);
 
   // Now process the files:
   flatbuffers::Parser parser(opts.strict_json, proto_mode);
@@ -205,15 +189,23 @@ int main(int argc, const char *argv[]) {
                                flatbuffers::StripExtension(*file_it));
 
       for (size_t i = 0; i < num_generators; ++i) {
+        opts.lang = generators[i].lang;
         if (generator_enabled[i]) {
+          if (!print_make_rules) {
             flatbuffers::EnsureDirExists(output_path);
-          opts.lang = generators[i].lang;
             if (!generators[i].generate(parser, output_path, filebase, opts)) {
               Error((std::string("Unable to generate ") +
-                   generators[i].name +
+                     generators[i].lang_name +
                      " for " +
                      filebase).c_str());
             }
+          } else {
+            std::string make_rule = generators[i].make_rule(
+                parser, output_path, *file_it, opts);
+            if (!make_rule.empty())
+              printf("%s\n", flatbuffers::WordWrap(
+                  make_rule, 80, " ", " \\").c_str());
+          }
         }
       }
 

src/idl_gen_cpp.cpp

@@ -696,13 +696,33 @@ std::string GenerateCPP(const Parser &parser,
   return std::string();
 }
 
+static std::string GeneratedFileName(const std::string &path,
+                                     const std::string &file_name) {
+  return path + file_name + "_generated.h";
+}
+
 bool GenerateCPP(const Parser &parser,
                  const std::string &path,
                  const std::string &file_name,
                  const GeneratorOptions &opts) {
     auto code = GenerateCPP(parser, file_name, opts);
     return !code.length() ||
-           SaveFile((path + file_name + "_generated.h").c_str(), code, false);
+           SaveFile(GeneratedFileName(path, file_name).c_str(), code, false);
+}
+
+std::string CPPMakeRule(const Parser &parser,
+                        const std::string &path,
+                        const std::string &file_name,
+                        const GeneratorOptions & /*opts*/) {
+  std::string filebase = flatbuffers::StripPath(
+      flatbuffers::StripExtension(file_name));
+  std::string make_rule = GeneratedFileName(path, filebase) + ": ";
+  auto included_files = parser.GetIncludedFilesRecursive(file_name);
+  for (auto it = included_files.begin();
+       it != included_files.end(); ++it) {
+    make_rule += " " + *it;
+  }
+  return make_rule;
 }
 
 }  // namespace flatbuffers

src/idl_gen_general.cpp

@@ -98,6 +98,23 @@ LanguageParameters language_parameters[] = {
     "\n}\n",
     "",
     "using FlatBuffers;\n\n",
+  },
+  // TODO: add Go support to the general generator.
+  // WARNING: this is currently only used for generating make rules for Go.
+  {
+    GeneratorOptions::kGo,
+    true,
+    ".go",
+    "string",
+    "bool ",
+    "\n{\n",
+    "const ",
+    "",
+    "package ",
+    "",
+    "",
+    "",
+    "import (\n\tflatbuffers \"github.com/google/flatbuffers/go\"\n)",
   }
 };
 
@@ -116,10 +133,11 @@ static std::string GenTypeBasic(const LanguageParameters &lang,
                                 const Type &type) {
   static const char *gtypename[] = {
     #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, JTYPE, GTYPE, NTYPE) \
-        #JTYPE, #NTYPE,
+        #JTYPE, #NTYPE, #GTYPE,
       FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD)
     #undef FLATBUFFERS_TD
   };
+
   return gtypename[type.base_type * GeneratorOptions::kMAX + lang.language];
 }
 
@@ -653,4 +671,91 @@ bool GenerateGeneral(const Parser &parser,
   return true;
 }
 
+static std::string ClassFileName(const LanguageParameters &lang,
+                                 const Parser &parser, const Definition &def,
+                                 const std::string &path) {
+  std::string namespace_general;
+  std::string namespace_dir = path;
+  auto &namespaces = parser.namespaces_.back()->components;
+  for (auto it = namespaces.begin(); it != namespaces.end(); ++it) {
+    if (namespace_general.length()) {
+      namespace_general += ".";
+      namespace_dir += kPathSeparator;
+    }
+    namespace_general += *it;
+    namespace_dir += *it;
+  }
+
+  return namespace_dir + kPathSeparator + def.name + lang.file_extension;
+}
+
+std::string GeneralMakeRule(const Parser &parser,
+                            const std::string &path,
+                            const std::string &file_name,
+                            const GeneratorOptions &opts) {
+  assert(opts.lang <= GeneratorOptions::kMAX);
+  auto lang = language_parameters[opts.lang];
+
+  std::string make_rule;
+
+  for (auto it = parser.enums_.vec.begin();
+       it != parser.enums_.vec.end(); ++it) {
+    if (make_rule != "")
+      make_rule += " ";
+    make_rule += ClassFileName(lang, parser, **it, path);
+  }
+
+  for (auto it = parser.structs_.vec.begin();
+       it != parser.structs_.vec.end(); ++it) {
+    if (make_rule != "")
+      make_rule += " ";
+    make_rule += ClassFileName(lang, parser, **it, path);
+  }
+
+  make_rule += ": ";
+  auto included_files = parser.GetIncludedFilesRecursive(file_name);
+  for (auto it = included_files.begin();
+       it != included_files.end(); ++it) {
+    make_rule += " " + *it;
+  }
+  return make_rule;
+}
+
+std::string BinaryFileName(const Parser &parser,
+                           const std::string &path,
+                           const std::string &file_name) {
+  auto ext = parser.file_extension_.length() ? parser.file_extension_ : "bin";
+  return path + file_name + "." + ext;
+}
+
+bool GenerateBinary(const Parser &parser,
+                    const std::string &path,
+                    const std::string &file_name,
+                    const GeneratorOptions & /*opts*/) {
+  return !parser.builder_.GetSize() ||
+         flatbuffers::SaveFile(
+           BinaryFileName(parser, path, file_name).c_str(),
+           reinterpret_cast<char *>(parser.builder_.GetBufferPointer()),
+           parser.builder_.GetSize(),
+           true);
+}
+
+std::string BinaryMakeRule(const Parser &parser,
+                           const std::string &path,
+                           const std::string &file_name,
+                           const GeneratorOptions & /*opts*/) {
+  if (!parser.builder_.GetSize()) return "";
+  std::string filebase = flatbuffers::StripPath(
+      flatbuffers::StripExtension(file_name));
+  std::string make_rule = BinaryFileName(parser, path, filebase) + ": " +
+      file_name;
+  auto included_files = parser.GetIncludedFilesRecursive(
+      parser.root_struct_def->file);
+  for (auto it = included_files.begin();
+       it != included_files.end(); ++it) {
+    make_rule += " " + *it;
+  }
+  return make_rule;
+}
+
 }  // namespace flatbuffers

src/idl_gen_text.cpp

@@ -268,5 +268,40 @@ void GenerateText(const Parser &parser, const void *flatbuffer,
   text += NewLine(opts);
 }
 
+std::string TextFileName(const std::string &path,
+                         const std::string &file_name) {
+  return path + file_name + ".json";
+}
+
+bool GenerateTextFile(const Parser &parser,
+                      const std::string &path,
+                      const std::string &file_name,
+                      const GeneratorOptions &opts) {
+  if (!parser.builder_.GetSize() || !parser.root_struct_def) return true;
+  std::string text;
+  GenerateText(parser, parser.builder_.GetBufferPointer(), opts,
+               &text);
+  return flatbuffers::SaveFile(TextFileName(path, file_name).c_str(),
+                               text,
+                               false);
+}
+
+std::string TextMakeRule(const Parser &parser,
+                         const std::string &path,
+                         const std::string &file_name,
+                         const GeneratorOptions & /*opts*/) {
+  if (!parser.builder_.GetSize() || !parser.root_struct_def) return "";
+  std::string filebase = flatbuffers::StripPath(
+      flatbuffers::StripExtension(file_name));
+  std::string make_rule = TextFileName(path, filebase) + ": " + file_name;
+  auto included_files = parser.GetIncludedFilesRecursive(
+      parser.root_struct_def->file);
+  for (auto it = included_files.begin();
+       it != included_files.end(); ++it) {
+    make_rule += " " + *it;
+  }
+  return make_rule;
+}
+
 }  // namespace flatbuffers
 

src/idl_parser.cpp

@@ -15,6 +15,7 @@
  */
 
 #include <algorithm>
+#include <list>
 
 #include "flatbuffers/flatbuffers.h"
 #include "flatbuffers/idl.h"
@@ -333,6 +334,7 @@ FieldDef &Parser::AddField(StructDef &struct_def,
   field.value.offset =
     FieldIndexToOffset(static_cast<voffset_t>(struct_def.fields.vec.size()));
   field.name = name;
+  field.file = struct_def.file;
   field.value.type = type;
   if (struct_def.fixed) {  // statically compute the field offset
     auto size = InlineSize(type);
@@ -732,6 +734,7 @@ void Parser::ParseEnum(bool is_union) {
   Expect(kTokenIdentifier);
   auto &enum_def = *new EnumDef();
   enum_def.name = name;
+  if (!files_being_parsed_.empty()) enum_def.file = files_being_parsed_.top();
   enum_def.doc_comment = dc;
   enum_def.is_union = is_union;
   enum_def.defined_namespace = namespaces_.back();
@@ -799,6 +802,7 @@ StructDef &Parser::StartStruct() {
   if (!struct_def.predecl) Error("datatype already exists: " + name);
   struct_def.predecl = false;
   struct_def.name = name;
+  if (!files_being_parsed_.empty()) struct_def.file = files_being_parsed_.top();
   // Move this struct to the back of the vector just in case it was predeclared,
   // to preserve declaration order.
   remove(structs_.vec.begin(), structs_.vec.end(), &struct_def);
@@ -1022,7 +1026,16 @@ Type Parser::ParseTypeFromProtoType() {
 
 bool Parser::Parse(const char *source, const char **include_paths,
                    const char *source_filename) {
-  if (source_filename) included_files_[source_filename] = true;
+  if (source_filename &&
+      included_files_.find(source_filename) == included_files_.end()) {
+    included_files_[source_filename] = true;
+    files_included_per_file_[source_filename] = std::set<std::string>();
+    files_being_parsed_.push(source_filename);
+  }
+  if (!include_paths) {
+    const char *current_directory[] = { "", nullptr };
+    include_paths = current_directory;
+  }
   source_ = cursor_ = source;
   line_ = 1;
   error_.clear();
@@ -1033,22 +1046,23 @@ bool Parser::Parse(const char *source, const char **include_paths,
     while (IsNext(kTokenInclude)) {
       auto name = attribute_;
       Expect(kTokenStringConstant);
-      if (included_files_.find(name) == included_files_.end()) {
+      // Look for the file in include_paths.
+      std::string filepath;
+      for (auto paths = include_paths; paths && *paths; paths++) {
+        filepath = flatbuffers::ConCatPathFileName(*paths, name);
+        if(FileExists(filepath.c_str())) break;
+      }
+      if (filepath.empty())
+        Error("unable to locate include file: " + name);
+      if (source_filename)
+        files_included_per_file_[source_filename].insert(filepath);
+      if (included_files_.find(filepath) == included_files_.end()) {
         // We found an include file that we have not parsed yet.
         // Load it and parse it.
         std::string contents;
-        if (!include_paths) {
-          const char *current_directory[] = { "", nullptr };
-          include_paths = current_directory;
-        }
-        for (auto paths = include_paths; paths && *paths; paths++) {
-          auto filepath = flatbuffers::ConCatPathFileName(*paths, name);
-          if(LoadFile(filepath.c_str(), true, &contents)) break;
-        }
-        if (contents.empty())
+        if (!LoadFile(filepath.c_str(), true, &contents))
           Error("unable to load include file: " + name);
-        included_files_[name] = true;
-        if (!Parse(contents.c_str(), include_paths)) {
+        if (!Parse(contents.c_str(), include_paths, filepath.c_str())) {
           // Any errors, we're done.
           return false;
         }
@@ -1143,10 +1157,35 @@ bool Parser::Parse(const char *source, const char **include_paths,
       error_ += NumToString(line_) + ":0";  // gcc alike
     #endif
     error_ += ": error: " + msg;
+    if (source_filename) files_being_parsed_.pop();
     return false;
   }
+  if (source_filename) files_being_parsed_.pop();
   assert(!struct_stack_.size());
   return true;
 }
 
+std::set<std::string> Parser::GetIncludedFilesRecursive(
+    const std::string &file_name) const {
+  std::set<std::string> included_files;
+  std::list<std::string> to_process;
+
+  if (file_name.empty()) return included_files;
+  to_process.push_back(file_name);
+
+  while (!to_process.empty()) {
+    std::string current = to_process.front();
+    to_process.pop_front();
+    included_files.insert(current);
+
+    auto new_files = files_included_per_file_.at(current);
+    for (auto it = new_files.begin(); it != new_files.end(); ++it) {
+      if (included_files.find(*it) == included_files.end())
+        to_process.push_back(*it);
+    }
+  }
+
+  return included_files;
+}
+
 }  // namespace flatbuffers