Update SchemaParser to use KJ filesystem API.

This required some hairy backwards-compatibility hacks as the parseDiskFile() method is widely used.

c++/src/capnp/schema-parser-test.c++

@@ -30,26 +30,22 @@
 namespace capnp {
 namespace {
 
-class FakeFileReader final: public SchemaFile::FileReader {
+class FakeFileReader final: public kj::Filesystem {
 public:
   void add(kj::StringPtr name, kj::StringPtr content) {
-    files[name] = content;
+    root->openFile(cwd.eval(name), kj::WriteMode::CREATE | kj::WriteMode::CREATE_PARENT)
+        ->writeAll(content);
   }
 
-  bool exists(kj::StringPtr path) const override {
-    return files.count(path) > 0;
-  }
-
-  kj::Array<const char> read(kj::StringPtr path) const override {
-    auto iter = files.find(path);
-    KJ_ASSERT(iter != files.end(), "FakeFileReader has no such file.", path);
-    auto result = kj::heapArray<char>(iter->second.size());
-    memcpy(result.begin(), iter->second.begin(), iter->second.size());
-    return kj::mv(result);
-  }
+  kj::Directory& getRoot() override { return *root; }
+  kj::Directory& getCurrent() override { return *current; }
+  kj::PathPtr getCurrentPath() override { return cwd; }
 
 private:
-  std::map<kj::StringPtr, kj::StringPtr> files;
+  kj::Own<kj::Directory> root = kj::newInMemoryDirectory(kj::nullClock());
+  kj::Path cwd = kj::Path({"path", "to", "current", "dir"});
+  kj::Own<kj::Directory> current = root->openSubdir(cwd,
+      kj::WriteMode::CREATE | kj::WriteMode::CREATE_PARENT);
 };
 
 static uint64_t getFieldTypeFileId(StructSchema::Field field) {
@@ -59,8 +55,9 @@ static uint64_t getFieldTypeFileId(StructSchema::Field field) {
 }
 
 TEST(SchemaParser, Basic) {
-  SchemaParser parser;
   FakeFileReader reader;
+  SchemaParser parser;
+  parser.setDiskFilesystem(reader);
 
   reader.add("src/foo/bar.capnp",
       "@0x8123456789abcdef;\n"
@@ -90,8 +87,8 @@ TEST(SchemaParser, Basic) {
     "/usr/include", "/usr/local/include", "/opt/include"
   };
 
-  ParsedSchema barSchema = parser.parseFile(SchemaFile::newDiskFile(
-      "foo2/bar2.capnp", "src/foo/bar.capnp", importPath, reader));
+  ParsedSchema barSchema = parser.parseDiskFile(
+      "foo2/bar2.capnp", "src/foo/bar.capnp", importPath);
 
   auto barProto = barSchema.getProto();
   EXPECT_EQ(0x8123456789abcdefull, barProto.getId());
@@ -109,25 +106,25 @@ TEST(SchemaParser, Basic) {
   EXPECT_EQ("garply", barFields[3].getProto().getName());
   EXPECT_EQ(0x856789abcdef1234ull, getFieldTypeFileId(barFields[3]));
 
-  auto bazSchema = parser.parseFile(SchemaFile::newDiskFile(
+  auto bazSchema = parser.parseDiskFile(
       "not/used/because/already/loaded",
-      "src/foo/baz.capnp", importPath, reader));
+      "src/foo/baz.capnp", importPath);
   EXPECT_EQ(0x823456789abcdef1ull, bazSchema.getProto().getId());
   EXPECT_EQ("foo2/baz.capnp", bazSchema.getProto().getDisplayName());
   auto bazStruct = bazSchema.getNested("Baz").asStruct();
   EXPECT_EQ(bazStruct, barStruct.getDependency(bazStruct.getProto().getId()));
 
-  auto corgeSchema = parser.parseFile(SchemaFile::newDiskFile(
+  auto corgeSchema = parser.parseDiskFile(
       "not/used/because/already/loaded",
-      "src/qux/corge.capnp", importPath, reader));
+      "src/qux/corge.capnp", importPath);
   EXPECT_EQ(0x83456789abcdef12ull, corgeSchema.getProto().getId());
   EXPECT_EQ("qux/corge.capnp", corgeSchema.getProto().getDisplayName());
   auto corgeStruct = corgeSchema.getNested("Corge").asStruct();
   EXPECT_EQ(corgeStruct, barStruct.getDependency(corgeStruct.getProto().getId()));
 
-  auto graultSchema = parser.parseFile(SchemaFile::newDiskFile(
+  auto graultSchema = parser.parseDiskFile(
       "not/used/because/already/loaded",
-      "/usr/include/grault.capnp", importPath, reader));
+      "/usr/include/grault.capnp", importPath);
   EXPECT_EQ(0x8456789abcdef123ull, graultSchema.getProto().getId());
   EXPECT_EQ("grault.capnp", graultSchema.getProto().getDisplayName());
   auto graultStruct = graultSchema.getNested("Grault").asStruct();
@@ -135,9 +132,9 @@ TEST(SchemaParser, Basic) {
 
   // Try importing the other grault.capnp directly.  It'll get the display name we specify since
   // it wasn't imported before.
-  auto wrongGraultSchema = parser.parseFile(SchemaFile::newDiskFile(
+  auto wrongGraultSchema = parser.parseDiskFile(
       "weird/display/name.capnp",
-      "/opt/include/grault.capnp", importPath, reader));
+      "/opt/include/grault.capnp", importPath);
   EXPECT_EQ(0x8000000000000001ull, wrongGraultSchema.getProto().getId());
   EXPECT_EQ("weird/display/name.capnp", wrongGraultSchema.getProto().getDisplayName());
 }
@@ -147,8 +144,9 @@ TEST(SchemaParser, Constants) {
   // constants are not actually accessible from the generated code API, so the only way to ever
   // get a ConstSchema is by parsing it.
 
-  SchemaParser parser;
   FakeFileReader reader;
+  SchemaParser parser;
+  parser.setDiskFilesystem(reader);
 
   reader.add("const.capnp",
       "@0x8123456789abcdef;\n"
@@ -164,8 +162,8 @@ TEST(SchemaParser, Constants) {
       "  value @0 :T;\n"
       "}\n");
 
-  ParsedSchema fileSchema = parser.parseFile(SchemaFile::newDiskFile(
-      "const.capnp", "const.capnp", nullptr, reader));
+  ParsedSchema fileSchema = parser.parseDiskFile(
+      "const.capnp", "const.capnp", nullptr);
 
   EXPECT_EQ(1234, fileSchema.getNested("uint32Const").asConst().as<uint32_t>());
 
@@ -198,8 +196,9 @@ void expectSourceInfo(schema::Node::SourceInfo::Reader sourceInfo,
 }
 
 TEST(SchemaParser, SourceInfo) {
-  SchemaParser parser;
   FakeFileReader reader;
+  SchemaParser parser;
+  parser.setDiskFilesystem(reader);
 
   reader.add("foo.capnp",
       "@0x84a2c6051e1061ed;\n"
@@ -234,8 +233,8 @@ TEST(SchemaParser, SourceInfo) {
       "struct Thud @0xcca9972702b730b4 {}\n"
       "# post-comment\n");
 
-  ParsedSchema file = parser.parseFile(SchemaFile::newDiskFile(
-      "foo.capnp", "foo.capnp", nullptr, reader));
+  ParsedSchema file = parser.parseDiskFile(
+      "foo.capnp", "foo.capnp", nullptr);
   ParsedSchema foo = file.getNested("Foo");
 
   expectSourceInfo(file.getSourceInfo(), 0x84a2c6051e1061edull, "file doc comment\n", {});

c++/src/capnp/schema-parser.h

@@ -28,6 +28,7 @@
 
 #include "schema-loader.h"
 #include <kj/string.h>
+#include <kj/filesystem.h>
 
 namespace capnp {
 
@@ -43,31 +44,85 @@ public:
   SchemaParser();
   ~SchemaParser() noexcept(false);
 
-  ParsedSchema parseDiskFile(kj::StringPtr displayName, kj::StringPtr diskPath,
-                             kj::ArrayPtr<const kj::StringPtr> importPath) const;
-  // Parse a file located on disk.  Throws an exception if the file dosen't exist.
-  //
-  // Parameters:
-  // * `displayName`:  The name that will appear in the file's schema node.  (If the file has
-  //   already been parsed, this will be ignored and the display name from the first time it was
-  //   parsed will be kept.)
-  // * `diskPath`:  The path to the file on disk.
-  // * `importPath`:  Directories to search when resolving absolute imports within this file
-  //   (imports that start with a `/`).  Must remain valid until the SchemaParser is destroyed.
-  //   (If the file has already been parsed, this will be ignored and the import path from the
-  //   first time it was parsed will be kept.)
+  ParsedSchema parseFromDirectory(kj::ReadableDirectory& baseDir, kj::Path path,
+                                  kj::ArrayPtr<kj::ReadableDirectory* const> importPath) const;
+  // Parse a file from the KJ filesystem API.  Throws an exception if the file dosen't exist.
+  //
+  // `baseDir` and `path` are used together to resolve relative imports. `path` is the source
+  // file's path within `baseDir`. Relative imports will be interpreted relative to `path` and
+  // will be opened using `baseDir`. Note that the KJ filesystem API prohibits "breaking out" of
+  // a directory using "..", so relative imports will be restricted to children of `baseDir`.
+  //
+  // `importPath` is used for absolute imports (imports that start with a '/'). Each directory in
+  // the array will be searched in order until a file is found.
+  //
+  // All `ReadableDirectory` objects must remain valid until the `SchemaParser` is destroyed. Also,
+  // the `importPath` array must remain valid. `path` will be copied; it need not remain valid.
   //
   // This method is a shortcut, equivalent to:
-  //     parser.parseFile(SchemaFile::newDiskFile(displayName, diskPath, importPath))`;
+  //     parser.parseFromDirectory(SchemaFile::newDiskFile(baseDir, path, importPath))`;
   //
   // This method throws an exception if any errors are encountered in the file or in anything the
   // file depends on.  Note that merely importing another file does not count as a dependency on
   // anything in the imported file -- only the imported types which are actually used are
   // "dependencies".
+  //
+  // Hint: Use kj::newDiskFilesystem() to initialize the KJ filesystem API. Usually you should do
+  //   this at a high level in your program, e.g. the main() function, and then pass down the
+  //   appropriate File/Directory objects to the components that need them. Example:
+  //
+  //     auto fs = kj::newDiskFilesystem();
+  //     SchemaParser parser;
+  //     auto schema = parser->parseFromDirectory(fs->getCurrent(),
+  //         kj::Path::parse("foo/bar.capnp"), nullptr);
+  //
+  // Hint: To use in-memory data rather than real disk, you can use kj::newInMemoryDirectory(),
+  //   write the files you want, then pass it to SchemaParser. Example:
+  //
+  //     auto dir = kj::newInMemoryDirectory(kj::nullClock());
+  //     auto path = kj::Path::parse("foo/bar.capnp");
+  //     dir->openFile(path, kj::WriteMode::CREATE | kj::WriteMode::CREATE_PARENT)
+  //        ->writeAll("struct Foo {}");
+  //     auto schema = parser->parseFromDirectory(*dir, path, nullptr);
+  //
+  // Hint: You can create an in-memory directory but then populate it with real files from disk,
+  //   in order to control what is visible while also avoiding reading files yourself or making
+  //   extra copies. Example:
+  //
+  //     auto fs = kj::newDiskFilesystem();
+  //     auto dir = kj::newInMemoryDirectory(kj::nullClock());
+  //     auto fakePath = kj::Path::parse("foo/bar.capnp");
+  //     auto realPath = kj::Path::parse("path/to/some/file.capnp");
+  //     dir->transfer(fakePath, kj::WriteMode::CREATE | kj::WriteMode::CREATE_PARENT,
+  //                   fs->getCurrent(), realPath, kj::TransferMode::LINK);
+  //     auto schema = parser->parseFromDirectory(*dir, fakePath, nullptr);
+  //
+  //   In this example, note that any imports in the file will fail, since the in-memory directory
+  //   you created contains no files except the specific one you linked in.
+
+  ParsedSchema parseDiskFile(kj::StringPtr displayName, kj::StringPtr diskPath,
+                             kj::ArrayPtr<const kj::StringPtr> importPath) const
+      CAPNP_DEPRECATED("Use parseFromDirectory() instead.");
+  // Creates a private kj::Filesystem and uses it to parse files from the real disk.
+  //
+  // DO NOT USE in new code. Use parseFromDirectory() instead.
+  //
+  // This API has a serious problem: the file can import and embed files located anywhere on disk
+  // using relative paths. Even if you specify no `importPath`, relative imports still work. By
+  // using `parseFromDirectory()`, you can arrange so that imports are only allowed within a
+  // particular directory, or even set up a dummy filesystem where other files are not visible.
+
+  void setDiskFilesystem(kj::Filesystem& fs)
+      CAPNP_DEPRECATED("Use parseFromDirectory() instead.");
+  // Call before calling parseDiskFile() to choose an alternative disk filesystem implementation.
+  // This exists mostly for testing purposes; new code should use parseFromDirectory() instead.
+  //
+  // If parseDiskFile() is called without having called setDiskFilesystem(), then
+  // kj::newDiskFilesystem() will be used instead.
 
   ParsedSchema parseFile(kj::Own<SchemaFile>&& file) const;
   // Advanced interface for parsing a file that may or may not be located in any global namespace.
-  // Most users will prefer `parseDiskFile()`.
+  // Most users will prefer `parseFromDirectory()`.
   //
   // If the file has already been parsed (that is, a SchemaFile that compares equal to this one
   // was parsed previously), the existing schema will be returned again.
@@ -90,6 +145,7 @@ public:
 
 private:
   struct Impl;
+  struct DiskFileCompat;
   class ModuleImpl;
   kj::Own<Impl> impl;
   mutable bool hadErrors = false;
@@ -135,44 +191,20 @@ class SchemaFile {
   // `SchemaFile::newDiskFile()`.
 
 public:
-  class FileReader {
-  public:
-    virtual bool exists(kj::StringPtr path) const = 0;
-    virtual kj::Array<const char> read(kj::StringPtr path) const = 0;
-  };
-
-  class DiskFileReader final: public FileReader {
-    // Implementation of FileReader that uses the local disk.  Files are read using mmap() if
-    // possible.
-
-  public:
-    static const DiskFileReader instance;
-
-    bool exists(kj::StringPtr path) const override;
-    kj::Array<const char> read(kj::StringPtr path) const override;
-  };
-
-  static kj::Own<SchemaFile> newDiskFile(
-      kj::StringPtr displayName, kj::StringPtr diskPath,
-      kj::ArrayPtr<const kj::StringPtr> importPath,
-      const FileReader& fileReader = DiskFileReader::instance);
-  // Construct a SchemaFile representing a file on disk (or located in the filesystem-like
-  // namespace represented by `fileReader`).
-  //
-  // Parameters:
-  // * `displayName`:  The name that will appear in the file's schema node.
-  // * `diskPath`:  The path to the file on disk.
-  // * `importPath`:  Directories to search when resolving absolute imports within this file
-  //   (imports that start with a `/`).  The array content must remain valid as long as the
-  //   SchemaFile exists (which is at least as long as the SchemaParser that parses it exists).
-  // * `fileReader`:  Allows you to use a filesystem other than the actual local disk.  Although,
-  //   if you find yourself using this, it may make more sense for you to implement SchemaFile
-  //   yourself.
-  //
-  // The SchemaFile compares equal to any other SchemaFile that has exactly the same disk path,
-  // after canonicalization.
-  //
-  // The SchemaFile will throw an exception if any errors are reported.
+  // Note: Cap'n Proto 0.6.x and below had classes FileReader and DiskFileReader and a method
+  //   newDiskFile() defined here. These were removed when SchemaParser was transitioned to use the
+  //   KJ filesystem API. You should be able to get the same effect by subclassing
+  //   kj::ReadableDirectory, or using kj::newInMemoryDirectory().
+
+  static kj::Own<SchemaFile> newFromDirectory(
+      kj::ReadableDirectory& baseDir, kj::Path path,
+      kj::ArrayPtr<kj::ReadableDirectory* const> importPath,
+      kj::Maybe<kj::String> displayNameOverride = nullptr);
+  // Construct a SchemaFile representing a file in a kj::ReadableDirectory. This is used to
+  // implement SchemaParser::parseFromDirectory(); see there for details.
+  //
+  // The SchemaFile compares equal to any other SchemaFile that has exactly the same `baseDir`
+  // object (by identity) and `path` (by value).
 
   // -----------------------------------------------------------------
   // For more control, you can implement this interface.