Merge pull request #605 from capnproto/compiler-use-filesystem-api

Change capnp compiler to use KJ filesystem API.

c++/src/capnp/compiler/capnp-test.sh

@@ -40,6 +40,13 @@ fi
 SCHEMA=`dirname "$0"`/../test.capnp
 TESTDATA=`dirname "$0"`/../testdata
 
+SUFFIX=${TESTDATA#*/src/}
+PREFIX=${TESTDATA%${SUFFIX}}
+
+if [ "$PREFIX" = "" ]; then
+  PREFIX=.
+fi
+
 # ========================================================================================
 # convert
 
@@ -102,5 +109,5 @@ test_eval 'TestListDefaults.lists.int32ListList[2][0]' 12341234
 
 test "x`$CAPNP eval $SCHEMA -ojson globalPrintableStruct | tr -d '\r'`" = "x{\"someText\": \"foo\"}" || fail eval json "globalPrintableStruct == {someText = \"foo\"}"
 
-$CAPNP compile -ofoo $TESTDATA/errors.capnp.nobuild 2>&1 | sed -e "s,^.*/errors[.]capnp[.]nobuild,file,g" | tr -d '\r' |
+$CAPNP compile --src-prefix="$PREFIX" -ofoo $TESTDATA/errors.capnp.nobuild 2>&1 | sed -e "s,^.*errors[.]capnp[.]nobuild:,file:,g" | tr -d '\r' |
     cmp $TESTDATA/errors.txt - || fail error output

c++/src/capnp/compiler/capnpc-c++.c++

@@ -28,19 +28,25 @@
 #include <kj/string-tree.h>
 #include <kj/tuple.h>
 #include <kj/vector.h>
+#include <kj/filesystem.h>
 #include "../schema-loader.h"
 #include "../dynamic.h"
-#include <kj/miniposix.h>
 #include <unordered_map>
 #include <unordered_set>
 #include <map>
 #include <set>
 #include <kj/main.h>
 #include <algorithm>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <errno.h>
+
+#if _WIN32
+#define WIN32_LEAN_AND_MEAN  // ::eyeroll::
+#include <windows.h>
+#include <kj/windows-sanity.h>
+#undef VOID
+#undef CONST
+#else
+#include <sys/time.h>
+#endif
 
 #if HAVE_CONFIG_H
 #include "config.h"
@@ -2995,42 +3001,67 @@ private:
 
   // -----------------------------------------------------------------
 
-  void makeDirectory(kj::StringPtr path) {
-    KJ_IF_MAYBE(slashpos, path.findLast('/')) {
-      // Make the parent dir.
-      makeDirectory(kj::str(path.slice(0, *slashpos)));
-    }
-
-    if (kj::miniposix::mkdir(path.cStr(), 0777) < 0) {
-      int error = errno;
-      if (error != EEXIST) {
-        KJ_FAIL_SYSCALL("mkdir(path)", error, path);
-      }
-    }
-  }
+  kj::Own<kj::Filesystem> fs = kj::newDiskFilesystem();
 
   void writeFile(kj::StringPtr filename, const kj::StringTree& text) {
-    if (!filename.startsWith("/")) {
-      KJ_IF_MAYBE(slashpos, filename.findLast('/')) {
-        // Make the parent dir.
-        makeDirectory(kj::str(filename.slice(0, *slashpos)));
+    // We don't use replaceFile() here because atomic replacements are actually detrimental for
+    // build tools:
+    // - It's the responsibility of the build pipeline to ensure that no one else is concurrently
+    //   reading the file when we write it, so atomicity brings no benefit.
+    // - Atomic replacements force disk syncs which could slow us down for no benefit at all.
+    // - Opening the existing file and overwriting it may allow the filesystem to reuse
+    //   already-allocated blocks, or maybe even notice that no actual changes occurred.
+    // - In a power outage scenario, the user would obviously restart the build from scratch
+    //   anyway.
+    //
+    // We do, however, use mmap(), allowing us to write directly to page cache, avoiding a copy,
+    // and even allowing us to avoid dirtying the file pages if they're not modified, which will
+    // commonly be the case with incremental builds.
+    //
+    // Yes, I overengineered this a bit... but it was fun.
+
+    auto path = kj::Path::parse(filename);
+    auto file = fs->getCurrent().openFile(path,
+        kj::WriteMode::CREATE | kj::WriteMode::MODIFY | kj::WriteMode::CREATE_PARENT);
+    file->truncate(text.size());
+    auto mapping = file->mmapWritable(0, text.size());
+    auto bytes = mapping->get();
+
+    byte* target = bytes.begin();
+    byte* firstModified = nullptr;
+    text.visit([&](kj::ArrayPtr<const char> text) {
+      if (firstModified == nullptr) {
+        if (memcmp(target, text.begin(), text.size()) == 0) {
+          target += text.size();
+          return;
+        }
+        firstModified = target;
       }
+      memcpy(target, text.begin(), text.size());
+      target += text.size();
+    });
+    KJ_ASSERT(target == bytes.end());
+    if (firstModified == nullptr) {
+      // The file is completely unchanged. But we should probably update the modification time
+      // anyway so that build systems don't get confused.
+      //
+      // TODO(cleanup): Add touch() to kj::FsNode.
+#if _WIN32
+      FILETIME time;
+      GetSystemTimeAsFileTime(&time);
+      KJ_WIN32(SetFileTime(KJ_ASSERT_NONNULL(file->getWin32Handle()), NULL, &time, &time));
+#else
+      KJ_SYSCALL(futimes(KJ_ASSERT_NONNULL(file->getFd()), nullptr));
+#endif
+    } else {
+      mapping->changed(kj::arrayPtr(firstModified, bytes.end()));
     }
-
-    int fd;
-    KJ_SYSCALL(fd = open(filename.cStr(), O_CREAT | O_WRONLY | O_TRUNC, 0666), filename);
-    kj::FdOutputStream out((kj::AutoCloseFd(fd)));
-
-    text.visit(
-        [&](kj::ArrayPtr<const char> text) {
-          out.write(text.begin(), text.size());
-        });
   }
 
   kj::MainBuilder::Validity run() {
     ReaderOptions options;
     options.traversalLimitInWords = 1 << 30;  // Don't limit.
-    StreamFdMessageReader reader(STDIN_FILENO, options);
+    StreamFdMessageReader reader(0, options);
     auto request = reader.getRoot<schema::CodeGeneratorRequest>();
 
     auto capnpVersion = request.getCapnpVersion();
@@ -3058,9 +3089,6 @@ private:
       schemaLoader.load(node);
     }
 
-    kj::FdOutputStream rawOut(STDOUT_FILENO);
-    kj::BufferedOutputStreamWrapper out(rawOut);
-
     for (auto requestedFile: request.getRequestedFiles()) {
       auto schema = schemaLoader.get(requestedFile.getId());
       auto fileText = makeFileText(schema, requestedFile);

c++/src/capnp/compiler/error-reporter.h

@@ -30,6 +30,7 @@
 #include <kj/string.h>
 #include <kj/exception.h>
 #include <kj/vector.h>
+#include <kj/filesystem.h>
 
 namespace capnp {
 namespace compiler {
@@ -68,7 +69,8 @@ public:
     uint column;
   };
 
-  virtual void addError(kj::StringPtr file, SourcePos start, SourcePos end,
+  virtual void addError(const kj::ReadableDirectory& directory, kj::PathPtr path,
+                        SourcePos start, SourcePos end,
                         kj::StringPtr message) = 0;
   // Report an error at the given location in the given file.
 

c++/src/capnp/compiler/module-loader.h

@@ -31,6 +31,7 @@
 #include <kj/memory.h>
 #include <kj/array.h>
 #include <kj/string.h>
+#include <kj/filesystem.h>
 
 namespace capnp {
 namespace compiler {
@@ -44,13 +45,12 @@ public:
 
   ~ModuleLoader() noexcept(false);
 
-  void addImportPath(kj::String path);
+  void addImportPath(const kj::ReadableDirectory& dir);
   // Add a directory to the list of paths that is searched for imports that start with a '/'.
 
-  kj::Maybe<Module&> loadModule(kj::StringPtr localName, kj::StringPtr sourceName);
-  // Tries to load the module with the given filename.  `localName` is the path to the file on
-  // disk (as you'd pass to open(2)), and `sourceName` is the canonical name it should be given
-  // in the schema (this is used e.g. to decide output file locations).  Often, these are the same.
+  kj::Maybe<Module&> loadModule(const kj::ReadableDirectory& dir, kj::PathPtr path);
+  // Tries to load a module with the given path inside the given directory. Returns nullptr if the
+  // file doesn't exist.
 
 private:
   class Impl;

c++/src/capnp/schema-parser-test.c++

@@ -30,26 +30,28 @@
 namespace capnp {
 namespace {
 
-class FakeFileReader final: public SchemaFile::FileReader {
+#if _WIN32
+#define ABS(x) "C:\\" x
+#else
+#define ABS(x) "/" x
+#endif
+
+class FakeFileReader final: public kj::Filesystem {
 public:
   void add(kj::StringPtr name, kj::StringPtr content) {
-    files[name] = content;
+    root->openFile(cwd.evalNative(name), kj::WriteMode::CREATE | kj::WriteMode::CREATE_PARENT)
+        ->writeAll(content);
   }
 
-  bool exists(kj::StringPtr path) const override {
-    return files.count(path) > 0;
-  }
-
-  kj::Array<const char> read(kj::StringPtr path) const override {
-    auto iter = files.find(path);
-    KJ_ASSERT(iter != files.end(), "FakeFileReader has no such file.", path);
-    auto result = kj::heapArray<char>(iter->second.size());
-    memcpy(result.begin(), iter->second.begin(), iter->second.size());
-    return kj::mv(result);
-  }
+  const kj::Directory& getRoot() const override { return *root; }
+  const kj::Directory& getCurrent() const override { return *current; }
+  kj::PathPtr getCurrentPath() const override { return cwd; }
 
 private:
-  std::map<kj::StringPtr, kj::StringPtr> files;
+  kj::Own<const kj::Directory> root = kj::newInMemoryDirectory(kj::nullClock());
+  kj::Path cwd = kj::Path({}).evalNative(ABS("path/to/current/dir"));
+  kj::Own<const kj::Directory> current = root->openSubdir(cwd,
+      kj::WriteMode::CREATE | kj::WriteMode::CREATE_PARENT);
 };
 
 static uint64_t getFieldTypeFileId(StructSchema::Field field) {
@@ -59,8 +61,9 @@ static uint64_t getFieldTypeFileId(StructSchema::Field field) {
 }
 
 TEST(SchemaParser, Basic) {
-  SchemaParser parser;
   FakeFileReader reader;
+  SchemaParser parser;
+  parser.setDiskFilesystem(reader);
 
   reader.add("src/foo/bar.capnp",
       "@0x8123456789abcdef;\n"
@@ -76,22 +79,22 @@ TEST(SchemaParser, Basic) {
   reader.add("src/qux/corge.capnp",
       "@0x83456789abcdef12;\n"
       "struct Corge {}\n");
-  reader.add("/usr/include/grault.capnp",
+  reader.add(ABS("usr/include/grault.capnp"),
       "@0x8456789abcdef123;\n"
       "struct Grault {}\n");
-  reader.add("/opt/include/grault.capnp",
+  reader.add(ABS("opt/include/grault.capnp"),
       "@0x8000000000000001;\n"
       "struct WrongGrault {}\n");
-  reader.add("/usr/local/include/garply.capnp",
+  reader.add(ABS("usr/local/include/garply.capnp"),
       "@0x856789abcdef1234;\n"
       "struct Garply {}\n");
 
   kj::StringPtr importPath[] = {
-    "/usr/include", "/usr/local/include", "/opt/include"
+    ABS("usr/include"), ABS("usr/local/include"), ABS("opt/include")
   };
 
-  ParsedSchema barSchema = parser.parseFile(SchemaFile::newDiskFile(
-      "foo2/bar2.capnp", "src/foo/bar.capnp", importPath, reader));
+  ParsedSchema barSchema = parser.parseDiskFile(
+      "foo2/bar2.capnp", "src/foo/bar.capnp", importPath);
 
   auto barProto = barSchema.getProto();
   EXPECT_EQ(0x8123456789abcdefull, barProto.getId());
@@ -109,25 +112,25 @@ TEST(SchemaParser, Basic) {
   EXPECT_EQ("garply", barFields[3].getProto().getName());
   EXPECT_EQ(0x856789abcdef1234ull, getFieldTypeFileId(barFields[3]));
 
-  auto bazSchema = parser.parseFile(SchemaFile::newDiskFile(
+  auto bazSchema = parser.parseDiskFile(
       "not/used/because/already/loaded",
-      "src/foo/baz.capnp", importPath, reader));
+      "src/foo/baz.capnp", importPath);
   EXPECT_EQ(0x823456789abcdef1ull, bazSchema.getProto().getId());
   EXPECT_EQ("foo2/baz.capnp", bazSchema.getProto().getDisplayName());
   auto bazStruct = bazSchema.getNested("Baz").asStruct();
   EXPECT_EQ(bazStruct, barStruct.getDependency(bazStruct.getProto().getId()));
 
-  auto corgeSchema = parser.parseFile(SchemaFile::newDiskFile(
+  auto corgeSchema = parser.parseDiskFile(
       "not/used/because/already/loaded",
-      "src/qux/corge.capnp", importPath, reader));
+      "src/qux/corge.capnp", importPath);
   EXPECT_EQ(0x83456789abcdef12ull, corgeSchema.getProto().getId());
   EXPECT_EQ("qux/corge.capnp", corgeSchema.getProto().getDisplayName());
   auto corgeStruct = corgeSchema.getNested("Corge").asStruct();
   EXPECT_EQ(corgeStruct, barStruct.getDependency(corgeStruct.getProto().getId()));
 
-  auto graultSchema = parser.parseFile(SchemaFile::newDiskFile(
+  auto graultSchema = parser.parseDiskFile(
       "not/used/because/already/loaded",
-      "/usr/include/grault.capnp", importPath, reader));
+      ABS("usr/include/grault.capnp"), importPath);
   EXPECT_EQ(0x8456789abcdef123ull, graultSchema.getProto().getId());
   EXPECT_EQ("grault.capnp", graultSchema.getProto().getDisplayName());
   auto graultStruct = graultSchema.getNested("Grault").asStruct();
@@ -135,9 +138,9 @@ TEST(SchemaParser, Basic) {
 
   // Try importing the other grault.capnp directly.  It'll get the display name we specify since
   // it wasn't imported before.
-  auto wrongGraultSchema = parser.parseFile(SchemaFile::newDiskFile(
+  auto wrongGraultSchema = parser.parseDiskFile(
       "weird/display/name.capnp",
-      "/opt/include/grault.capnp", importPath, reader));
+      ABS("opt/include/grault.capnp"), importPath);
   EXPECT_EQ(0x8000000000000001ull, wrongGraultSchema.getProto().getId());
   EXPECT_EQ("weird/display/name.capnp", wrongGraultSchema.getProto().getDisplayName());
 }
@@ -147,8 +150,9 @@ TEST(SchemaParser, Constants) {
   // constants are not actually accessible from the generated code API, so the only way to ever
   // get a ConstSchema is by parsing it.
 
-  SchemaParser parser;
   FakeFileReader reader;
+  SchemaParser parser;
+  parser.setDiskFilesystem(reader);
 
   reader.add("const.capnp",
       "@0x8123456789abcdef;\n"
@@ -164,8 +168,8 @@ TEST(SchemaParser, Constants) {
       "  value @0 :T;\n"
       "}\n");
 
-  ParsedSchema fileSchema = parser.parseFile(SchemaFile::newDiskFile(
-      "const.capnp", "const.capnp", nullptr, reader));
+  ParsedSchema fileSchema = parser.parseDiskFile(
+      "const.capnp", "const.capnp", nullptr);
 
   EXPECT_EQ(1234, fileSchema.getNested("uint32Const").asConst().as<uint32_t>());
 
@@ -198,8 +202,9 @@ void expectSourceInfo(schema::Node::SourceInfo::Reader sourceInfo,
 }
 
 TEST(SchemaParser, SourceInfo) {
-  SchemaParser parser;
   FakeFileReader reader;
+  SchemaParser parser;
+  parser.setDiskFilesystem(reader);
 
   reader.add("foo.capnp",
       "@0x84a2c6051e1061ed;\n"
@@ -234,8 +239,8 @@ TEST(SchemaParser, SourceInfo) {
       "struct Thud @0xcca9972702b730b4 {}\n"
       "# post-comment\n");
 
-  ParsedSchema file = parser.parseFile(SchemaFile::newDiskFile(
-      "foo.capnp", "foo.capnp", nullptr, reader));
+  ParsedSchema file = parser.parseDiskFile(
+      "foo.capnp", "foo.capnp", nullptr);
   ParsedSchema foo = file.getNested("Foo");
 
   expectSourceInfo(file.getSourceInfo(), 0x84a2c6051e1061edull, "file doc comment\n", {});

c++/src/capnp/schema-parser.h

@@ -28,6 +28,7 @@
 
 #include "schema-loader.h"
 #include <kj/string.h>
+#include <kj/filesystem.h>
 
 namespace capnp {
 
@@ -43,31 +44,86 @@ public:
   SchemaParser();
   ~SchemaParser() noexcept(false);
 
-  ParsedSchema parseDiskFile(kj::StringPtr displayName, kj::StringPtr diskPath,
-                             kj::ArrayPtr<const kj::StringPtr> importPath) const;
-  // Parse a file located on disk.  Throws an exception if the file dosen't exist.
-  //
-  // Parameters:
-  // * `displayName`:  The name that will appear in the file's schema node.  (If the file has
-  //   already been parsed, this will be ignored and the display name from the first time it was
-  //   parsed will be kept.)
-  // * `diskPath`:  The path to the file on disk.
-  // * `importPath`:  Directories to search when resolving absolute imports within this file
-  //   (imports that start with a `/`).  Must remain valid until the SchemaParser is destroyed.
-  //   (If the file has already been parsed, this will be ignored and the import path from the
-  //   first time it was parsed will be kept.)
+  ParsedSchema parseFromDirectory(
+      const kj::ReadableDirectory& baseDir, kj::Path path,
+      kj::ArrayPtr<const kj::ReadableDirectory* const> importPath) const;
+  // Parse a file from the KJ filesystem API.  Throws an exception if the file dosen't exist.
+  //
+  // `baseDir` and `path` are used together to resolve relative imports. `path` is the source
+  // file's path within `baseDir`. Relative imports will be interpreted relative to `path` and
+  // will be opened using `baseDir`. Note that the KJ filesystem API prohibits "breaking out" of
+  // a directory using "..", so relative imports will be restricted to children of `baseDir`.
+  //
+  // `importPath` is used for absolute imports (imports that start with a '/'). Each directory in
+  // the array will be searched in order until a file is found.
+  //
+  // All `ReadableDirectory` objects must remain valid until the `SchemaParser` is destroyed. Also,
+  // the `importPath` array must remain valid. `path` will be copied; it need not remain valid.
   //
   // This method is a shortcut, equivalent to:
-  //     parser.parseFile(SchemaFile::newDiskFile(displayName, diskPath, importPath))`;
+  //     parser.parseFromDirectory(SchemaFile::newDiskFile(baseDir, path, importPath))`;
   //
   // This method throws an exception if any errors are encountered in the file or in anything the
   // file depends on.  Note that merely importing another file does not count as a dependency on
   // anything in the imported file -- only the imported types which are actually used are
   // "dependencies".
+  //
+  // Hint: Use kj::newDiskFilesystem() to initialize the KJ filesystem API. Usually you should do
+  //   this at a high level in your program, e.g. the main() function, and then pass down the
+  //   appropriate File/Directory objects to the components that need them. Example:
+  //
+  //     auto fs = kj::newDiskFilesystem();
+  //     SchemaParser parser;
+  //     auto schema = parser->parseFromDirectory(fs->getCurrent(),
+  //         kj::Path::parse("foo/bar.capnp"), nullptr);
+  //
+  // Hint: To use in-memory data rather than real disk, you can use kj::newInMemoryDirectory(),
+  //   write the files you want, then pass it to SchemaParser. Example:
+  //
+  //     auto dir = kj::newInMemoryDirectory(kj::nullClock());
+  //     auto path = kj::Path::parse("foo/bar.capnp");
+  //     dir->openFile(path, kj::WriteMode::CREATE | kj::WriteMode::CREATE_PARENT)
+  //        ->writeAll("struct Foo {}");
+  //     auto schema = parser->parseFromDirectory(*dir, path, nullptr);
+  //
+  // Hint: You can create an in-memory directory but then populate it with real files from disk,
+  //   in order to control what is visible while also avoiding reading files yourself or making
+  //   extra copies. Example:
+  //
+  //     auto fs = kj::newDiskFilesystem();
+  //     auto dir = kj::newInMemoryDirectory(kj::nullClock());
+  //     auto fakePath = kj::Path::parse("foo/bar.capnp");
+  //     auto realPath = kj::Path::parse("path/to/some/file.capnp");
+  //     dir->transfer(fakePath, kj::WriteMode::CREATE | kj::WriteMode::CREATE_PARENT,
+  //                   fs->getCurrent(), realPath, kj::TransferMode::LINK);
+  //     auto schema = parser->parseFromDirectory(*dir, fakePath, nullptr);
+  //
+  //   In this example, note that any imports in the file will fail, since the in-memory directory
+  //   you created contains no files except the specific one you linked in.
+
+  ParsedSchema parseDiskFile(kj::StringPtr displayName, kj::StringPtr diskPath,
+                             kj::ArrayPtr<const kj::StringPtr> importPath) const
+      CAPNP_DEPRECATED("Use parseFromDirectory() instead.");
+  // Creates a private kj::Filesystem and uses it to parse files from the real disk.
+  //
+  // DO NOT USE in new code. Use parseFromDirectory() instead.
+  //
+  // This API has a serious problem: the file can import and embed files located anywhere on disk
+  // using relative paths. Even if you specify no `importPath`, relative imports still work. By
+  // using `parseFromDirectory()`, you can arrange so that imports are only allowed within a
+  // particular directory, or even set up a dummy filesystem where other files are not visible.
+
+  void setDiskFilesystem(kj::Filesystem& fs)
+      CAPNP_DEPRECATED("Use parseFromDirectory() instead.");
+  // Call before calling parseDiskFile() to choose an alternative disk filesystem implementation.
+  // This exists mostly for testing purposes; new code should use parseFromDirectory() instead.
+  //
+  // If parseDiskFile() is called without having called setDiskFilesystem(), then
+  // kj::newDiskFilesystem() will be used instead.
 
   ParsedSchema parseFile(kj::Own<SchemaFile>&& file) const;
   // Advanced interface for parsing a file that may or may not be located in any global namespace.
-  // Most users will prefer `parseDiskFile()`.
+  // Most users will prefer `parseFromDirectory()`.
   //
   // If the file has already been parsed (that is, a SchemaFile that compares equal to this one
   // was parsed previously), the existing schema will be returned again.
@@ -90,6 +146,7 @@ public:
 
 private:
   struct Impl;
+  struct DiskFileCompat;
   class ModuleImpl;
   kj::Own<Impl> impl;
   mutable bool hadErrors = false;
@@ -135,44 +192,20 @@ class SchemaFile {
   // `SchemaFile::newDiskFile()`.
 
 public:
-  class FileReader {
-  public:
-    virtual bool exists(kj::StringPtr path) const = 0;
-    virtual kj::Array<const char> read(kj::StringPtr path) const = 0;
-  };
-
-  class DiskFileReader final: public FileReader {
-    // Implementation of FileReader that uses the local disk.  Files are read using mmap() if
-    // possible.
-
-  public:
-    static const DiskFileReader instance;
-
-    bool exists(kj::StringPtr path) const override;
-    kj::Array<const char> read(kj::StringPtr path) const override;
-  };
-
-  static kj::Own<SchemaFile> newDiskFile(
-      kj::StringPtr displayName, kj::StringPtr diskPath,
-      kj::ArrayPtr<const kj::StringPtr> importPath,
-      const FileReader& fileReader = DiskFileReader::instance);
-  // Construct a SchemaFile representing a file on disk (or located in the filesystem-like
-  // namespace represented by `fileReader`).
-  //
-  // Parameters:
-  // * `displayName`:  The name that will appear in the file's schema node.
-  // * `diskPath`:  The path to the file on disk.
-  // * `importPath`:  Directories to search when resolving absolute imports within this file
-  //   (imports that start with a `/`).  The array content must remain valid as long as the
-  //   SchemaFile exists (which is at least as long as the SchemaParser that parses it exists).
-  // * `fileReader`:  Allows you to use a filesystem other than the actual local disk.  Although,
-  //   if you find yourself using this, it may make more sense for you to implement SchemaFile
-  //   yourself.
-  //
-  // The SchemaFile compares equal to any other SchemaFile that has exactly the same disk path,
-  // after canonicalization.
-  //
-  // The SchemaFile will throw an exception if any errors are reported.
+  // Note: Cap'n Proto 0.6.x and below had classes FileReader and DiskFileReader and a method
+  //   newDiskFile() defined here. These were removed when SchemaParser was transitioned to use the
+  //   KJ filesystem API. You should be able to get the same effect by subclassing
+  //   kj::ReadableDirectory, or using kj::newInMemoryDirectory().
+
+  static kj::Own<SchemaFile> newFromDirectory(
+      const kj::ReadableDirectory& baseDir, kj::Path path,
+      kj::ArrayPtr<const kj::ReadableDirectory* const> importPath,
+      kj::Maybe<kj::String> displayNameOverride = nullptr);
+  // Construct a SchemaFile representing a file in a kj::ReadableDirectory. This is used to
+  // implement SchemaParser::parseFromDirectory(); see there for details.
+  //
+  // The SchemaFile compares equal to any other SchemaFile that has exactly the same `baseDir`
+  // object (by identity) and `path` (by value).
 
   // -----------------------------------------------------------------
   // For more control, you can implement this interface.

c++/src/kj/filesystem-disk-test.c++

@@ -310,10 +310,10 @@ KJ_TEST("DiskFile") {
     KJ_EXPECT(kj::str(writableMapping->get().slice(0, 6).asChars()) == "foobaz");
     KJ_EXPECT(kj::str(privateMapping.slice(0, 6).asChars()) == "Foobaz");
 
-    writableMapping->get()[0] = 'D';
-    writableMapping->changed(writableMapping->get().slice(0, 1));
-    KJ_EXPECT(kj::str(mapping.slice(0, 6).asChars()) == "Doobaz");
-    KJ_EXPECT(kj::str(writableMapping->get().slice(0, 6).asChars()) == "Doobaz");
+    writableMapping->get()[1] = 'D';
+    writableMapping->changed(writableMapping->get().slice(1, 2));
+    KJ_EXPECT(kj::str(mapping.slice(0, 6).asChars()) == "fDobaz");
+    KJ_EXPECT(kj::str(writableMapping->get().slice(0, 6).asChars()) == "fDobaz");
     KJ_EXPECT(kj::str(privateMapping.slice(0, 6).asChars()) == "Foobaz");
 
     file->write(0, StringPtr("qux").asBytes());

c++/src/kj/filesystem-test.c++

@@ -86,6 +86,47 @@ KJ_TEST("Path") {
   KJ_EXPECT(kj::str(Path({"foo", "bar"})) == "foo/bar");
 }
 
+KJ_TEST("Path comparisons") {
+  KJ_EXPECT(Path({"foo", "bar"}) == Path({"foo", "bar"}));
+  KJ_EXPECT(!(Path({"foo", "bar"}) != Path({"foo", "bar"})));
+  KJ_EXPECT(Path({"foo", "bar"}) != Path({"foo", "baz"}));
+  KJ_EXPECT(!(Path({"foo", "bar"}) == Path({"foo", "baz"})));
+
+  KJ_EXPECT(Path({"foo", "bar"}) != Path({"fob", "bar"}));
+  KJ_EXPECT(Path({"foo", "bar"}) != Path({"foo", "bar", "baz"}));
+  KJ_EXPECT(Path({"foo", "bar", "baz"}) != Path({"foo", "bar"}));
+
+  KJ_EXPECT(Path({"foo", "bar"}) <= Path({"foo", "bar"}));
+  KJ_EXPECT(Path({"foo", "bar"}) >= Path({"foo", "bar"}));
+  KJ_EXPECT(!(Path({"foo", "bar"}) < Path({"foo", "bar"})));
+  KJ_EXPECT(!(Path({"foo", "bar"}) > Path({"foo", "bar"})));
+
+  KJ_EXPECT(Path({"foo", "bar"}) < Path({"foo", "bar", "baz"}));
+  KJ_EXPECT(!(Path({"foo", "bar"}) > Path({"foo", "bar", "baz"})));
+  KJ_EXPECT(Path({"foo", "bar", "baz"}) > Path({"foo", "bar"}));
+  KJ_EXPECT(!(Path({"foo", "bar", "baz"}) < Path({"foo", "bar"})));
+
+  KJ_EXPECT(Path({"foo", "bar"}) < Path({"foo", "baz"}));
+  KJ_EXPECT(Path({"foo", "bar"}) > Path({"foo", "baa"}));
+  KJ_EXPECT(Path({"foo", "bar"}) > Path({"foo"}));
+
+  KJ_EXPECT(Path({"foo", "bar"}).startsWith(Path({})));
+  KJ_EXPECT(Path({"foo", "bar"}).startsWith(Path({"foo"})));
+  KJ_EXPECT(Path({"foo", "bar"}).startsWith(Path({"foo", "bar"})));
+  KJ_EXPECT(!Path({"foo", "bar"}).startsWith(Path({"foo", "bar", "baz"})));
+  KJ_EXPECT(!Path({"foo", "bar"}).startsWith(Path({"foo", "baz"})));
+  KJ_EXPECT(!Path({"foo", "bar"}).startsWith(Path({"baz", "foo", "bar"})));
+  KJ_EXPECT(!Path({"foo", "bar"}).startsWith(Path({"baz"})));
+
+  KJ_EXPECT(Path({"foo", "bar"}).endsWith(Path({})));
+  KJ_EXPECT(Path({"foo", "bar"}).endsWith(Path({"bar"})));
+  KJ_EXPECT(Path({"foo", "bar"}).endsWith(Path({"foo", "bar"})));
+  KJ_EXPECT(!Path({"foo", "bar"}).endsWith(Path({"baz", "foo", "bar"})));
+  KJ_EXPECT(!Path({"foo", "bar"}).endsWith(Path({"fob", "bar"})));
+  KJ_EXPECT(!Path({"foo", "bar"}).endsWith(Path({"foo", "bar", "baz"})));
+  KJ_EXPECT(!Path({"foo", "bar"}).endsWith(Path({"baz"})));
+}
+
 KJ_TEST("Path exceptions") {
   KJ_EXPECT_THROW_MESSAGE("invalid path component", Path(""));
   KJ_EXPECT_THROW_MESSAGE("invalid path component", Path("."));
@@ -223,13 +264,13 @@ public:
     time += 1 * SECONDS;
   }
 
-  Date now() override { return time; }
+  Date now() const override { return time; }
 
-  void expectChanged(FsNode& file) {
+  void expectChanged(const FsNode& file) {
     KJ_EXPECT(file.stat().lastModified == time);
     time += 1 * SECONDS;
   }
-  void expectUnchanged(FsNode& file) {
+  void expectUnchanged(const FsNode& file) {
     KJ_EXPECT(file.stat().lastModified != time);
   }
 

c++/src/kj/time.c++

@@ -26,12 +26,12 @@
 
 namespace kj {
 
-Clock& nullClock() {
+const Clock& nullClock() {
   class NullClock final: public Clock {
   public:
-    Date now() override { return UNIX_EPOCH; }
+    Date now() const override { return UNIX_EPOCH; }
   };
-  static NullClock NULL_CLOCK;
+  static KJ_CONSTEXPR(const) NullClock NULL_CLOCK;
   return NULL_CLOCK;
 }
 

c++/src/kj/time.h

@@ -63,10 +63,10 @@ constexpr Date UNIX_EPOCH = origin<Date>();
 class Clock {
   // Interface to read the current date and time.
 public:
-  virtual Date now() = 0;
+  virtual Date now() const = 0;
 };
 
-Clock& nullClock();
+const Clock& nullClock();
 // A clock which always returns UNIX_EPOCH as the current time. Useful when you don't care about
 // time.