Add public interface for schema file parsing.

c++/src/capnp/compiler/compiler.h

@@ -34,10 +34,6 @@ namespace compiler {
 
 class Module: public ErrorReporter {
 public:
-  virtual kj::StringPtr getLocalName() const = 0;
-  // Typically, the absolute or cwd-relative path name of the module file, used in error messages.
-  // This is only for display purposes.
-
   virtual kj::StringPtr getSourceName() const = 0;
   // The name of the module file relative to the source tree.  Used to decide where to output
   // generated code and to form the `displayName` in the schema.

c++/src/capnp/compiler/md5.h

@@ -49,6 +49,9 @@ public:
   inline void update(kj::StringPtr data) {
     return update(data.asArray());
   }
+  inline void update(const char* data) {
+    return update(kj::StringPtr(data));
+  }
 
   kj::ArrayPtr<const kj::byte> finish();
   kj::StringPtr finishAsHex();

c++/src/capnp/compiler/module-loader.c++

@@ -216,7 +216,7 @@ public:
   ModuleImpl(const ModuleLoader::Impl& loader, kj::String localName, kj::String sourceName)
       : loader(loader), localName(kj::mv(localName)), sourceName(kj::mv(sourceName)) {}
 
-  kj::StringPtr getLocalName() const override {
+  kj::StringPtr getLocalName() const {
     return localName;
   }
 
@@ -275,7 +275,7 @@ public:
         message);
   }
 
-  bool hadErrors() const {
+  bool hadErrors() const override {
     return loader.getErrorReporter().hadErrors();
   }
 

c++/src/capnp/compiler/node-translator.c++

@@ -752,6 +752,7 @@ public:
       errorReporter.addErrorOn(ordinal,
           kj::str("Skipped ordinal @", expectedOrdinal, ".  Ordinals must be sequential with no "
                   "holes."));
+      expectedOrdinal = ordinal.getValue() + 1;
     } else {
       ++expectedOrdinal;
       lastOrdinalLocation = ordinal;

c++/src/capnp/schema-parser-test.c++

+// Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "schema-parser.h"
+#include <gtest/gtest.h>
+#include "test-util.h"
+#include <kj/debug.h>
+#include <map>
+
+namespace capnp {
+namespace {
+
+class FakeFileReader final: public SchemaFile::FileReader {
+public:
+  void add(kj::StringPtr name, kj::StringPtr content) {
+    files[name] = content;
+  }
+
+  bool exists(kj::StringPtr path) const override {
+    return files.count(path) > 0;
+  }
+
+  kj::Array<const char> read(kj::StringPtr path) const override {
+    auto iter = files.find(path);
+    KJ_ASSERT(iter != files.end(), "FakeFileReader has no such file.", path);
+    auto result = kj::heapArray<char>(iter->second.size());
+    memcpy(result.begin(), iter->second.begin(), iter->second.size());
+    return kj::mv(result);
+  }
+
+private:
+  std::map<kj::StringPtr, kj::StringPtr> files;
+};
+
+TEST(SchemaParser, Basic) {
+  SchemaParser parser;
+  FakeFileReader reader;
+
+  reader.add("src/foo/bar.capnp",
+      "@0x8123456789abcdef;\n"
+      "struct Bar {\n"
+      "  baz @0: import \"baz.capnp\".Baz;\n"
+      "  corge @1: import \"../qux/corge.capnp\".Corge;\n"
+      "  grault @2: import \"/grault.capnp\".Grault;\n"
+      "  garply @3: import \"/garply.capnp\".Garply;\n"
+      "}\n");
+  reader.add("src/foo/baz.capnp",
+      "@0x823456789abcdef1;\n"
+      "struct Baz {}\n");
+  reader.add("src/qux/corge.capnp",
+      "@0x83456789abcdef12;\n"
+      "struct Corge {}\n");
+  reader.add("/usr/include/grault.capnp",
+      "@0x8456789abcdef123;\n"
+      "struct Grault {}\n");
+  reader.add("/opt/include/grault.capnp",
+      "@0x8000000000000001;\n"
+      "struct WrongGrault {}\n");
+  reader.add("/usr/local/include/garply.capnp",
+      "@0x856789abcdef1234;\n"
+      "struct Garply {}\n");
+
+  kj::StringPtr importPath[] = {
+    "/usr/include", "/usr/local/include", "/opt/include"
+  };
+
+  ParsedSchema barSchema = parser.parseFile(SchemaFile::newDiskFile(
+      kj::str("foo2/bar2.capnp"), kj::str("src/foo/bar.capnp"), importPath, reader));
+
+  auto barProto = barSchema.getProto();
+  EXPECT_EQ(0x8123456789abcdefull, barProto.getId());
+  EXPECT_EQ("foo2/bar2.capnp", barProto.getDisplayName());
+  auto barImports = barProto.getBody().getFileNode().getImports();
+  ASSERT_EQ(4, barImports.size());
+  EXPECT_EQ("../qux/corge.capnp", barImports[0].getName());
+  EXPECT_EQ(0x83456789abcdef12ull, barImports[0].getId());
+  EXPECT_EQ("/garply.capnp", barImports[1].getName());
+  EXPECT_EQ(0x856789abcdef1234ull, barImports[1].getId());
+  EXPECT_EQ("/grault.capnp", barImports[2].getName());
+  EXPECT_EQ(0x8456789abcdef123ull, barImports[2].getId());
+  EXPECT_EQ("baz.capnp", barImports[3].getName());
+  EXPECT_EQ(0x823456789abcdef1ull, barImports[3].getId());
+
+  auto barStruct = barSchema.getNested("Bar");
+  auto barMembers = barStruct.asStruct().getMembers();
+  ASSERT_EQ(4, barMembers.size());
+  EXPECT_EQ("baz", barMembers[0].getProto().getName());
+  EXPECT_EQ("corge", barMembers[1].getProto().getName());
+  EXPECT_EQ("grault", barMembers[2].getProto().getName());
+  EXPECT_EQ("garply", barMembers[3].getProto().getName());
+
+  auto bazSchema = parser.parseFile(SchemaFile::newDiskFile(
+      kj::str("not/used/because/already/loaded"),
+      kj::str("src/foo/baz.capnp"), importPath, reader));
+  EXPECT_EQ(0x823456789abcdef1ull, bazSchema.getProto().getId());
+  EXPECT_EQ("foo2/baz.capnp", bazSchema.getProto().getDisplayName());
+  auto bazStruct = bazSchema.getNested("Baz").asStruct();
+  EXPECT_EQ(bazStruct, barStruct.getDependency(bazStruct.getProto().getId()));
+
+  auto corgeSchema = parser.parseFile(SchemaFile::newDiskFile(
+      kj::str("not/used/because/already/loaded"),
+      kj::str("src/qux/corge.capnp"), importPath, reader));
+  EXPECT_EQ(0x83456789abcdef12ull, corgeSchema.getProto().getId());
+  EXPECT_EQ("qux/corge.capnp", corgeSchema.getProto().getDisplayName());
+  auto corgeStruct = corgeSchema.getNested("Corge").asStruct();
+  EXPECT_EQ(corgeStruct, barStruct.getDependency(corgeStruct.getProto().getId()));
+
+  auto graultSchema = parser.parseFile(SchemaFile::newDiskFile(
+      kj::str("not/used/because/already/loaded"),
+      kj::str("/usr/include/grault.capnp"), importPath, reader));
+  EXPECT_EQ(0x8456789abcdef123ull, graultSchema.getProto().getId());
+  EXPECT_EQ("grault.capnp", graultSchema.getProto().getDisplayName());
+  auto graultStruct = graultSchema.getNested("Grault").asStruct();
+  EXPECT_EQ(graultStruct, barStruct.getDependency(graultStruct.getProto().getId()));
+
+  auto wrongGraultSchema = parser.parseFile(SchemaFile::newDiskFile(
+      kj::str("weird/display/name.capnp"),
+      kj::str("/opt/include/grault.capnp"), importPath, reader));
+  EXPECT_EQ(0x8000000000000001ull, wrongGraultSchema.getProto().getId());
+  EXPECT_EQ("weird/display/name.capnp", wrongGraultSchema.getProto().getDisplayName());
+}
+
+}  // namespace
+}  // namespace capnp

c++/src/capnp/schema-parser.c++

+// Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "schema-parser.h"
+#include "message.h"
+#include <capnp/compiler/compiler.h>
+#include <capnp/compiler/lexer.capnp.h>
+#include <capnp/compiler/lexer.h>
+#include <capnp/compiler/grammar.capnp.h>
+#include <capnp/compiler/parser.h>
+#include <unordered_map>
+#include <kj/mutex.h>
+#include <kj/vector.h>
+#include <kj/debug.h>
+#include <kj/io.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <errno.h>
+
+namespace capnp {
+
+namespace {
+
+template <typename T>
+size_t findLargestElementBefore(const kj::Vector<T>& vec, const T& key) {
+  KJ_REQUIRE(vec.size() > 0 && vec[0] <= key);
+
+  size_t lower = 0;
+  size_t upper = vec.size();
+
+  while (upper - lower > 1) {
+    size_t mid = (lower + upper) / 2;
+    if (vec[mid] > key) {
+      upper = mid;
+    } else {
+      lower = mid;
+    }
+  }
+
+  return lower;
+}
+
+}  // namespace
+
+// =======================================================================================
+
+class SchemaParser::ModuleImpl: public compiler::Module {
+public:
+  ModuleImpl(const SchemaParser& parser, kj::Own<const SchemaFile>&& file)
+      : parser(parser), file(kj::mv(file)) {}
+
+  kj::StringPtr getSourceName() const override {
+    return file->getDisplayName();
+  }
+
+  Orphan<compiler::ParsedFile> loadContent(Orphanage orphanage) const override {
+    kj::Array<const char> content = file->readContent();
+
+    lineBreaks.get([&](kj::SpaceFor<kj::Vector<uint>>& space) {
+      auto vec = space.construct(content.size() / 40);
+      vec->add(0);
+      for (const char* pos = content.begin(); pos < content.end(); ++pos) {
+        if (*pos == '\n') {
+          vec->add(pos + 1 - content.begin());
+        }
+      }
+      return vec;
+    });
+
+    MallocMessageBuilder lexedBuilder;
+    auto statements = lexedBuilder.initRoot<compiler::LexedStatements>();
+    compiler::lex(content, statements, *this);
+
+    auto parsed = orphanage.newOrphan<compiler::ParsedFile>();
+    compiler::parseFile(statements.getStatements(), parsed.get(), *this);
+    return parsed;
+  }
+
+  kj::Maybe<const Module&> importRelative(kj::StringPtr importPath) const override {
+    KJ_IF_MAYBE(importedFile, file->import(importPath)) {
+      return parser.getModuleImpl(kj::mv(importedFile));
+    } else {
+      return nullptr;
+    }
+  }
+
+  void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) const override {
+    auto& lines = lineBreaks.get(
+        [](kj::SpaceFor<kj::Vector<uint>>& space) {
+          KJ_FAIL_REQUIRE("Can't report errors until loadContent() is called.");
+          return space.construct();
+        });
+
+    // TODO(someday):  This counts tabs as single characters.  Do we care?
+    uint startLine = findLargestElementBefore(lines, startByte);
+    uint startCol = startByte - lines[startLine];
+    uint endLine = findLargestElementBefore(lines, endByte);
+    uint endCol = endByte - lines[endLine];
+
+    file->reportError(
+        SchemaFile::SourcePos { startByte, startLine, startCol },
+        SchemaFile::SourcePos { endByte, endLine, endCol },
+        message);
+
+    // We intentionally only set hadErrors true if reportError() didn't throw.
+    __atomic_store_n(&parser.hadErrors, true, __ATOMIC_RELAXED);
+  }
+
+  bool hadErrors() const override {
+    return __atomic_load_n(&parser.hadErrors, __ATOMIC_RELAXED);
+  }
+
+private:
+  const SchemaParser& parser;
+  kj::Own<const SchemaFile> file;
+
+  kj::Lazy<kj::Vector<uint>> lineBreaks;
+  // Byte offsets of the first byte in each source line.  The first element is always zero.
+  // Initialized the first time the module is loaded.
+};
+
+// =======================================================================================
+
+namespace {
+
+struct SchemaFileHash {
+  inline bool operator()(const SchemaFile* f) const {
+    return f->hashCode();
+  }
+};
+
+struct SchemaFileEq {
+  inline bool operator()(const SchemaFile* a, const SchemaFile* b) const {
+    return *a == *b;
+  }
+};
+
+}  // namespace
+
+struct SchemaParser::Impl {
+  typedef std::unordered_map<
+      const SchemaFile*, kj::Own<const ModuleImpl>, SchemaFileHash, SchemaFileEq> FileMap;
+  kj::MutexGuarded<FileMap> fileMap;
+  compiler::Compiler compiler;
+};
+
+SchemaParser::SchemaParser(): impl(kj::heap<Impl>()) {}
+SchemaParser::~SchemaParser() noexcept(false) {}
+
+ParsedSchema SchemaParser::parseDiskFile(
+    kj::StringPtr displayName, kj::StringPtr diskPath,
+    kj::ArrayPtr<const kj::StringPtr> importPath) {
+  return parseFile(SchemaFile::newDiskFile(displayName, diskPath, importPath));
+}
+
+ParsedSchema SchemaParser::parseFile(kj::Own<SchemaFile>&& file) {
+  KJ_DEFER(impl->compiler.clearWorkspace());
+  uint64_t id = impl->compiler.add(getModuleImpl(kj::mv(file)));
+  impl->compiler.eagerlyCompile(id,
+      compiler::Compiler::NODE | compiler::Compiler::CHILDREN |
+      compiler::Compiler::DEPENDENCIES | compiler::Compiler::DEPENDENCY_DEPENDENCIES);
+  return ParsedSchema(impl->compiler.getLoader().get(id), *this);
+}
+
+const SchemaParser::ModuleImpl& SchemaParser::getModuleImpl(kj::Own<SchemaFile>&& file) const {
+  auto lock = impl->fileMap.lockExclusive();
+
+  auto insertResult = lock->insert(std::make_pair(file.get(), kj::Own<ModuleImpl>()));
+  if (insertResult.second) {
+    // This is a newly-inserted entry.  Construct the ModuleImpl.
+    insertResult.first->second = kj::heap<ModuleImpl>(*this, kj::mv(file));
+  }
+  return *insertResult.first->second;
+}
+
+kj::Maybe<ParsedSchema> ParsedSchema::findNested(kj::StringPtr name) {
+  return parser->impl->compiler.lookup(getProto().getId(), name).map(
+      [this](uint64_t childId) {
+        return ParsedSchema(parser->impl->compiler.getLoader().get(childId), *parser);
+      });
+}
+
+ParsedSchema ParsedSchema::getNested(kj::StringPtr nestedName) {
+  KJ_IF_MAYBE(nested, findNested(nestedName)) {
+    return *nested;
+  } else {
+    KJ_FAIL_REQUIRE("no such nested declaration", getProto().getDisplayName(), nestedName);
+  }
+}
+
+// =======================================================================================
+
+namespace {
+
+class MmapDisposer: public kj::ArrayDisposer {
+protected:
+  void disposeImpl(void* firstElement, size_t elementSize, size_t elementCount,
+                   size_t capacity, void (*destroyElement)(void*)) const {
+    munmap(firstElement, elementSize * elementCount);
+  }
+};
+
+constexpr MmapDisposer mmapDisposer = MmapDisposer();
+
+static char* canonicalizePath(char* path) {
+  // Taken from some old C code of mine.
+
+  // Preconditions:
+  // - path has already been determined to be relative, perhaps because the pointer actually points
+  //   into the middle of some larger path string, in which case it must point to the character
+  //   immediately after a '/'.
+
+  // Invariants:
+  // - src points to the beginning of a path component.
+  // - dst points to the location where the path component should end up, if it is not special.
+  // - src == path or src[-1] == '/'.
+  // - dst == path or dst[-1] == '/'.
+
+  char* src = path;
+  char* dst = path;
+  char* locked = dst;  // dst cannot backtrack past this
+  char* partEnd;
+  bool hasMore;
+
+  for (;;) {
+    while (*src == '/') {
+      // Skip duplicate slash.
+      ++src;
+    }
+
+    partEnd = strchr(src, '/');
+    hasMore = partEnd != NULL;
+    if (hasMore) {
+      *partEnd = '\0';
+    } else {
+      partEnd = src + strlen(src);
+    }
+
+    if (strcmp(src, ".") == 0) {
+      // Skip it.
+    } else if (strcmp(src, "..") == 0) {
+      if (dst > locked) {
+        // Backtrack over last path component.
+        --dst;
+        while (dst > locked && dst[-1] != '/') --dst;
+      } else {
+        locked += 3;
+        goto copy;
+      }
+    } else {
+      // Copy if needed.
+    copy:
+      if (dst < src) {
+        memmove(dst, src, partEnd - src);
+        dst += partEnd - src;
+      } else {
+        dst = partEnd;
+      }
+      *dst++ = '/';
+    }
+
+    if (hasMore) {
+      src = partEnd + 1;
+    } else {
+      // Oops, we have to remove the trailing '/'.
+      if (dst == path) {
+        // Oops, there is no trailing '/'.  We have to return ".".
+        strcpy(path, ".");
+        return path + 1;
+      } else {
+        // Remove the trailing '/'.  Note that this means that opening the file will work even
+        // if it is not a directory, where normally it should fail on non-directories when a
+        // trailing '/' is present.  If this is a problem, we need to add some sort of special
+        // handling for this case where we stat() it separately to check if it is a directory,
+        // because Ekam findInput will not accept a trailing '/'.
+        --dst;
+        *dst = '\0';
+        return dst;
+      }
+    }
+  }
+}
+
+kj::String canonicalizePath(kj::StringPtr path) {
+  KJ_STACK_ARRAY(char, result, path.size() + 1, 128, 512);
+  strcpy(result.begin(), path.begin());
+
+  char* start = path.startsWith("/") ? result.begin() + 1 : result.begin();
+  char* end = canonicalizePath(start);
+  return kj::heapString(result.slice(0, end - result.begin()));
+}
+
+kj::String relativePath(kj::StringPtr base, kj::StringPtr add) {
+  if (add.size() > 0 && add[0] == '/') {
+    return kj::heapString(add);
+  }
+
+  const char* pos = base.end();
+  while (pos > base.begin() && pos[-1] != '/') {
+    --pos;
+  }
+
+  return kj::str(base.slice(0, pos - base.begin()), add);
+}
+
+kj::String joinPath(kj::StringPtr base, kj::StringPtr add) {
+  KJ_REQUIRE(!add.startsWith("/"));
+
+  return kj::str(base, '/', add);
+}
+
+}  // namespace
+
+const SchemaFile::DiskFileReader SchemaFile::DiskFileReader::instance =
+    SchemaFile::DiskFileReader();
+
+bool SchemaFile::DiskFileReader::exists(kj::StringPtr path) const {
+  return access(path.cStr(), F_OK) == 0;
+}
+
+kj::Array<const char> SchemaFile::DiskFileReader::read(kj::StringPtr path) const {
+  int fd;
+  // We already established that the file exists, so this should not fail.
+  KJ_SYSCALL(fd = open(path.cStr(), O_RDONLY), path);
+  kj::AutoCloseFd closer(fd);
+
+  struct stat stats;
+  KJ_SYSCALL(fstat(fd, &stats));
+
+  const void* mapping = mmap(NULL, stats.st_size, PROT_READ, MAP_SHARED, fd, 0);
+  if (mapping == MAP_FAILED) {
+    KJ_FAIL_SYSCALL("mmap", errno, path);
+  }
+
+  return kj::Array<const char>(
+      reinterpret_cast<const char*>(mapping), stats.st_size, mmapDisposer);
+}
+
+// -------------------------------------------------------------------
+
+class SchemaFile::DiskSchemaFile final: public SchemaFile {
+public:
+  DiskSchemaFile(const FileReader& fileReader, kj::String displayName,
+                 kj::String diskPath, kj::ArrayPtr<const kj::StringPtr> importPath)
+      : fileReader(fileReader),
+        displayName(kj::mv(displayName)),
+        diskPath(kj::mv(diskPath)),
+        importPath(importPath) {}
+
+  kj::StringPtr getDisplayName() const override {
+    return displayName;
+  }
+
+  kj::Array<const char> readContent() const override {
+    return fileReader.read(diskPath);
+  }
+
+  kj::Maybe<kj::Own<SchemaFile>> import(kj::StringPtr path) const override {
+    if (path.startsWith("/")) {
+      for (auto candidate: importPath) {
+        kj::String newDiskPath = canonicalizePath(joinPath(candidate, path.slice(1)));
+        if (fileReader.exists(newDiskPath)) {
+          return kj::implicitCast<kj::Own<SchemaFile>>(kj::heap<DiskSchemaFile>(
+              fileReader, canonicalizePath(path.slice(1)),
+              kj::mv(newDiskPath), importPath));
+        }
+      }
+      return nullptr;
+    } else {
+      kj::String newDiskPath = canonicalizePath(relativePath(diskPath, path));
+      if (fileReader.exists(newDiskPath)) {
+        return kj::implicitCast<kj::Own<SchemaFile>>(kj::heap<DiskSchemaFile>(
+            fileReader, canonicalizePath(relativePath(displayName, path)),
+            kj::mv(newDiskPath), importPath));
+      } else {
+        return nullptr;
+      }
+    }
+  }
+
+  bool operator==(const SchemaFile& other) const override {
+    return diskPath == kj::downcast<const DiskSchemaFile>(other).diskPath;
+  }
+  bool operator!=(const SchemaFile& other) const override {
+    return diskPath != kj::downcast<const DiskSchemaFile>(other).diskPath;
+  }
+  size_t hashCode() const override {
+    // djb hash with xor
+    // TODO(someday):  Add hashing library to KJ.
+    size_t result = 5381;
+    for (char c: diskPath) {
+      result = (result * 33) ^ c;
+    }
+    return result;
+  }
+
+  void reportError(SourcePos start, SourcePos end, kj::StringPtr message) const override {
+    kj::getExceptionCallback().onRecoverableException(kj::Exception(
+        kj::Exception::Nature::LOCAL_BUG, kj::Exception::Durability::PERMANENT,
+        kj::heapString(diskPath), start.line, kj::heapString(message)));
+  }
+
+private:
+  const FileReader& fileReader;
+  kj::String displayName;
+  kj::String diskPath;
+  kj::ArrayPtr<const kj::StringPtr> importPath;
+};
+
+kj::Own<SchemaFile> SchemaFile::newDiskFile(
+    kj::StringPtr displayName, kj::StringPtr diskPath,
+    kj::ArrayPtr<const kj::StringPtr> importPath,
+    const FileReader& fileReader) {
+  return kj::heap<DiskSchemaFile>(fileReader, canonicalizePath(displayName),
+                                  canonicalizePath(diskPath), importPath);
+}
+
+}  // namespace capnp

c++/src/capnp/schema-parser.h

+// Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef CAPNP_SCHEMA_PARSER_H_
+#define CAPNP_SCHEMA_PARSER_H_
+
+#include "schema-loader.h"
+#include <kj/string.h>
+
+namespace capnp {
+
+class ParsedSchema;
+class SchemaFile;
+
+class SchemaParser {
+  // Parses `.capnp` files to produce `Schema` objects.
+
+public:
+  SchemaParser();
+  ~SchemaParser() noexcept(false);
+
+  ParsedSchema parseDiskFile(kj::StringPtr displayName, kj::StringPtr diskPath,
+                             kj::ArrayPtr<const kj::StringPtr> importPath);
+  // Parse a file located on disk.  Throws an exception if the file dosen't exist.
+  //
+  // Parameters:
+  // * `displayName`:  The name that will appear in the file's schema node.  (If the file has
+  //   already been parsed, this will be ignored and the display name from the first time it was
+  //   parsed will be kept.)
+  // * `diskPath`:  The path to the file on disk.
+  // * `importPath`:  Directories to search when resolving absolute imports within this file
+  //   (imports that start with a `/`).  Must remain valid until the SchemaParser is destroyed.
+  //   (If the file has already been parsed, this will be ignored and the import path from the
+  //   first time it was parsed will be kept.)
+  //
+  // This method is a shortcut, equivalent to:
+  //     parser.parseFile(SchemaFile::newDiskFile(displayName, diskPath, importPath))`;
+  //
+  // This method throws an exception if any errors are encountered in the file or in anything the
+  // file depends on.  Note that merely importing another file does not count as a dependency on
+  // anything in the imported file -- only the imported types which are actually used are
+  // "dependencies".
+
+  ParsedSchema parseFile(kj::Own<SchemaFile>&& file);
+  // Advanced interface for parsing a file that may or may not be located in any global namespace.
+  // Most users will prefer `parseDiskFile()`.
+  //
+  // If the file has already been parsed (that is, a SchemaFile that compares equal to this one
+  // was parsed previously), the existing schema will be returned again.
+  //
+  // This method reports errors by calling SchemaFile::reportError() on the file where the error
+  // is located.  If that call does not throw an exception, `parseFile()` may in fact return
+  // normally.  In this case, the result is a best-effort attempt to compile the schema, but it
+  // may be invalid or corrupt, and using it for anything may cause exceptions to be thrown.
+
+private:
+  struct Impl;
+  class ModuleImpl;
+  kj::Own<Impl> impl;
+  mutable bool hadErrors = false;
+
+  const ModuleImpl& getModuleImpl(kj::Own<SchemaFile>&& file) const;
+
+  friend class ParsedSchema;
+};
+
+class ParsedSchema: public Schema {
+  // ParsedSchema is an extension of Schema which also has the ability to look up nested nodes
+  // by name.  See `SchemaParser`.
+
+public:
+  inline ParsedSchema(): parser(nullptr) {}
+
+  kj::Maybe<ParsedSchema> findNested(kj::StringPtr name);
+  // Gets the nested node with the given name, or returns null if there is no such nested
+  // declaration.
+
+  ParsedSchema getNested(kj::StringPtr name);
+  // Gets the nested node with the given name, or throws an exception if there is no such nested
+  // declaration.
+
+private:
+  inline ParsedSchema(Schema inner, const SchemaParser& parser): Schema(inner), parser(&parser) {}
+
+  const SchemaParser* parser;
+  friend class SchemaParser;
+};
+
+// =======================================================================================
+// Advanced API
+
+class SchemaFile {
+  // Abstract interface representing a schema file.  You can implement this yourself in order to
+  // gain more control over how the compiler resolves imports and reads files.  For the
+  // common case of files on disk or other global filesystem-like namespaces, use
+  // `SchemaFile::newDiskFile()`.
+
+public:
+  class FileReader {
+  public:
+    virtual bool exists(kj::StringPtr path) const = 0;
+    virtual kj::Array<const char> read(kj::StringPtr path) const = 0;
+  };
+
+  class DiskFileReader final: public FileReader {
+    // Implementation of FileReader that uses the local disk.  Files are read using mmap() if
+    // possible.
+
+  public:
+    static const DiskFileReader instance;
+
+    bool exists(kj::StringPtr path) const override;
+    kj::Array<const char> read(kj::StringPtr path) const override;
+  };
+
+  static kj::Own<SchemaFile> newDiskFile(
+      kj::StringPtr displayName, kj::StringPtr diskPath,
+      kj::ArrayPtr<const kj::StringPtr> importPath,
+      const FileReader& fileReader = DiskFileReader::instance);
+  // Construct a SchemaFile representing a file on disk (or located in the filesystem-like
+  // namespace represented by `fileReader`).
+  //
+  // Parameters:
+  // * `displayName`:  The name that will appear in the file's schema node.
+  // * `diskPath`:  The path to the file on disk.
+  // * `importPath`:  Directories to search when resolving absolute imports within this file
+  //   (imports that start with a `/`).  The array content must remain valid as long as the
+  //   SchemaFile exists (which is at least as long as the SchemaParser that parses it exists).
+  // * `fileReader`:  Allows you to use a filesystem other than the actual local disk.  Although,
+  //   if you find yourself using this, it may make more sense for you to implement SchemaFile
+  //   yourself.
+  //
+  // The SchemaFile compares equal to any other SchemaFile that has exactly the same disk path,
+  // after canonicalization.
+  //
+  // The SchemaFile will throw an exception if any errors are reported.
+
+  // -----------------------------------------------------------------
+  // For more control, you can implement this interface.
+
+  virtual kj::StringPtr getDisplayName() const = 0;
+  // Get the file's name, as it should appear in the schema.
+
+  virtual kj::Array<const char> readContent() const = 0;
+  // Read the file's entire content and return it as a byte array.
+
+  virtual kj::Maybe<kj::Own<SchemaFile>> import(kj::StringPtr path) const = 0;
+  // Resolve an import, relative to this file.
+  //
+  // `path` is exactly what appears between quotes after the `import` keyword in the source code.
+  // It is entirely up to the `SchemaFile` to decide how to map this to another file.  Typically,
+  // a leading '/' means that the file is an "absolute" path and is searched for in some list of
+  // schema file repositories.  On the other hand, a path that doesn't start with '/' is relative
+  // to the importing file.
+
+  virtual bool operator==(const SchemaFile& other) const = 0;
+  virtual bool operator!=(const SchemaFile& other) const = 0;
+  virtual size_t hashCode() const = 0;
+  // Compare two SchemaFiles to see if they refer to the same underlying file.  This is an
+  // optimization used to avoid the need to re-parse a file to check its ID.
+
+  struct SourcePos {
+    uint byte;
+    uint line;
+    uint column;
+  };
+  virtual void reportError(SourcePos start, SourcePos end, kj::StringPtr message) const = 0;
+  // Report that the file contains an error at the given interval.
+
+private:
+  class DiskSchemaFile;
+};
+
+}  // namespace capnp
+
+#endif  // CAPNP_SCHEMA_PARSER_H_

c++/src/kj/common.h

@@ -734,6 +734,10 @@ public:
   inline constexpr ArrayPtr(T* ptr, size_t size): ptr(ptr), size_(size) {}
   inline constexpr ArrayPtr(T* begin, T* end): ptr(begin), size_(end - begin) {}
 
+  template <size_t size>
+  inline constexpr ArrayPtr(T (&native)[size]): ptr(native), size_(size) {}
+  // Construct an ArrayPtr from a native C-style array.
+
   inline operator ArrayPtr<const T>() const {
     return ArrayPtr<const T>(ptr, size_);
   }

c++/src/kj/exception.c++

@@ -184,9 +184,25 @@ Exception::Exception(Nature nature, Durability durability, const char* file, int
 #endif
 }
 
+Exception::Exception(Nature nature, Durability durability, String file, int line,
+                     String description) noexcept
+    : ownFile(kj::mv(file)), file(ownFile.cStr()), line(line), nature(nature),
+      durability(durability), description(mv(description)) {
+#ifdef __CYGWIN__
+  traceCount = 0;
+#else
+  traceCount = backtrace(trace, 16);
+#endif
+}
+
 Exception::Exception(const Exception& other) noexcept
     : file(other.file), line(other.line), nature(other.nature), durability(other.durability),
-      description(str(other.description)), traceCount(other.traceCount) {
+      description(heapString(other.description)), traceCount(other.traceCount) {
+  if (file == other.ownFile.cStr()) {
+    ownFile = heapString(other.ownFile);
+    file = ownFile.cStr();
+  }
+
   memcpy(trace, other.trace, sizeof(trace[0]) * traceCount);
 
   KJ_IF_MAYBE(c, other.context) {

c++/src/kj/exception.h

@@ -69,6 +69,8 @@ public:
 
   Exception(Nature nature, Durability durability, const char* file, int line,
             String description = nullptr) noexcept;
+  Exception(Nature nature, Durability durability, String file, int line,
+            String description = nullptr) noexcept;
   Exception(const Exception& other) noexcept;
   Exception(Exception&& other) = default;
   ~Exception() noexcept;
@@ -107,6 +109,7 @@ public:
   // callback stack.
 
 private:
+  String ownFile;
   const char* file;
   int line;
   Nature nature;