Add public interface for schema file parsing.

c++/src/capnp/compiler/compiler.h

@@ -34,10 +34,6 @@ namespace compiler {
 
 class Module: public ErrorReporter {
 public:
-  virtual kj::StringPtr getLocalName() const = 0;
-  // Typically, the absolute or cwd-relative path name of the module file, used in error messages.
-  // This is only for display purposes.
-
   virtual kj::StringPtr getSourceName() const = 0;
   // The name of the module file relative to the source tree.  Used to decide where to output
   // generated code and to form the `displayName` in the schema.

c++/src/capnp/compiler/md5.h

@@ -49,6 +49,9 @@ public:
   inline void update(kj::StringPtr data) {
     return update(data.asArray());
   }
+  inline void update(const char* data) {
+    return update(kj::StringPtr(data));
+  }
 
   kj::ArrayPtr<const kj::byte> finish();
   kj::StringPtr finishAsHex();

c++/src/capnp/compiler/module-loader.c++

@@ -216,7 +216,7 @@ public:
   ModuleImpl(const ModuleLoader::Impl& loader, kj::String localName, kj::String sourceName)
       : loader(loader), localName(kj::mv(localName)), sourceName(kj::mv(sourceName)) {}
 
-  kj::StringPtr getLocalName() const override {
+  kj::StringPtr getLocalName() const {
     return localName;
   }
 
@@ -275,7 +275,7 @@ public:
         message);
   }
 
-  bool hadErrors() const {
+  bool hadErrors() const override {
     return loader.getErrorReporter().hadErrors();
   }
 

c++/src/capnp/compiler/node-translator.c++

@@ -752,6 +752,7 @@ public:
       errorReporter.addErrorOn(ordinal,
           kj::str("Skipped ordinal @", expectedOrdinal, ".  Ordinals must be sequential with no "
                   "holes."));
+      expectedOrdinal = ordinal.getValue() + 1;
     } else {
       ++expectedOrdinal;
       lastOrdinalLocation = ordinal;

c++/src/capnp/schema-parser-test.c++

+// Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "schema-parser.h"
+#include <gtest/gtest.h>
+#include "test-util.h"
+#include <kj/debug.h>
+#include <map>
+
+namespace capnp {
+namespace {
+
+class FakeFileReader final: public SchemaFile::FileReader {
+public:
+  void add(kj::StringPtr name, kj::StringPtr content) {
+    files[name] = content;
+  }
+
+  bool exists(kj::StringPtr path) const override {
+    return files.count(path) > 0;
+  }
+
+  kj::Array<const char> read(kj::StringPtr path) const override {
+    auto iter = files.find(path);
+    KJ_ASSERT(iter != files.end(), "FakeFileReader has no such file.", path);
+    auto result = kj::heapArray<char>(iter->second.size());
+    memcpy(result.begin(), iter->second.begin(), iter->second.size());
+    return kj::mv(result);
+  }
+
+private:
+  std::map<kj::StringPtr, kj::StringPtr> files;
+};
+
+TEST(SchemaParser, Basic) {
+  SchemaParser parser;
+  FakeFileReader reader;
+
+  reader.add("src/foo/bar.capnp",
+      "@0x8123456789abcdef;\n"
+      "struct Bar {\n"
+      "  baz @0: import \"baz.capnp\".Baz;\n"
+      "  corge @1: import \"../qux/corge.capnp\".Corge;\n"
+      "  grault @2: import \"/grault.capnp\".Grault;\n"
+      "  garply @3: import \"/garply.capnp\".Garply;\n"
+      "}\n");
+  reader.add("src/foo/baz.capnp",
+      "@0x823456789abcdef1;\n"
+      "struct Baz {}\n");
+  reader.add("src/qux/corge.capnp",
+      "@0x83456789abcdef12;\n"
+      "struct Corge {}\n");
+  reader.add("/usr/include/grault.capnp",
+      "@0x8456789abcdef123;\n"
+      "struct Grault {}\n");
+  reader.add("/opt/include/grault.capnp",
+      "@0x8000000000000001;\n"
+      "struct WrongGrault {}\n");
+  reader.add("/usr/local/include/garply.capnp",
+      "@0x856789abcdef1234;\n"
+      "struct Garply {}\n");
+
+  kj::StringPtr importPath[] = {
+    "/usr/include", "/usr/local/include", "/opt/include"
+  };
+
+  ParsedSchema barSchema = parser.parseFile(SchemaFile::newDiskFile(
+      kj::str("foo2/bar2.capnp"), kj::str("src/foo/bar.capnp"), importPath, reader));
+
+  auto barProto = barSchema.getProto();
+  EXPECT_EQ(0x8123456789abcdefull, barProto.getId());
+  EXPECT_EQ("foo2/bar2.capnp", barProto.getDisplayName());
+  auto barImports = barProto.getBody().getFileNode().getImports();
+  ASSERT_EQ(4, barImports.size());
+  EXPECT_EQ("../qux/corge.capnp", barImports[0].getName());
+  EXPECT_EQ(0x83456789abcdef12ull, barImports[0].getId());
+  EXPECT_EQ("/garply.capnp", barImports[1].getName());
+  EXPECT_EQ(0x856789abcdef1234ull, barImports[1].getId());
+  EXPECT_EQ("/grault.capnp", barImports[2].getName());
+  EXPECT_EQ(0x8456789abcdef123ull, barImports[2].getId());
+  EXPECT_EQ("baz.capnp", barImports[3].getName());
+  EXPECT_EQ(0x823456789abcdef1ull, barImports[3].getId());
+
+  auto barStruct = barSchema.getNested("Bar");
+  auto barMembers = barStruct.asStruct().getMembers();
+  ASSERT_EQ(4, barMembers.size());
+  EXPECT_EQ("baz", barMembers[0].getProto().getName());
+  EXPECT_EQ("corge", barMembers[1].getProto().getName());
+  EXPECT_EQ("grault", barMembers[2].getProto().getName());
+  EXPECT_EQ("garply", barMembers[3].getProto().getName());
+
+  auto bazSchema = parser.parseFile(SchemaFile::newDiskFile(
+      kj::str("not/used/because/already/loaded"),
+      kj::str("src/foo/baz.capnp"), importPath, reader));
+  EXPECT_EQ(0x823456789abcdef1ull, bazSchema.getProto().getId());
+  EXPECT_EQ("foo2/baz.capnp", bazSchema.getProto().getDisplayName());
+  auto bazStruct = bazSchema.getNested("Baz").asStruct();
+  EXPECT_EQ(bazStruct, barStruct.getDependency(bazStruct.getProto().getId()));
+
+  auto corgeSchema = parser.parseFile(SchemaFile::newDiskFile(
+      kj::str("not/used/because/already/loaded"),
+      kj::str("src/qux/corge.capnp"), importPath, reader));
+  EXPECT_EQ(0x83456789abcdef12ull, corgeSchema.getProto().getId());
+  EXPECT_EQ("qux/corge.capnp", corgeSchema.getProto().getDisplayName());
+  auto corgeStruct = corgeSchema.getNested("Corge").asStruct();
+  EXPECT_EQ(corgeStruct, barStruct.getDependency(corgeStruct.getProto().getId()));
+
+  auto graultSchema = parser.parseFile(SchemaFile::newDiskFile(
+      kj::str("not/used/because/already/loaded"),
+      kj::str("/usr/include/grault.capnp"), importPath, reader));
+  EXPECT_EQ(0x8456789abcdef123ull, graultSchema.getProto().getId());
+  EXPECT_EQ("grault.capnp", graultSchema.getProto().getDisplayName());
+  auto graultStruct = graultSchema.getNested("Grault").asStruct();
+  EXPECT_EQ(graultStruct, barStruct.getDependency(graultStruct.getProto().getId()));
+
+  auto wrongGraultSchema = parser.parseFile(SchemaFile::newDiskFile(
+      kj::str("weird/display/name.capnp"),
+      kj::str("/opt/include/grault.capnp"), importPath, reader));
+  EXPECT_EQ(0x8000000000000001ull, wrongGraultSchema.getProto().getId());
+  EXPECT_EQ("weird/display/name.capnp", wrongGraultSchema.getProto().getDisplayName());
+}
+
+}  // namespace
+}  // namespace capnp

c++/src/capnp/schema-parser.h

+// Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef CAPNP_SCHEMA_PARSER_H_
+#define CAPNP_SCHEMA_PARSER_H_
+
+#include "schema-loader.h"
+#include <kj/string.h>
+
+namespace capnp {
+
+class ParsedSchema;
+class SchemaFile;
+
+class SchemaParser {
+  // Parses `.capnp` files to produce `Schema` objects.
+
+public:
+  SchemaParser();
+  ~SchemaParser() noexcept(false);
+
+  ParsedSchema parseDiskFile(kj::StringPtr displayName, kj::StringPtr diskPath,
+                             kj::ArrayPtr<const kj::StringPtr> importPath);
+  // Parse a file located on disk.  Throws an exception if the file dosen't exist.
+  //
+  // Parameters:
+  // * `displayName`:  The name that will appear in the file's schema node.  (If the file has
+  //   already been parsed, this will be ignored and the display name from the first time it was
+  //   parsed will be kept.)
+  // * `diskPath`:  The path to the file on disk.
+  // * `importPath`:  Directories to search when resolving absolute imports within this file
+  //   (imports that start with a `/`).  Must remain valid until the SchemaParser is destroyed.
+  //   (If the file has already been parsed, this will be ignored and the import path from the
+  //   first time it was parsed will be kept.)
+  //
+  // This method is a shortcut, equivalent to:
+  //     parser.parseFile(SchemaFile::newDiskFile(displayName, diskPath, importPath))`;
+  //
+  // This method throws an exception if any errors are encountered in the file or in anything the
+  // file depends on.  Note that merely importing another file does not count as a dependency on
+  // anything in the imported file -- only the imported types which are actually used are
+  // "dependencies".
+
+  ParsedSchema parseFile(kj::Own<SchemaFile>&& file);
+  // Advanced interface for parsing a file that may or may not be located in any global namespace.
+  // Most users will prefer `parseDiskFile()`.
+  //
+  // If the file has already been parsed (that is, a SchemaFile that compares equal to this one
+  // was parsed previously), the existing schema will be returned again.
+  //
+  // This method reports errors by calling SchemaFile::reportError() on the file where the error
+  // is located.  If that call does not throw an exception, `parseFile()` may in fact return
+  // normally.  In this case, the result is a best-effort attempt to compile the schema, but it
+  // may be invalid or corrupt, and using it for anything may cause exceptions to be thrown.
+
+private:
+  struct Impl;
+  class ModuleImpl;
+  kj::Own<Impl> impl;
+  mutable bool hadErrors = false;
+
+  const ModuleImpl& getModuleImpl(kj::Own<SchemaFile>&& file) const;
+
+  friend class ParsedSchema;
+};
+
+class ParsedSchema: public Schema {
+  // ParsedSchema is an extension of Schema which also has the ability to look up nested nodes
+  // by name.  See `SchemaParser`.
+
+public:
+  inline ParsedSchema(): parser(nullptr) {}
+
+  kj::Maybe<ParsedSchema> findNested(kj::StringPtr name);
+  // Gets the nested node with the given name, or returns null if there is no such nested
+  // declaration.
+
+  ParsedSchema getNested(kj::StringPtr name);
+  // Gets the nested node with the given name, or throws an exception if there is no such nested
+  // declaration.
+
+private:
+  inline ParsedSchema(Schema inner, const SchemaParser& parser): Schema(inner), parser(&parser) {}
+
+  const SchemaParser* parser;
+  friend class SchemaParser;
+};
+
+// =======================================================================================
+// Advanced API
+
+class SchemaFile {
+  // Abstract interface representing a schema file.  You can implement this yourself in order to
+  // gain more control over how the compiler resolves imports and reads files.  For the
+  // common case of files on disk or other global filesystem-like namespaces, use
+  // `SchemaFile::newDiskFile()`.
+
+public:
+  class FileReader {
+  public:
+    virtual bool exists(kj::StringPtr path) const = 0;
+    virtual kj::Array<const char> read(kj::StringPtr path) const = 0;
+  };
+
+  class DiskFileReader final: public FileReader {
+    // Implementation of FileReader that uses the local disk.  Files are read using mmap() if
+    // possible.
+
+  public:
+    static const DiskFileReader instance;
+
+    bool exists(kj::StringPtr path) const override;
+    kj::Array<const char> read(kj::StringPtr path) const override;
+  };
+
+  static kj::Own<SchemaFile> newDiskFile(
+      kj::StringPtr displayName, kj::StringPtr diskPath,
+      kj::ArrayPtr<const kj::StringPtr> importPath,
+      const FileReader& fileReader = DiskFileReader::instance);
+  // Construct a SchemaFile representing a file on disk (or located in the filesystem-like
+  // namespace represented by `fileReader`).
+  //
+  // Parameters:
+  // * `displayName`:  The name that will appear in the file's schema node.
+  // * `diskPath`:  The path to the file on disk.
+  // * `importPath`:  Directories to search when resolving absolute imports within this file
+  //   (imports that start with a `/`).  The array content must remain valid as long as the
+  //   SchemaFile exists (which is at least as long as the SchemaParser that parses it exists).
+  // * `fileReader`:  Allows you to use a filesystem other than the actual local disk.  Although,
+  //   if you find yourself using this, it may make more sense for you to implement SchemaFile
+  //   yourself.
+  //
+  // The SchemaFile compares equal to any other SchemaFile that has exactly the same disk path,
+  // after canonicalization.
+  //
+  // The SchemaFile will throw an exception if any errors are reported.
+
+  // -----------------------------------------------------------------
+  // For more control, you can implement this interface.
+
+  virtual kj::StringPtr getDisplayName() const = 0;
+  // Get the file's name, as it should appear in the schema.
+
+  virtual kj::Array<const char> readContent() const = 0;
+  // Read the file's entire content and return it as a byte array.
+
+  virtual kj::Maybe<kj::Own<SchemaFile>> import(kj::StringPtr path) const = 0;
+  // Resolve an import, relative to this file.
+  //
+  // `path` is exactly what appears between quotes after the `import` keyword in the source code.
+  // It is entirely up to the `SchemaFile` to decide how to map this to another file.  Typically,
+  // a leading '/' means that the file is an "absolute" path and is searched for in some list of
+  // schema file repositories.  On the other hand, a path that doesn't start with '/' is relative
+  // to the importing file.
+
+  virtual bool operator==(const SchemaFile& other) const = 0;
+  virtual bool operator!=(const SchemaFile& other) const = 0;
+  virtual size_t hashCode() const = 0;
+  // Compare two SchemaFiles to see if they refer to the same underlying file.  This is an
+  // optimization used to avoid the need to re-parse a file to check its ID.
+
+  struct SourcePos {
+    uint byte;
+    uint line;
+    uint column;
+  };
+  virtual void reportError(SourcePos start, SourcePos end, kj::StringPtr message) const = 0;
+  // Report that the file contains an error at the given interval.
+
+private:
+  class DiskSchemaFile;
+};
+
+}  // namespace capnp
+
+#endif  // CAPNP_SCHEMA_PARSER_H_

c++/src/kj/common.h

@@ -734,6 +734,10 @@ public:
   inline constexpr ArrayPtr(T* ptr, size_t size): ptr(ptr), size_(size) {}
   inline constexpr ArrayPtr(T* begin, T* end): ptr(begin), size_(end - begin) {}
 
+  template <size_t size>
+  inline constexpr ArrayPtr(T (&native)[size]): ptr(native), size_(size) {}
+  // Construct an ArrayPtr from a native C-style array.
+
   inline operator ArrayPtr<const T>() const {
     return ArrayPtr<const T>(ptr, size_);
   }

c++/src/kj/exception.c++

@@ -184,9 +184,25 @@ Exception::Exception(Nature nature, Durability durability, const char* file, int
 #endif
 }
 
+Exception::Exception(Nature nature, Durability durability, String file, int line,
+                     String description) noexcept
+    : ownFile(kj::mv(file)), file(ownFile.cStr()), line(line), nature(nature),
+      durability(durability), description(mv(description)) {
+#ifdef __CYGWIN__
+  traceCount = 0;
+#else
+  traceCount = backtrace(trace, 16);
+#endif
+}
+
 Exception::Exception(const Exception& other) noexcept
     : file(other.file), line(other.line), nature(other.nature), durability(other.durability),
-      description(str(other.description)), traceCount(other.traceCount) {
+      description(heapString(other.description)), traceCount(other.traceCount) {
+  if (file == other.ownFile.cStr()) {
+    ownFile = heapString(other.ownFile);
+    file = ownFile.cStr();
+  }
+
   memcpy(trace, other.trace, sizeof(trace[0]) * traceCount);
 
   KJ_IF_MAYBE(c, other.context) {

c++/src/kj/exception.h

@@ -69,6 +69,8 @@ public:
 
   Exception(Nature nature, Durability durability, const char* file, int line,
             String description = nullptr) noexcept;
+  Exception(Nature nature, Durability durability, String file, int line,
+            String description = nullptr) noexcept;
   Exception(const Exception& other) noexcept;
   Exception(Exception&& other) = default;
   ~Exception() noexcept;
@@ -107,6 +109,7 @@ public:
   // callback stack.
 
 private:
+  String ownFile;
   const char* file;
   int line;
   Nature nature;