Compiler core WIP.

c++/src/capnp/compiler/capnpc2.c++

@@ -31,10 +31,10 @@
 #include "../message.h"
 #include <iostream>
 
-class CoutErrorReporter: public capnp::compiler::ErrorReporter {
+class CerrErrorReporter: public capnp::compiler::ErrorReporter {
 public:
-  void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) override {
-    std::cout << "input:" << startByte << "-" << endByte << ": " << message.cStr() << std::endl;
+  void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) const override {
+    std::cerr << "input:" << startByte << "-" << endByte << ": " << message.cStr() << std::endl;
   }
 };
 
@@ -52,7 +52,7 @@ int main(int argc, char* argv[]) {
     input.addAll(buffer, buffer + n);
   }
 
-  CoutErrorReporter errorReporter;
+  CerrErrorReporter errorReporter;
 
   std::cout << "=========================================================================\n"
             << "lex\n"

c++/src/capnp/compiler/compiler.h

+// Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef CAPNP_COMPILER_COMPILER_H_
+#define CAPNP_COMPILER_COMPILER_H_
+
+#include <capnp/compiler/grammar.capnp.h>
+#include <capnp/schema.capnp.h>
+#include <capnp/schema-loader.h>
+#include "error-reporter.h"
+
+namespace capnp {
+namespace compiler {
+
+template <typename ContentType>
+class Module: public ErrorReporter {
+public:
+  virtual kj::StringPtr getLocalName() const = 0;
+  // Typically, the absolute or cwd-relative path name of the module file, used in error messages.
+  // This is only for display purposes.
+
+  virtual kj::StringPtr getSourceName() const = 0;
+  // The name of the module file relative to the source tree.  Used to decide where to output
+  // generated code and to form the `displayName` in the schema.
+
+  virtual ContentType loadContent(Orphanage orphanage) const = 0;
+  // Loads the module content, using the given orphanage to allocate objects if necessary.
+
+  virtual kj::Maybe<const Module&> importRelative(kj::StringPtr importPath) const = 0;
+  // Find another module, relative to this one.  Importing the same logical module twice should
+  // produce the exact same object, comparable by identity.  These objects are owned by some
+  // outside pool that outlives the Compiler instance.
+};
+
+class Compiler {
+  // Cross-links separate modules (schema files) and translates them into schema nodes.
+
+public:
+  explicit Compiler();
+  ~Compiler();
+  KJ_DISALLOW_COPY(Compiler);
+
+  enum Mode {
+    EAGER,
+    // Completely traverse the module's parse tree and translate it into schema nodes before
+    // returning from add().
+
+    LAZY
+    // Only interpret the module's definitions when they are requested.  The main advantage of this
+    // mode is that imports will only be loaded if they are actually needed.
+    //
+    // Since the parse tree is traversed lazily, any particular schema node only becomes findable
+    // by ID (using the SchemaLoader) once one of its neighbors in the graph has been examined.
+    // As long as you are only traversing the graph -- only looking up IDs that you obtained from
+    // other schema nodes from the same loader -- you shouldn't be able to tell the difference.
+    // But if you receive IDs from some external source and want to look those up, you'd better
+    // use EAGER mode.
+  };
+
+  Schema add(Module<ParsedFile::Reader>& module, Mode mode) const;
+  // Add a module to the Compiler, returning its root Schema object.
+
+  const SchemaLoader& getLoader() const;
+  // Get a SchemaLoader backed by this compiler.  Schema nodes will be lazily constructed as you
+  // traverse them using this loader.
+
+private:
+  class Impl;
+  kj::Own<Impl> impl;
+
+  class CompiledModule;
+  class Node;
+  class Alias;
+};
+
+}  // namespace compiler
+}  // namespace capnp
+
+#endif  // CAPNP_COMPILER_COMPILER_H_

c++/src/capnp/compiler/error-reporter.h

@@ -34,10 +34,18 @@ class ErrorReporter {
 public:
   virtual ~ErrorReporter() noexcept(false);
 
-  virtual void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) = 0;
+  virtual void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) const = 0;
   // Report an error at the given location in the input text.  `startByte` and `endByte` indicate
   // the span of text that is erroneous.  They may be equal, in which case the parser was only
   // able to identify where the error begins, not where it ends.
+
+  template <typename T>
+  inline void addErrorOn(T&& decl, kj::StringPtr message) const {
+    // Works for any `T` that defines `getStartByte()` and `getEndByte()` methods, which many
+    // of the Cap'n Proto types defined in `grammar.capnp` do.
+
+    addError(decl.getStartByte(), decl.getEndByte(), message);
+  }
 };
 
 }  // namespace compiler

c++/src/capnp/compiler/grammar.capnp

@@ -139,6 +139,7 @@ struct Declaration {
   docComment @20 :Text;
 
   body @6 union {
+    fileDecl @24 :File;
     usingDecl @7 :Using;
     constDecl @8 :Const;
     enumDecl @9 :Enum;
@@ -154,10 +155,38 @@ struct Declaration {
     nakedId @21 :LocatedInteger;
     nakedAnnotation @22 :AnnotationApplication;
     # A floating UID or annotation (allowed at the file top level).
+
+    # The following declaration types are not produced by the parser, but are declared here
+    # so that the compiler can handle symbol name lookups more uniformly.
+    #
+    # New union members added here will magically become visible in the global scope.
+    # "builtinFoo" becomes visible as "Foo", while "builtinFooValue" becomes visible as "foo".
+    builtinVoid @25 :Void;
+    builtinBool @26 :Void;
+    builtinInt8 @27 :Void;
+    builtinInt16 @28 :Void;
+    builtinInt32 @29 :Void;
+    builtinInt64 @30 :Void;
+    builtinUInt8 @31 :Void;
+    builtinUInt16 @32 :Void;
+    builtinUInt32 @33 :Void;
+    builtinUInt64 @34 :Void;
+    builtinFloat32 @35 :Void;
+    builtinFloat64 @36 :Void;
+    builtinText @37 :Void;
+    builtinData @38 :Void;
+    builtinList @39 :Void;
+    builtinObject @40 :Void;
+
+    builtinTrueValue @41 :Void;
+    builtinFalseValue @42 :Void;
+    builtinVoidValue @43 :Void;
   }
 
+  struct File {}
+
   struct Using {
-    target @0 :TypeExpression;
+    target @0 :DeclName;
   }
 
   struct Const {

c++/src/capnp/compiler/lexer-test.c++

@@ -31,7 +31,7 @@ namespace {
 
 class TestFailingErrorReporter: public ErrorReporter {
 public:
-  void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) override {
+  void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) const override {
     ADD_FAILURE() << "Parse failed: (" << startByte << "-" << endByte << ") " << message.cStr();
   }
 };

c++/src/capnp/compiler/node-translator.h

+// Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef CAPNP_COMPILER_NODE_TRANSLATOR_H_
+#define CAPNP_COMPILER_NODE_TRANSLATOR_H_
+
+#include <capnp/orphan.h>
+#include <capnp/compiler/grammar.capnp.h>
+#include <capnp/schema.capnp.h>
+#include <capnp/schema-loader.h>
+#include <capnp/dynamic.h>
+#include <kj/vector.h>
+#include "error-reporter.h"
+
+namespace capnp {
+namespace compiler {
+
+class NodeTranslator {
+  // Translates one node in the schema from AST form to final schema form.  A "node" is anything
+  // that has a unique ID, such as structs, enums, constants, and annotations, but not fields,
+  // unions, enumerants, or methods (the latter set have 16-bit ordinals but not 64-bit global IDs).
+
+public:
+  class Resolver {
+    // Callback class used to find other nodes relative to this one.
+
+  public:
+    struct ResolvedName {
+      uint64_t id;
+      Declaration::Body::Which kind;
+    };
+
+    virtual kj::Maybe<ResolvedName> resolve(const DeclName::Reader& name) const = 0;
+    // Look up the given name, relative to this node, and return basic information about the
+    // target.
+
+    virtual kj::Maybe<Schema> resolveMaybeBootstrapSchema(uint64_t id) const = 0;
+    // Get the schema for the given ID.  Returning either a bootstrap schema or a final schema
+    // is acceptable.
+
+    virtual kj::Maybe<Schema> resolveFinalSchema(uint64_t id) const = 0;
+    // Get the final schema for the given ID.  A bootstrap schema is not acceptable.
+  };
+
+  NodeTranslator(const Resolver& resolver, const ErrorReporter& errorReporter,
+                 const Declaration::Reader& decl, Orphan<schema::Node> wipNode);
+  // Construct a NodeTranslator to translate the given declaration.  The wipNode starts out with
+  // `displayName`, `id`, `scopeId`, and `nestedNodes` already initialized.  The `NodeTranslator`
+  // fills in the rest.
+
+  schema::Node::Reader getBootstrapNode() { return wipNode.getReader(); }
+  // Get an incomplete version of the node in which pointer-typed value expressions have not yet
+  // been translated.  Instead, for all `schema.Value` objects representing pointer-type values,
+  // the value is set to an appropriate "empty" value.  This version of the schema can be used to
+  // bootstrap the dynamic API which can then in turn be used to encode the missing complex values.
+  //
+  // If the final node has already been built, this will actually return the final node (in fact,
+  // it's the same node object).
+
+  schema::Node::Reader finish();
+  // Finish translating the node (including filling in all the pieces that are missing from the
+  // bootstrap node) and return it.
+
+private:
+  const Resolver& resolver;
+  const ErrorReporter& errorReporter;
+
+  Orphan<schema::Node> wipNode;
+  // The work-in-progress schema node.
+
+  struct UnfinishedValue {
+    ValueExpression::Reader source;
+    schema::Type::Reader type;
+    schema::Value::Builder target;
+  };
+  kj::Vector<UnfinishedValue> unfinishedValues;
+  // List of values in `wipNode` which have not yet been interpreted, because they are structs
+  // or lists and as such interpreting them require using the types' schemas (to take advantage
+  // of the dynamic API).  Once bootstrap schemas have been built, they can be used to interpret
+  // these values.
+
+  void compileNode(Declaration::Reader decl, schema::Node::Builder builder);
+
+  void checkMembers(List<Declaration>::Reader nestedDecls, Declaration::Body::Which parentKind);
+  // Check the given member list for errors, including detecting duplicate names and detecting
+  // out-of-place declarations.
+
+  void disallowNested(List<Declaration>::Reader nestedDecls);
+  // Complain if the nested decl list is non-empty.
+
+  void compileFile(Declaration::Reader decl, schema::FileNode::Builder builder);
+  void compileConst(Declaration::Const::Reader decl, schema::ConstNode::Builder builder);
+  void compileAnnotation(Declaration::Annotation::Reader decl,
+                         schema::AnnotationNode::Builder builder);
+
+  class DuplicateOrdinalDetector;
+  class StructLayout;
+  class StructTranslator;
+
+  void compileEnum(Declaration::Enum::Reader decl, List<Declaration>::Reader members,
+                   schema::EnumNode::Builder builder);
+  void compileStruct(Declaration::Struct::Reader decl, List<Declaration>::Reader members,
+                     schema::StructNode::Builder builder);
+  void compileInterface(Declaration::Interface::Reader decl, List<Declaration>::Reader members,
+                        schema::InterfaceNode::Builder builder);
+  // The `members` arrays contain only members with ordinal numbers, in code order.  Other members
+  // are handled elsewhere.
+
+  bool compileType(TypeExpression::Reader source, schema::Type::Builder target);
+  // Returns false if there was a problem, in which case value expressions of this type should
+  // not be parsed.
+
+  void compileDefaultDefaultValue(schema::Type::Reader type, schema::Value::Builder target);
+  // Initializes `target` to contain the "default default" value for `type`.
+
+  void compileBootstrapValue(ValueExpression::Reader source, schema::Type::Reader type,
+                             schema::Value::Builder target);
+  // Interprets the value expression and initializes `target` with the result.  If some parts of
+  // the value cannot be built at bootstrap time, they'll be added to `unfinishedValues`
+  // automatically for later processing.
+
+  void compileFinalValue(ValueExpression::Reader source,
+                         schema::Type::Reader type, schema::Value::Builder target);
+  // Compile a previously-unfinished value.  See `unfinishedValues`.
+
+  Orphan<List<schema::Annotation>> compileAnnotationApplications(
+      List<Declaration::AnnotationApplication>::Reader annotations,
+      kj::StringPtr targetsFlagName);
+};
+
+}  // namespace compiler
+}  // namespace capnp
+
+#endif  // CAPNP_COMPILER_NODE_TRANSLATOR_H_

c++/src/capnp/compiler/parser.c++

@@ -57,6 +57,7 @@ void parseFile(List<Statement>::Reader statements, ParsedFile::Builder result,
   kj::Vector<Orphan<Declaration::AnnotationApplication>> annotations;
 
   auto fileDecl = result.getRoot();
+  fileDecl.getBody().initFileDecl();
 
   for (auto statement: statements) {
     KJ_IF_MAYBE(decl, parser.parseStatement(statement, parser.getParsers().fileLevelDecl)) {
@@ -556,13 +557,13 @@ CapnpParser::CapnpParser(Orphanage orphanageParam, ErrorReporter& errorReporterP
   // -----------------------------------------------------------------
 
   parsers.usingDecl = arena.copy(p::transform(
-      p::sequence(keyword("using"), identifier, op("="), parsers.typeExpression),
-      [this](Located<Text::Reader>&& name, Orphan<TypeExpression>&& type) -> DeclParserResult {
+      p::sequence(keyword("using"), identifier, op("="), parsers.declName),
+      [this](Located<Text::Reader>&& name, Orphan<DeclName>&& target) -> DeclParserResult {
         auto decl = orphanage.newOrphan<Declaration>();
         auto builder = decl.get();
         name.copyTo(builder.initName());
         // no id, no annotations for using decl
-        builder.getBody().initUsingDecl().adoptTarget(kj::mv(type));
+        builder.getBody().initUsingDecl().adoptTarget(kj::mv(target));
         return DeclParserResult(kj::mv(decl));
       }));
 

c++/src/capnp/schema.capnp

@@ -151,7 +151,17 @@ struct FileNode {
   imports @0 :List(Import);
   struct Import {
     id @0 :Id;
-    # ID of the imported file.
+    # DEPRECATED:  ID of the imported file.  This is no longer filled in because it is hostile to
+    # lazy importing:  since this import list appears in the FileNode, and since the FileNode must
+    # necessarily be cosntructed if any schemas in the file are used, the implication of listing
+    # import IDs here is that if a schema file is used at all, all of its imports must be parsed,
+    # just to get their IDs.  We'd much rather delay parsing a file until something inside it is
+    # actually used.
+    #
+    # In any case, this import list's main reason for existing is to make it easy to generate
+    # the appropriate #include statements in C++.  The IDs of files aren't needed for that.
+    #
+    # TODO(someday):  Perhaps provide an alternative way to identify the remote file.
 
     name @1 :Text;
     # Name which *this* file used to refer to the foreign file.  This may be a relative name.
@@ -197,6 +207,11 @@ struct StructNode {
     name @0 :Text;
 
     ordinal @1 :UInt16;
+    # For fields, the ordinal number.  For unions, if an explicit ordinal was given, that number.
+    # Otherwise, for unions and groups, this is the ordinal of the lowest-numbered field in the
+    # union/group.
+    #
+    # TODO(someday):  When revamping the meta-schema, move this into Field.
 
     codeOrder @2 :UInt16;
     # Indicates where this member appeared in the code, relative to other members.
@@ -214,6 +229,7 @@ struct StructNode {
 
       fieldMember @5 :Field;
       unionMember @6 :Union;
+      groupMember @7 :Group;
     }
   }
 
@@ -236,6 +252,10 @@ struct StructNode {
     # consumers should skip member types that they don't understand.  The first member in this list
     # gets discriminant value zero, the next gets one, and so on.
   }
+
+  struct Group {
+    members @0 :List(Member);
+  }
 }
 
 struct EnumNode {

c++/src/kj/array.h

@@ -325,10 +325,10 @@ public:
   }
 
   template <typename... Params>
-  void add(Params&&... params) {
+  T& add(Params&&... params) {
     KJ_IREQUIRE(pos < endPtr, "Added too many elements to ArrayBuilder.");
     ctor(*pos, kj::fwd<Params>(params)...);
-    ++pos;
+    return *pos++;
   }
 
   template <typename Container>

c++/src/kj/vector.h

@@ -70,9 +70,9 @@ public:
   }
 
   template <typename... Params>
-  inline void add(Params&&... params) {
+  inline T& add(Params&&... params) {
     if (builder.isFull()) grow();
-    builder.add(kj::fwd<Params>(params)...);
+    return builder.add(kj::fwd<Params>(params)...);
   }
 
   template <typename Iterator>