Implement 'capnp encode' command which does the opposite of 'capnp decode'.

c++/src/capnp/compiler/capnp.c++

@@ -25,6 +25,7 @@
 #include "parser.h"
 #include "compiler.h"
 #include "module-loader.h"
+#include "node-translator.h"
 #include <capnp/pretty-print.h>
 #include <capnp/schema.capnp.h>
 #include <kj/vector.h>
@@ -99,8 +100,9 @@ public:
              .addSubCommand("id", KJ_BIND_METHOD(*this, getGenIdMain),
                             "Generate a new unique ID.")
              .addSubCommand("decode", KJ_BIND_METHOD(*this, getDecodeMain),
-                            "Decode binary Cap'n Proto message to text.");
-      // TODO(someday):  "encode" -- requires the ability to parse text format.
+                            "Decode binary Cap'n Proto message to text.")
+             .addSubCommand("encode", KJ_BIND_METHOD(*this, getEncodeMain),
+                            "Encode text Cap'n Proto message to binary.");
       addGlobalOptions(builder);
       return builder.build();
     }
@@ -150,6 +152,37 @@ public:
     return builder.build();
   }
 
+  kj::MainFunc getEncodeMain() {
+    // Only parse the schemas we actually need for decoding.
+    compileEagerness = Compiler::NODE;
+
+    // Drop annotations since we don't need them.  This avoids importing files like c++.capnp.
+    annotationFlag = Compiler::DROP_ANNOTATIONS;
+
+    kj::MainBuilder builder(context, VERSION_STRING,
+          "Encodes one or more textual Cap'n Proto messages to binary.  The messages have root "
+          "type <type> defined in <schema-file>.  Messages are read from standard input.  Each "
+          "mesage is a parenthesized struct literal, like the format used to specify constants "
+          "and default values of struct type in the schema language.  For example:\n"
+          "    (foo = 123, bar = \"hello\", baz = [true, false, true])\n"
+          "The input may contain any number of such values; each will be encoded as a separate "
+          "message.",
+          "Note that the current implementation reads the entire input into memory before "
+          "beginning to encode.  A better implementation would read and encode one message at "
+          "a time.");
+    addGlobalOptions(builder);
+    builder.addOption({'f', "flat"}, KJ_BIND_METHOD(*this, codeFlat),
+                      "Expect only one input value, serializing it as a single-segment message "
+                      "with no framing.")
+           .addOption({'p', "packed"}, KJ_BIND_METHOD(*this, codePacked),
+                      "Pack the output message with standard Cap'n Proto packing, which "
+                      "deflates zero-valued bytes.")
+           .expectArg("<schema-file>", KJ_BIND_METHOD(*this, addSource))
+           .expectArg("<type>", KJ_BIND_METHOD(*this, setRootType))
+           .callAfterParsing(KJ_BIND_METHOD(*this, encode));
+    return builder.build();
+  }
+
   void addGlobalOptions(kj::MainBuilder& builder) {
     builder.addOptionWithArg({'I', "import-path"}, KJ_BIND_METHOD(*this, addImportPath), "<dir>",
                              "Add <dir> to the list of directories searched for non-relative "
@@ -208,6 +241,19 @@ public:
       addStandardImportPaths = false;
     }
 
+    KJ_IF_MAYBE(module, loadModule(file)) {
+      uint64_t id = compiler->add(*module);
+      compiler->eagerlyCompile(id, compileEagerness);
+      sourceFiles.add(SourceFile { id, module->getSourceName(), &*module });
+    } else {
+      return "no such file";
+    }
+
+    return true;
+  }
+
+private:
+  kj::Maybe<const Module&> loadModule(kj::StringPtr file) {
     size_t longestPrefix = 0;
 
     for (auto& prefix: sourcePrefixes) {
@@ -217,17 +263,10 @@ public:
     }
 
     kj::StringPtr canonicalName = file.slice(longestPrefix);
-    KJ_IF_MAYBE(module, loader.loadModule(file, canonicalName)) {
-      uint64_t id = compiler->add(*module);
-      compiler->eagerlyCompile(id, compileEagerness);
-      sourceFiles.add(SourceFile { id, canonicalName, &*module });
-    } else {
-      return "no such file";
-    }
-
-    return true;
+    return loader.loadModule(file, canonicalName);
   }
 
+public:
   // =====================================================================================
   // "id" command
 
@@ -512,6 +551,150 @@ private:
     }
   }
 
+public:
+  // =====================================================================================
+
+  kj::MainBuilder::Validity encode() {
+    kj::Vector<char> allText;
+
+    {
+      kj::FdInputStream rawInput(STDIN_FILENO);
+      kj::BufferedInputStreamWrapper input(rawInput);
+
+      for (;;) {
+        auto buf = input.tryGetReadBuffer();
+        if (buf.size() == 0) break;
+        allText.addAll(reinterpret_cast<const char*>(buf.begin()),
+                       reinterpret_cast<const char*>(buf.end()));
+        input.skip(buf.size());
+      }
+    }
+
+    EncoderErrorReporter errorReporter(*this, allText);
+    MallocMessageBuilder arena;
+
+    // Lex the input.
+    auto lexedTokens = arena.initRoot<LexedTokens>();
+    lex(allText, lexedTokens, errorReporter);
+
+    // Set up the parser.
+    CapnpParser parser(arena.getOrphanage(), errorReporter);
+    auto tokens = lexedTokens.asReader().getTokens();
+    CapnpParser::ParserInput parserInput(tokens.begin(), tokens.end());
+
+    // Allocate some scratch space.
+    kj::Array<word> scratch = kj::heapArray<word>(8192);
+    memset(scratch.begin(), 0, scratch.size() * sizeof(word));
+
+    // Set up stuff for the ValueTranslator.
+    ValueResolverGlue resolver(compiler->getLoader(), errorReporter);
+    auto type = arena.getOrphanage().newOrphan<schema::Type>();
+    type.get().setStruct(rootType.getProto().getId());
+
+    // Set up output stream.
+    kj::FdOutputStream rawOutput(STDOUT_FILENO);
+    kj::BufferedOutputStreamWrapper output(rawOutput);
+
+    while (parserInput.getPosition() != tokens.end()) {
+      KJ_IF_MAYBE(expression, parser.getParsers().parenthesizedValueExpression(parserInput)) {
+        MallocMessageBuilder item(scratch);
+        ValueTranslator translator(resolver, errorReporter, item.getOrphanage());
+
+        KJ_IF_MAYBE(value, translator.compileValue(expression->getReader(), type.getReader())) {
+          writeEncoded(value->getReader().as<DynamicStruct>(), output);
+        } else {
+          // Errors were reported, so we'll exit with a failure status later.
+        }
+      } else {
+        auto best = parserInput.getBest();
+        if (best == tokens.end()) {
+          context.exitError("Premature EOF.");
+        } else {
+          errorReporter.addErrorOn(*best, "Parse error.");
+          context.exit();
+        }
+      }
+    }
+
+    return true;
+  }
+
+private:
+  void writeEncoded(DynamicStruct::Reader value, kj::BufferedOutputStream& output) {
+    // Always copy the message to a flat array so that the output is predictable (one segment,
+    // in canonical order).
+    size_t size = value.totalSizeInWords() + 1;
+    kj::Array<word> space = kj::heapArray<word>(size);
+    memset(space.begin(), 0, size * sizeof(word));
+    FlatMessageBuilder flatMessage(space);
+    flatMessage.setRoot(value);
+    flatMessage.requireFilled();
+
+    if (flat) {
+      output.write(space.begin(), space.size() * sizeof(word));
+    } else if (packed) {
+      writePackedMessage(output, flatMessage);
+    } else {
+      writeMessage(output, flatMessage);
+    }
+  }
+
+  class EncoderErrorReporter final: public ErrorReporter {
+  public:
+    EncoderErrorReporter(GlobalErrorReporter& globalReporter,
+                         kj::ArrayPtr<const char> content)
+      : globalReporter(globalReporter), lineBreaks(content) {}
+
+    void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) const override {
+      globalReporter.addError("<stdin>", lineBreaks.toSourcePos(startByte),
+                              lineBreaks.toSourcePos(endByte), message);
+    }
+
+    bool hadErrors() const override {
+      return globalReporter.hadErrors();
+    }
+
+  private:
+    GlobalErrorReporter& globalReporter;
+    LineBreakTable lineBreaks;
+  };
+
+  class ValueResolverGlue final: public ValueTranslator::Resolver {
+  public:
+    ValueResolverGlue(const SchemaLoader& loader, const ErrorReporter& errorReporter)
+        : loader(loader), errorReporter(errorReporter) {}
+
+    kj::Maybe<Schema> resolveType(uint64_t id) {
+      // Don't use tryGet() here because we shouldn't even be here if there were compile errors.
+      return loader.get(id);
+    }
+
+    kj::Maybe<DynamicValue::Reader> resolveConstant(DeclName::Reader name) {
+      auto base = name.getBase();
+      switch (base.which()) {
+        case DeclName::Base::RELATIVE_NAME: {
+          auto value = base.getRelativeName();
+          errorReporter.addErrorOn(value, kj::str("Not defined: ", value.getValue()));
+          return nullptr;
+        }
+        case DeclName::Base::ABSOLUTE_NAME: {
+          auto value = base.getAbsoluteName();
+          errorReporter.addErrorOn(value, kj::str("Not defined: ", value.getValue()));
+          return nullptr;
+        }
+        case DeclName::Base::IMPORT_NAME: {
+          auto value = base.getImportName();
+          errorReporter.addErrorOn(value, "Imports not allowed in encode input.");
+          return nullptr;
+        }
+      }
+    }
+
+  private:
+    const SchemaLoader& loader;
+    const ErrorReporter& errorReporter;
+  };
+
 public:
   // =====================================================================================
 

c++/src/capnp/compiler/error-reporter.c++

@@ -22,12 +22,49 @@
 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "error-reporter.h"
-#include <unistd.h>
+#include <kj/debug.h>
 
 namespace capnp {
 namespace compiler {
 
-ErrorReporter::~ErrorReporter() noexcept(false) {}
+namespace {
+
+template <typename T>
+static size_t findLargestElementBefore(const kj::Vector<T>& vec, const T& key) {
+  KJ_REQUIRE(vec.size() > 0 && vec[0] <= key);
+
+  size_t lower = 0;
+  size_t upper = vec.size();
+
+  while (upper - lower > 1) {
+    size_t mid = (lower + upper) / 2;
+    if (vec[mid] > key) {
+      upper = mid;
+    } else {
+      lower = mid;
+    }
+  }
+
+  return lower;
+}
+
+}  // namespace
+
+LineBreakTable::LineBreakTable(kj::ArrayPtr<const char> content)
+    : lineBreaks(content.size() / 40) {
+  lineBreaks.add(0);
+  for (const char* pos = content.begin(); pos < content.end(); ++pos) {
+    if (*pos == '\n') {
+      lineBreaks.add(pos + 1 - content.begin());
+    }
+  }
+}
+
+GlobalErrorReporter::SourcePos LineBreakTable::toSourcePos(uint32_t byteOffset) const {
+  uint line = findLargestElementBefore(lineBreaks, byteOffset);
+  uint col = byteOffset - lineBreaks[line];
+  return GlobalErrorReporter::SourcePos { byteOffset, line, col };
+}
 
 }  // namespace compiler
 }  // namespace capnp

c++/src/capnp/compiler/error-reporter.h

@@ -27,6 +27,7 @@
 #include "../common.h"
 #include <kj/string.h>
 #include <kj/exception.h>
+#include <kj/vector.h>
 
 namespace capnp {
 namespace compiler {
@@ -35,8 +36,6 @@ class ErrorReporter {
   // Callback for reporting errors within a particular file.
 
 public:
-  virtual ~ErrorReporter() noexcept(false);
-
   virtual void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) const = 0;
   // Report an error at the given location in the input text.  `startByte` and `endByte` indicate
   // the span of text that is erroneous.  They may be equal, in which case the parser was only
@@ -78,6 +77,18 @@ public:
   // of previous errors.
 };
 
+class LineBreakTable {
+public:
+  LineBreakTable(kj::ArrayPtr<const char> content);
+
+  GlobalErrorReporter::SourcePos toSourcePos(uint32_t byteOffset) const;
+
+private:
+  kj::Vector<uint> lineBreaks;
+  // Byte offsets of the first byte in each source line.  The first element is always zero.
+  // Initialized the first time the module is loaded.
+};
+
 }  // namespace compiler
 }  // namespace capnp
 

c++/src/capnp/compiler/module-loader.c++

@@ -42,25 +42,6 @@ namespace compiler {
 
 namespace {
 
-template <typename T>
-size_t findLargestElementBefore(const kj::Vector<T>& vec, const T& key) {
-  KJ_REQUIRE(vec.size() > 0 && vec[0] <= key);
-
-  size_t lower = 0;
-  size_t upper = vec.size();
-
-  while (upper - lower > 1) {
-    size_t mid = (lower + upper) / 2;
-    if (vec[mid] > key) {
-      upper = mid;
-    } else {
-      lower = mid;
-    }
-  }
-
-  return lower;
-}
-
 class MmapDisposer: public kj::ArrayDisposer {
 protected:
   void disposeImpl(void* firstElement, size_t elementSize, size_t elementCount,
@@ -228,7 +209,7 @@ private:
   kj::MutexGuarded<std::map<kj::StringPtr, kj::Own<Module>>> modules;
 };
 
-class ModuleLoader::ModuleImpl: public Module {
+class ModuleLoader::ModuleImpl final: public Module {
 public:
   ModuleImpl(const ModuleLoader::Impl& loader, kj::String localName, kj::String sourceName)
       : loader(loader), localName(kj::mv(localName)), sourceName(kj::mv(sourceName)) {}
@@ -244,15 +225,8 @@ public:
   Orphan<ParsedFile> loadContent(Orphanage orphanage) const override {
     kj::Array<const char> content = mmapForRead(localName);
 
-    lineBreaks.get([&](kj::SpaceFor<kj::Vector<uint>>& space) {
-      auto vec = space.construct(content.size() / 40);
-      vec->add(0);
-      for (const char* pos = content.begin(); pos < content.end(); ++pos) {
-        if (*pos == '\n') {
-          vec->add(pos + 1 - content.begin());
-        }
-      }
-      return vec;
+    lineBreaks.get([&](kj::SpaceFor<LineBreakTable>& space) {
+      return space.construct(content);
     });
 
     MallocMessageBuilder lexedBuilder;
@@ -274,22 +248,12 @@ public:
 
   void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) const override {
     auto& lines = lineBreaks.get(
-        [](kj::SpaceFor<kj::Vector<uint>>& space) {
+        [](kj::SpaceFor<LineBreakTable>& space) -> kj::Own<LineBreakTable> {
           KJ_FAIL_REQUIRE("Can't report errors until loadContent() is called.");
-          return space.construct();
         });
 
-    // TODO(someday):  This counts tabs as single characters.  Do we care?
-    uint startLine = findLargestElementBefore(lines, startByte);
-    uint startCol = startByte - lines[startLine];
-    uint endLine = findLargestElementBefore(lines, endByte);
-    uint endCol = endByte - lines[endLine];
-
     loader.getErrorReporter().addError(
-        localName,
-        GlobalErrorReporter::SourcePos { startByte, startLine, startCol },
-        GlobalErrorReporter::SourcePos { endByte, endLine, endCol },
-        message);
+        localName, lines.toSourcePos(startByte), lines.toSourcePos(endByte), message);
   }
 
   bool hadErrors() const override {
@@ -301,9 +265,7 @@ private:
   kj::String localName;
   kj::String sourceName;
 
-  kj::Lazy<kj::Vector<uint>> lineBreaks;
-  // Byte offsets of the first byte in each source line.  The first element is always zero.
-  // Initialized the first time the module is loaded.
+  kj::Lazy<LineBreakTable> lineBreaks;
 };
 
 // =======================================================================================

c++/src/capnp/compiler/node-translator.h

@@ -169,35 +169,49 @@ private:
                     schema::Value::Builder target, bool isBootstrap);
   // Interprets the value expression and initializes `target` with the result.
 
+  kj::Maybe<DynamicValue::Reader> readConstant(DeclName::Reader name, bool isBootstrap);
+  // Get the value of the given constant.  May return null if some error occurs, which will already
+  // have been reported.
+
+  kj::Maybe<ListSchema> makeListSchemaOf(schema::Type::Reader elementType);
+  // Construct a list schema representing a list of elements of the given type.  May return null if
+  // some error occurs, which will already have been reported.
+
+  Orphan<List<schema::Annotation>> compileAnnotationApplications(
+      List<Declaration::AnnotationApplication>::Reader annotations,
+      kj::StringPtr targetsFlagName);
+};
+
+class ValueTranslator {
+public:
+  class Resolver {
+  public:
+    virtual kj::Maybe<Schema> resolveType(uint64_t id) = 0;
+    virtual kj::Maybe<DynamicValue::Reader> resolveConstant(DeclName::Reader name) = 0;
+  };
+
+  ValueTranslator(Resolver& resolver, const ErrorReporter& errorReporter, Orphanage orphanage)
+      : resolver(resolver), errorReporter(errorReporter), orphanage(orphanage) {}
+
   kj::Maybe<Orphan<DynamicValue>> compileValue(
-      ValueExpression::Reader src, schema::Type::Reader type, bool isBootstrap);
-  // Compile the given value as the given type.  Returns null if there was an error, including
-  // if the value doesn't match the type.
+      ValueExpression::Reader src, schema::Type::Reader type);
+
+private:
+  Resolver& resolver;
+  const ErrorReporter& errorReporter;
+  Orphanage orphanage;
 
-  Orphan<DynamicValue> compileValueInner(ValueExpression::Reader src, schema::Type::Reader type,
-                                         bool isBootstrap);
+  Orphan<DynamicValue> compileValueInner(ValueExpression::Reader src, schema::Type::Reader type);
   // Helper for compileValue().
 
   void fillStructValue(DynamicStruct::Builder builder,
-                       List<ValueExpression::FieldAssignment>::Reader assignments,
-                       bool isBootstrap);
+                       List<ValueExpression::FieldAssignment>::Reader assignments);
   // Interprets the given assignments and uses them to fill in the given struct builder.
 
   kj::String makeNodeName(uint64_t id);
   kj::String makeTypeName(schema::Type::Reader type);
 
-  kj::Maybe<DynamicValue::Reader> readConstant(DeclName::Reader name, bool isBootstrap,
-                                               ValueExpression::Reader errorLocation);
-  // Get the value of the given constant.  May return null if some error occurs, which will already
-  // have been reported.
-
   kj::Maybe<ListSchema> makeListSchemaOf(schema::Type::Reader elementType);
-  // Construct a list schema representing a list of elements of the given type.  May return null if
-  // some error occurs, which will already have been reported.
-
-  Orphan<List<schema::Annotation>> compileAnnotationApplications(
-      List<Declaration::AnnotationApplication>::Reader annotations,
-      kj::StringPtr targetsFlagName);
 };
 
 }  // namespace compiler

c++/src/capnp/dynamic.c++

@@ -1659,6 +1659,19 @@ DynamicStruct::Builder MessageBuilder::getRoot<DynamicStruct>(StructSchema schem
   return DynamicStruct::Builder(schema, getRoot(structSizeFromSchema(schema)));
 }
 
+template <>
+void MessageBuilder::setRoot<DynamicStruct::Reader>(DynamicStruct::Reader&& value) {
+  setRootInternal(value.reader);
+}
+template <>
+void MessageBuilder::setRoot<const DynamicStruct::Reader&>(const DynamicStruct::Reader& value) {
+  setRootInternal(value.reader);
+}
+template <>
+void MessageBuilder::setRoot<DynamicStruct::Reader&>(DynamicStruct::Reader& value) {
+  setRootInternal(value.reader);
+}
+
 namespace _ {  // private
 
 DynamicStruct::Reader PointerHelpers<DynamicStruct, Kind::UNKNOWN>::getDynamic(

c++/src/capnp/dynamic.h

@@ -216,6 +216,8 @@ public:
   template <typename T, typename = kj::EnableIf<kind<FromReader<T>>() == Kind::STRUCT>>
   inline Reader(T&& value): Reader(toDynamic(value)) {}
 
+  inline size_t totalSizeInWords() const { return reader.totalSize() / ::capnp::WORDS; }
+
   template <typename T>
   typename T::Reader as() const;
   // Convert the dynamic struct to its compiled-in type.
@@ -279,6 +281,8 @@ public:
   template <typename T, typename = kj::EnableIf<kind<FromBuilder<T>>() == Kind::STRUCT>>
   inline Builder(T&& value): Builder(toDynamic(value)) {}
 
+  inline size_t totalSizeInWords() const { return asReader().totalSizeInWords(); }
+
   template <typename T>
   typename T::Builder as();
   // Cast to a particular struct type.
@@ -926,6 +930,12 @@ template <>
 DynamicStruct::Builder MessageBuilder::initRoot<DynamicStruct>(StructSchema schema);
 template <>
 DynamicStruct::Builder MessageBuilder::getRoot<DynamicStruct>(StructSchema schema);
+template <>
+void MessageBuilder::setRoot<DynamicStruct::Reader>(DynamicStruct::Reader&& value);
+template <>
+void MessageBuilder::setRoot<const DynamicStruct::Reader&>(const DynamicStruct::Reader& value);
+template <>
+void MessageBuilder::setRoot<DynamicStruct::Reader&>(DynamicStruct::Reader& value);
 
 namespace _ {  // private
 

c++/src/capnp/schema-parser.c++

@@ -67,7 +67,7 @@ size_t findLargestElementBefore(const kj::Vector<T>& vec, const T& key) {
 
 // =======================================================================================
 
-class SchemaParser::ModuleImpl: public compiler::Module {
+class SchemaParser::ModuleImpl final: public compiler::Module {
 public:
   ModuleImpl(const SchemaParser& parser, kj::Own<const SchemaFile>&& file)
       : parser(parser), file(kj::mv(file)) {}