Compiler actually running plugins! Hooked it up to capnpc-capnp (the loopback…

Compiler actually running plugins!  Hooked it up to capnpc-capnp (the loopback plugin).  Working on getting the output identical to the old capnpc...  still a few bugs left.

c++/src/capnp/compiler/capnpc2.c++

@@ -26,6 +26,7 @@
 #include "compiler.h"
 #include "module-loader.h"
 #include <capnp/pretty-print.h>
+#include <capnp/schema.capnp.h>
 #include <kj/vector.h>
 #include <kj/io.h>
 #include <unistd.h>
@@ -33,6 +34,10 @@
 #include "../message.h"
 #include <iostream>
 #include <kj/main.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <capnp/serialize.h>
 
 namespace capnp {
 namespace compiler {
@@ -68,13 +73,36 @@ public:
           context, "Cap'n Proto compiler version 0.2",
           "Compiles Cap'n Proto schema files and generates corresponding source code in one or "
           "more languages.")
+        .addOptionWithArg({'o', "output"}, KJ_BIND_METHOD(*this, addOutput), "<lang>[:<dir>]",
+                          "Generate source code for language <lang> in directory <dir> (default: "
+                          "current directory).  <lang> actually specifies a plugin to use.  If "
+                          "<lang> is a simple word, the compiler for a plugin called "
+                          "'capnpc-<lang>' in $PATH.  If <lang> is a file path containing slashes, "
+                          "it is interpreted as the exact plugin executable file name, and $PATH "
+                          "is not searched.")
         .expectOneOrMoreArgs("source", KJ_BIND_METHOD(*this, addSource))
+        .callAfterParsing(KJ_BIND_METHOD(*this, generateOutput))
         .build();
   }
 
+  kj::MainBuilder::Validity addOutput(kj::StringPtr spec) {
+    KJ_IF_MAYBE(split, spec.findFirst(':')) {
+      kj::StringPtr dir = spec.slice(*split + 1);
+      struct stat stats;
+      if (stat(dir.cStr(), &stats) < 0 || !S_ISDIR(stats.st_mode)) {
+        return "output location is inaccessible or is not a directory";
+      }
+      outputs.add(OutputDirective { spec.slice(0, *split), dir });
+    } else {
+      outputs.add(OutputDirective { spec.asArray(), nullptr });
+    }
+
+    return true;
+  }
+
   kj::MainBuilder::Validity addSource(kj::StringPtr file) {
     KJ_IF_MAYBE(module, loader.loadModule(file, file)) {
-      compiler.add(*module, Compiler::EAGER);
+      sourceIds.add(compiler.add(*module, Compiler::EAGER));
     } else {
       return "no such file";
     }
@@ -82,68 +110,94 @@ public:
     return true;
   }
 
+  kj::MainBuilder::Validity generateOutput() {
+    if (outputs.size() == 0) {
+      return "no outputs specified";
+    }
+
+    MallocMessageBuilder message;
+    auto request = message.initRoot<schema::CodeGeneratorRequest>();
+
+    auto schemas = compiler.getLoader().getAllLoaded();
+    auto nodes = request.initNodes(schemas.size());
+    for (size_t i = 0; i < schemas.size(); i++) {
+      nodes.setWithCaveats(i, schemas[i].getProto());
+    }
+
+    auto requestedFiles = request.initRequestedFiles(sourceIds.size());
+    for (size_t i = 0; i < sourceIds.size(); i++) {
+      requestedFiles.set(i, sourceIds[i]);
+    }
+
+    for (auto& output: outputs) {
+      int pipeFds[2];
+      KJ_SYSCALL(pipe(pipeFds));
+
+      kj::String exeName;
+      bool shouldSearchPath = true;
+      for (char c: output.name) {
+        if (c == '/') {
+          shouldSearchPath = false;
+          break;
+        }
+      }
+      if (shouldSearchPath) {
+        exeName = kj::str("capnpc-", output.name);
+      } else {
+        exeName = kj::heapString(output.name);
+      }
+
+      pid_t child;
+      KJ_SYSCALL(child = fork());
+      if (child == 0) {
+        // I am the child!
+        KJ_SYSCALL(close(pipeFds[1]));
+        KJ_SYSCALL(dup2(pipeFds[0], STDIN_FILENO));
+        KJ_SYSCALL(close(pipeFds[0]));
+
+        if (output.dir != nullptr) {
+          KJ_SYSCALL(chdir(output.dir.cStr()), output.dir);
+        }
+
+        if (shouldSearchPath) {
+          KJ_SYSCALL(execlp(exeName.cStr(), exeName.cStr(), nullptr));
+        } else {
+          KJ_SYSCALL(execl(exeName.cStr(), exeName.cStr(), nullptr));
+        }
+
+        KJ_FAIL_ASSERT("execlp() returned?");
+      }
+
+      KJ_SYSCALL(close(pipeFds[0]));
+
+      writeMessageToFd(pipeFds[1], message);
+      KJ_SYSCALL(close(pipeFds[1]));
+
+      int status;
+      KJ_SYSCALL(waitpid(child, &status, 0));
+      if (WIFSIGNALED(status)) {
+        context.error(kj::str(exeName, ": plugin failed: ", strsignal(WTERMSIG(status))));
+      } else if (WIFEXITED(status) && WEXITSTATUS(status) != 0) {
+        context.error(kj::str(exeName, ": plugin failed: exit code ", WEXITSTATUS(status)));
+      }
+    }
+
+    return true;
+  }
+
 private:
   kj::ProcessContext& context;
   ModuleLoader loader;
   Compiler compiler;
-};
 
-int main(int argc, char* argv[]) {
-  // Eventually this will be capnpc.  For now it's just a dummy program that tests parsing.
-
-//  kj::Vector<char> input;
-//  char buffer[4096];
-//  for (;;) {
-//    ssize_t n;
-//    KJ_SYSCALL(n = read(STDIN_FILENO, buffer, sizeof(buffer)));
-//    if (n == 0) {
-//      break;
-//    }
-//    input.addAll(buffer, buffer + n);
-//  }
-
-  kj::StringPtr input =
-      "@0x8e001c75f6ff54c8;\n"
-      "struct Foo { bar @0 :Int32 = 123; baz @1 :Text; }\n"
-      "struct Qux { foo @0 :List(Int32) = [12, 34]; }";
-
-  DummyModule module;
-
-  std::cout << "=========================================================================\n"
-            << "lex\n"
-            << "========================================================================="
-            << std::endl;
-
-  capnp::MallocMessageBuilder lexerArena;
-  auto lexedFile = lexerArena.initRoot<capnp::compiler::LexedStatements>();
-  capnp::compiler::lex(input, lexedFile, module);
-  std::cout << capnp::prettyPrint(lexedFile).cStr() << std::endl;
-
-  std::cout << "=========================================================================\n"
-            << "parse\n"
-            << "========================================================================="
-            << std::endl;
-
-  capnp::MallocMessageBuilder parserArena;
-  auto parsedFile = parserArena.initRoot<capnp::compiler::ParsedFile>();
-  capnp::compiler::parseFile(lexedFile.getStatements(), parsedFile, module);
-  std::cout << capnp::prettyPrint(parsedFile).cStr() << std::endl;
-
-  std::cout << "=========================================================================\n"
-            << "compile\n"
-            << "========================================================================="
-            << std::endl;
-
-  module.content = parsedFile.asReader();
-  capnp::compiler::Compiler compiler;
-  compiler.add(module, capnp::compiler::Compiler::EAGER);
-
-  for (auto schema: compiler.getLoader().getAllLoaded()) {
-    std::cout << capnp::prettyPrint(schema.getProto()).cStr() << std::endl;
-  }
+  kj::Vector<uint64_t> sourceIds;
 
-  return 0;
-}
+  struct OutputDirective {
+    kj::ArrayPtr<const char> name;
+    kj::StringPtr dir;
+  };
+  kj::Vector<OutputDirective> outputs;
+};
 
 }  // namespace compiler
 }  // namespace capnp

c++/src/capnp/compiler/compiler.c++

@@ -145,6 +145,7 @@ private:
 
     typedef std::multimap<kj::StringPtr, kj::Own<Node>> NestedNodesMap;
     NestedNodesMap nestedNodes;
+    kj::Vector<Node*> orderedNestedNodes;
     // Filled in when lookupMember() is first called.  multimap in case of duplicate member names --
     // we still want to compile them, even if it's an error.
 
@@ -237,7 +238,9 @@ public:
     // These bootstrap schemas can then be plugged into the dynamic API and used to evaluate these
     // remaining values.
 
-    inline Workspace(): orphanage(message.getOrphanage()) {}
+    inline explicit Workspace(const SchemaLoader::LazyLoadCallback& loaderCallback)
+        : orphanage(message.getOrphanage()),
+          bootstrapLoader(loaderCallback) {}
   };
 
   const kj::Arena& getNodeArena() const { return nodeArena; }
@@ -373,7 +376,7 @@ uint64_t Compiler::Node::generateId(uint64_t parentId, kj::StringPtr declName,
     result = (result << 8) | resultBytes[i];
   }
 
-  return result;
+  return result | (1ull << 63);
 }
 
 kj::StringPtr Compiler::Node::joinDisplayName(
@@ -415,6 +418,7 @@ const Compiler::Node::Content& Compiler::Node::getContent(Content::State minimum
           case Declaration::Body::INTERFACE_DECL: {
             kj::Own<Node> subNode = arena.allocateOwn<Node>(*this, nestedDecl);
             kj::StringPtr name = nestedDecl.getName().getValue();
+            locked->orderedNestedNodes.add(subNode);
             locked->nestedNodes.insert(std::make_pair(name, kj::mv(subNode)));
             break;
           }
@@ -459,10 +463,10 @@ const Compiler::Node::Content& Compiler::Node::getContent(Content::State minimum
         builder.setScopeId(p->id);
       }
 
-      auto nestedIter = builder.initNestedNodes(locked->nestedNodes.size()).begin();
-      for (auto& entry: locked->nestedNodes) {
-        nestedIter->setName(entry.first);
-        nestedIter->setId(entry.second->id);
+      auto nestedIter = builder.initNestedNodes(locked->orderedNestedNodes.size()).begin();
+      for (auto node: locked->orderedNestedNodes) {
+        nestedIter->setName(node->declaration.getName().getValue());
+        nestedIter->setId(node->id);
         ++nestedIter;
       }
 
@@ -482,6 +486,7 @@ const Compiler::Node::Content& Compiler::Node::getContent(Content::State minimum
       // case.
       Content* contentPtr = locked.get();
       workspace.arena.copy(kj::defer([contentPtr]() {
+        contentPtr->bootstrapSchema = nullptr;
         if (contentPtr->state == Content::BOOTSTRAP) {
           contentPtr->state = Content::EXPANDED;
         }
@@ -751,7 +756,7 @@ kj::Maybe<const Compiler::Node&> Compiler::Impl::lookupBuiltin(kj::StringPtr nam
 }
 
 uint64_t Compiler::Impl::add(const Module& module, Mode mode) const {
-  Impl::Workspace workspace;
+  Impl::Workspace workspace(*this);
   auto lock = this->workspace.lockExclusive();
   *lock = &workspace;
   KJ_DEFER(*lock = nullptr);
@@ -766,7 +771,7 @@ uint64_t Compiler::Impl::add(const Module& module, Mode mode) const {
 void Compiler::Impl::load(const SchemaLoader& loader, uint64_t id) const {
   KJ_IF_MAYBE(node, findNode(id)) {
     if (&loader == &finalLoader) {
-      Workspace workspace;
+      Workspace workspace(*this);
       auto lock = this->workspace.lockExclusive();
       *lock = &workspace;
       KJ_DEFER(*lock = nullptr);

c++/src/capnp/compiler/node-translator.c++

@@ -169,6 +169,7 @@ public:
   struct StructOrGroup {
     // Abstract interface for scopes in which fields can be added.
 
+    virtual void addVoid() = 0;
     virtual uint addData(uint lgSize) = 0;
     virtual uint addPointer() = 0;
     virtual bool tryExpandData(uint oldLgSize, uint oldOffset, uint expansionFactor) = 0;
@@ -184,6 +185,8 @@ public:
 
     HoleSet<uint> holes;
 
+    void addVoid() override {}
+
     uint addData(uint lgSize) override {
       KJ_IF_MAYBE(hole, holes.tryAllocate(lgSize)) {
         return *hole;
@@ -226,7 +229,7 @@ public:
 
     StructOrGroup& parent;
     uint groupCount = 0;
-    kj::Maybe<int> discriminantOffset;
+    kj::Maybe<uint> discriminantOffset;
     kj::Vector<DataLocation> dataLocations;
     kj::Vector<uint> pointerLocations;
 
@@ -247,7 +250,7 @@ public:
       return pointerLocations.add(parent.addPointer());
     }
 
-    void newGroup() {
+    void newGroupAddingFirstMember() {
       if (++groupCount == 2) {
         addDiscriminant();
       }
@@ -273,11 +276,15 @@ public:
       kj::Maybe<uint> smallestHoleAtLeast(Union::DataLocation& location, uint lgSize) {
         // Find the smallest single hole that is at least the given size.  This is used to find the
         // optimal place to allocate each field -- it is placed in the smallest slot where it fits,
-        // to reduce fragmentation.
+        // to reduce fragmentation.  Returns the size of the hole, if found.
 
         if (!isUsed) {
           // The location is effectively one big hole.
-          return location.lgSize;
+          if (lgSize <= location.lgSize) {
+            return location.lgSize;
+          } else {
+            return nullptr;
+          }
         } else if (lgSize >= lgSizeUsed) {
           // Requested size is at least our current usage, so clearly won't fit in any current
           // holes, but if the location's size is larger than what we're using, we'd be able to
@@ -471,12 +478,21 @@ public:
     uint parentPointerLocationUsage = 0;
     // Number of parent's pointer locations that have been used by this group.
 
-    inline Group(Union& parent): parent(parent) {
-      parent.newGroup();
-    }
+    bool hasMembers = false;
+
+    inline Group(Union& parent): parent(parent) {}
     KJ_DISALLOW_COPY(Group);
 
+    void addVoid() override {
+      if (!hasMembers) {
+        hasMembers = true;
+        parent.newGroupAddingFirstMember();
+      }
+    }
+
     uint addData(uint lgSize) override {
+      addVoid();
+
       uint bestSize = std::numeric_limits<uint>::max();
       kj::Maybe<uint> bestLocation = nullptr;
 
@@ -516,6 +532,8 @@ public:
     }
 
     uint addPointer() override {
+      addVoid();
+
       if (parentPointerLocationUsage < parent.pointerLocations.size()) {
         return parent.pointerLocations[parentPointerLocationUsage++];
       } else {
@@ -817,7 +835,7 @@ void NodeTranslator::compileEnum(Declaration::Enum::Reader decl,
 
     dupDetector.check(enumerantDecl.getId().getOrdinal());
 
-    auto enumerantBuilder = list[i];
+    auto enumerantBuilder = list[i++];
     enumerantBuilder.setName(enumerantDecl.getName().getValue());
     enumerantBuilder.setCodeOrder(codeOrder);
     enumerantBuilder.adoptAnnotations(compileAnnotationApplications(
@@ -907,6 +925,7 @@ public:
             fieldBuilder.setOffset(member.fieldScope->addPointer());
           } else if (lgSize == -1) {
             // void
+            member.fieldScope->addVoid();
             fieldBuilder.setOffset(0);
           } else {
             fieldBuilder.setOffset(member.fieldScope->addData(lgSize));
@@ -1296,13 +1315,13 @@ void NodeTranslator::compileDefaultDefaultValue(
     case schema::Type::Body::UINT64_TYPE: target.getBody().setUint64Value(0); break;
     case schema::Type::Body::FLOAT32_TYPE: target.getBody().setFloat32Value(0); break;
     case schema::Type::Body::FLOAT64_TYPE: target.getBody().setFloat64Value(0); break;
-    case schema::Type::Body::TEXT_TYPE: target.getBody().initTextValue(0); break;
-    case schema::Type::Body::DATA_TYPE: target.getBody().initDataValue(0); break;
     case schema::Type::Body::ENUM_TYPE: target.getBody().setEnumValue(0); break;
     case schema::Type::Body::INTERFACE_TYPE: target.getBody().setInterfaceValue(); break;
 
     // Bit of a hack:  For "Object" types, we adopt a null orphan, which sets the field to null.
     // TODO(cleanup):  Create a cleaner way to do this.
+    case schema::Type::Body::TEXT_TYPE: target.getBody().adoptTextValue(Orphan<Text>()); break;
+    case schema::Type::Body::DATA_TYPE: target.getBody().adoptDataValue(Orphan<Data>()); break;
     case schema::Type::Body::STRUCT_TYPE: target.getBody().adoptStructValue(Orphan<Data>()); break;
     case schema::Type::Body::LIST_TYPE: target.getBody().adoptListValue(Orphan<Data>()); break;
     case schema::Type::Body::OBJECT_TYPE: target.getBody().adoptObjectValue(Orphan<Data>()); break;
@@ -1325,6 +1344,10 @@ public:
               ListSchema listMemberSchema)
       : type(LIST_OBJECT_UNION_MEMBER), unionBuilder(unionBuilder), unionMember(unionMember),
         listMemberSchema(listMemberSchema) {}
+  DynamicSlot(DynamicUnion::Builder unionBuilder, StructSchema::Member unionMember,
+              EnumSchema enumMemberSchema)
+      : type(ENUM_UNION_MEMBER), unionBuilder(unionBuilder), unionMember(unionMember),
+        enumMemberSchema(enumMemberSchema) {}
 
   DynamicStruct::Builder initStruct() {
     switch (type) {
@@ -1334,6 +1357,7 @@ public:
       case STRUCT_OBJECT_UNION_MEMBER:
         return unionBuilder.initObject(unionMember, structMemberSchema);
       case LIST_OBJECT_UNION_MEMBER: KJ_FAIL_REQUIRE("Type mismatch.");
+      case ENUM_UNION_MEMBER: KJ_FAIL_REQUIRE("Type mismatch.");
     }
     KJ_FAIL_ASSERT("can't get here");
   }
@@ -1346,6 +1370,7 @@ public:
       case STRUCT_OBJECT_UNION_MEMBER: KJ_FAIL_REQUIRE("Type mismatch.");
       case LIST_OBJECT_UNION_MEMBER:
         return unionBuilder.initObject(unionMember, listMemberSchema, size);
+      case ENUM_UNION_MEMBER: KJ_FAIL_REQUIRE("Type mismatch.");
     }
     KJ_FAIL_ASSERT("can't get here");
   }
@@ -1357,17 +1382,21 @@ public:
       case UNION_MEMBER: return unionBuilder.init(unionMember).as<DynamicUnion>();
       case STRUCT_OBJECT_UNION_MEMBER: KJ_FAIL_REQUIRE("Type mismatch.");
       case LIST_OBJECT_UNION_MEMBER: KJ_FAIL_REQUIRE("Type mismatch.");
+      case ENUM_UNION_MEMBER: KJ_FAIL_REQUIRE("Type mismatch.");
     }
     KJ_FAIL_ASSERT("can't get here");
   }
 
   void set(DynamicValue::Reader value) {
     switch (type) {
-      case FIELD: return structBuilder.set(member, value);
-      case ELEMENT: return listBuilder.set(index, value);
-      case UNION_MEMBER: return unionBuilder.set(unionMember, value);
-      case STRUCT_OBJECT_UNION_MEMBER: return unionBuilder.set(unionMember, value);
-      case LIST_OBJECT_UNION_MEMBER: return unionBuilder.set(unionMember, value);
+      case FIELD: structBuilder.set(member, value); return;
+      case ELEMENT: listBuilder.set(index, value); return;
+      case UNION_MEMBER: unionBuilder.set(unionMember, value); return;
+      case STRUCT_OBJECT_UNION_MEMBER: unionBuilder.set(unionMember, value); return;
+      case LIST_OBJECT_UNION_MEMBER: unionBuilder.set(unionMember, value); return;
+      case ENUM_UNION_MEMBER:
+        unionBuilder.set(unionMember, value.as<DynamicEnum>().getRaw());
+        return;
     }
     KJ_FAIL_ASSERT("can't get here");
   }
@@ -1388,13 +1417,15 @@ public:
       case UNION_MEMBER: return enumIdForMember(unionMember);
       case STRUCT_OBJECT_UNION_MEMBER: return nullptr;
       case LIST_OBJECT_UNION_MEMBER: return nullptr;
+      case ENUM_UNION_MEMBER: return enumMemberSchema.getProto().getId();
     }
     KJ_FAIL_ASSERT("can't get here");
   }
 
 private:
   enum Type {
-    FIELD, ELEMENT, UNION_MEMBER, STRUCT_OBJECT_UNION_MEMBER, LIST_OBJECT_UNION_MEMBER
+    FIELD, ELEMENT, UNION_MEMBER, STRUCT_OBJECT_UNION_MEMBER, LIST_OBJECT_UNION_MEMBER,
+    ENUM_UNION_MEMBER
   };
   Type type;
 
@@ -1413,6 +1444,7 @@ private:
       union {
         StructSchema structMemberSchema;
         ListSchema listMemberSchema;
+        EnumSchema enumMemberSchema;
       };
     };
   };
@@ -1494,6 +1526,12 @@ void NodeTranslator::compileValue(ValueExpression::Reader source, schema::Type::
         compileValue(source, slot, isBootstrap);
       }
       break;
+    case schema::Type::Body::ENUM_TYPE:
+      KJ_IF_MAYBE(enumSchema, resolver.resolveBootstrapSchema(type.getBody().getEnumType())) {
+        DynamicSlot slot(valueUnion, member, enumSchema->asEnum());
+        compileValue(source, slot, isBootstrap);
+      }
+      break;
     default:
       DynamicSlot slot(valueUnion, member);
       compileValue(source, slot, isBootstrap);

c++/src/capnp/compiler/parser.c++

@@ -699,7 +699,7 @@ CapnpParser::CapnpParser(Orphanage orphanageParam, const ErrorReporter& errorRep
         for (uint i = 0; i < annotations.size(); i++) {
           list.adoptWithCaveats(i, kj::mv(annotations[i]));
         }
-        builder.getBody().initGroupDecl();
+        builder.getBody().initUnionDecl();
         return DeclParserResult(kj::mv(decl), parsers.structLevelDecl);
       }));
 

c++/src/capnp/dynamic-test.c++

@@ -384,6 +384,14 @@ TEST(DynamicApi, SetEnumFromNative) {
   checkList<TestEnum>(root.get("enumList"), {TestEnum::BAR, TestEnum::FOO});
 }
 
+TEST(DynamicApi, SetDataFromText) {
+  MallocMessageBuilder builder;
+  auto root = builder.initRoot<DynamicStruct>(Schema::from<TestAllTypes>());
+
+  root.set("dataField", "foo");
+  EXPECT_EQ(data("foo"), root.get("dataField").as<Data>());
+}
+
 }  // namespace
 }  // namespace _ (private)
 }  // namespace capnp

c++/src/capnp/dynamic.c++

@@ -1459,7 +1459,6 @@ BuilderFor<typeName> DynamicValue::Builder::AsImpl<typeName>::apply(Builder& bui
 HANDLE_TYPE(bool, BOOL, bool)
 
 HANDLE_TYPE(text, TEXT, Text)
-HANDLE_TYPE(data, DATA, Data)
 HANDLE_TYPE(list, LIST, DynamicList)
 HANDLE_TYPE(struct, STRUCT, DynamicStruct)
 HANDLE_TYPE(enum, ENUM, DynamicEnum)
@@ -1468,6 +1467,29 @@ HANDLE_TYPE(union, UNION, DynamicUnion)
 
 #undef HANDLE_TYPE
 
+Data::Reader DynamicValue::Reader::AsImpl<Data>::apply(const Reader& reader) {
+  if (reader.type == TEXT) {
+    // Coerce text to data.
+    return Data::Reader(reinterpret_cast<const byte*>(reader.textValue.begin()),
+                        reader.textValue.size());
+  }
+  KJ_REQUIRE(reader.type == DATA, "Value type mismatch.") {
+    return Data::Reader();
+  }
+  return reader.dataValue;
+}
+Data::Builder DynamicValue::Builder::AsImpl<Data>::apply(Builder& builder) {
+  if (builder.type == TEXT) {
+    // Coerce text to data.
+    return Data::Builder(reinterpret_cast<byte*>(builder.textValue.begin()),
+                         builder.textValue.size());
+  }
+  KJ_REQUIRE(builder.type == DATA, "Value type mismatch.") {
+    return BuilderFor<Data>();
+  }
+  return builder.dataValue;
+}
+
 // As in the header, HANDLE_TYPE(void, VOID, Void) crashes GCC 4.7.
 Void DynamicValue::Reader::AsImpl<Void>::apply(const Reader& reader) {
   KJ_REQUIRE(reader.type == VOID, "Value type mismatch.") {

c++/src/capnp/layout.c++

@@ -2159,6 +2159,7 @@ void StructBuilder::transferContentFrom(StructBuilder other) {
   for (uint i = 0; i < pointerCount / POINTERS; i++) {
     WireHelpers::zeroObject(segment, pointers + i);
   }
+  memset(pointers, 0, pointerCount * BYTES_PER_POINTER / BYTES);
 
   // Transfer the pointers.
   WirePointerCount sharedPointerCount = kj::min(pointerCount, other.pointerCount);
@@ -2172,6 +2173,42 @@ void StructBuilder::transferContentFrom(StructBuilder other) {
   memset(other.pointers, 0, sharedPointerCount * BYTES_PER_POINTER / BYTES);
 }
 
+void StructBuilder::copyContentFrom(StructReader other) {
+  // Determine the amount of data the builders have in common.
+  BitCount sharedDataSize = kj::min(dataSize, other.dataSize);
+
+  if (dataSize > sharedDataSize) {
+    // Since the target is larger than the source, make sure to zero out the extra bits that the
+    // source doesn't have.
+    if (dataSize == 1 * BITS) {
+      setDataField<bool>(0 * ELEMENTS, false);
+    } else {
+      byte* unshared = reinterpret_cast<byte*>(data) + sharedDataSize / BITS_PER_BYTE / BYTES;
+      memset(unshared, 0, (dataSize - sharedDataSize) / BITS_PER_BYTE / BYTES);
+    }
+  }
+
+  // Copy over the shared part.
+  if (sharedDataSize == 1 * BITS) {
+    setDataField<bool>(0 * ELEMENTS, other.getDataField<bool>(0 * ELEMENTS));
+  } else {
+    memcpy(data, other.data, sharedDataSize / BITS_PER_BYTE / BYTES);
+  }
+
+  // Zero out all pointers in the target.
+  for (uint i = 0; i < pointerCount / POINTERS; i++) {
+    WireHelpers::zeroObject(segment, pointers + i);
+  }
+  memset(pointers, 0, pointerCount * BYTES_PER_POINTER / BYTES);
+
+  // Copy the pointers.
+  WirePointerCount sharedPointerCount = kj::min(pointerCount, other.pointerCount);
+  for (uint i = 0; i < sharedPointerCount / POINTERS; i++) {
+    WireHelpers::setObjectPointer(segment, pointers + i, WireHelpers::readObjectPointer(
+        other.segment, other.pointers + i, nullptr, other.nestingLimit));
+  }
+}
+
 bool StructBuilder::isPointerFieldNull(WirePointerCount ptrIndex) {
   return (pointers + ptrIndex)->isNull();
 }

c++/src/capnp/layout.h

@@ -369,6 +369,11 @@ public:
   // than this, the extra data is not transferred, meaning there is a risk of data loss when
   // transferring from messages built with future versions of the protocol.
 
+  void copyContentFrom(StructReader other);
+  // Copy content from `other`.  If `other`'s sections are larger than this, the extra data is not
+  // copied, meaning there is a risk of data loss when copying from messages built with future
+  // versions of the protocol.
+
   bool isPointerFieldNull(WirePointerCount ptrIndex);
 
   StructReader asReader() const;

c++/src/capnp/list.h

@@ -266,6 +266,15 @@ struct List<T, Kind::STRUCT> {
           orphan.builder.asStruct(_::StructSize(
               0 * WORDS, 0 * POINTERS, _::FieldSize::VOID)));
     }
+    inline void setWithCaveats(uint index, const typename T::Reader& reader) {
+      // Mostly behaves like you'd expect `set` to behave, but with a caveat originating from
+      // the fact that structs in a struct list are allocated inline rather than by pointer:
+      // If the source struct is larger than the target struct -- say, because the source was built
+      // using a newer version of the schema that has additional fields -- it will be truncated,
+      // losing data.
+
+      builder.getStructElement(index * ELEMENTS).copyContentFrom(reader._reader);
+    }
 
     // There are no init(), set(), adopt(), or disown() methods for lists of structs because the
     // elements of the list are inlined and are initialized when the list is initialized.  This

c++/src/kj/main.c++

@@ -123,7 +123,7 @@ int runMainAndExit(ProcessContext& context, MainFunc&& func, int argc, char* arg
     KJ_IF_MAYBE(exception, runCatchingExceptions([&]() {
       func(argv[0], params);
     })) {
-      context.error(str(exception));
+      context.error(str("*** Uncaught exception ***\n", *exception));
     }
     context.exit();
 #if !KJ_NO_EXCEPTIONS
@@ -453,7 +453,7 @@ void MainBuilder::MainImpl::operator()(StringPtr programName, ArrayPtr<const Str
     uint i = 0;
     for (; i < argSpec.minCount; i++) {
       if (argPos == arguments.end()) {
-        usageError(programName, str("missing argument <", argSpec.title, '>'));
+        usageError(programName, str("missing argument ", argSpec.title));
       } else {
         KJ_IF_MAYBE(error, argSpec.callback(*argPos).releaseError()) {
           usageError(programName, str(*argPos, ": ", *error));
@@ -562,9 +562,9 @@ void MainBuilder::MainImpl::printHelp(StringPtr programName) {
     for (auto& arg: impl->args) {
       text.add(' ');
       if (arg.minCount == 0) {
-        text.addAll(str("[<", arg.title, arg.maxCount > 1 ? ">...]" : ">]"));
+        text.addAll(str("[", arg.title, arg.maxCount > 1 ? "...]" : "]"));
       } else {
-        text.addAll(str('<', arg.title, arg.maxCount > 1 ? ">..." : ">"));
+        text.addAll(str(arg.title, arg.maxCount > 1 ? "..." : ""));
       }
     }
   } else {
@@ -609,12 +609,12 @@ void MainBuilder::MainImpl::printHelp(StringPtr programName) {
         if (name.isLong) {
           text.addAll(str("--", name.longName));
           if (opt->hasArg) {
-            text.addAll(str("=<", opt->argTitle, '>'));
+            text.addAll(str("=", opt->argTitle));
           }
         } else {
           text.addAll(str("-", name.shortName));
           if (opt->hasArg) {
-            text.addAll(str('<', opt->argTitle, '>'));
+            text.addAll(opt->argTitle);
           }
         }
       }

c++/src/kj/main.h

@@ -173,8 +173,8 @@ class MainBuilder {
   //             .addOption({'a', "all"}, KJ_BIND_METHOD(*this, setAll),
   //                 "Frob all the widgets.  Otherwise, only some widgets are frobbed.")
   //             .addOptionWithArg({'o', "output"}, KJ_BIND_METHOD(*this, setOutput),
-  //                 "filename", "Output to <filename>.  Must be a .foo file.")
-  //             .expectOneOrMoreArgs("source", KJ_BIND_METHOD(*this, processInput))
+  //                 "<filename>", "Output to <filename>.  Must be a .foo file.")
+  //             .expectOneOrMoreArgs("<source>", KJ_BIND_METHOD(*this, processInput))
   //             .build();
   //       }
   //
@@ -260,7 +260,7 @@ public:
   // Example:
   //
   //     builder.addOptionWithArg({'o', "output"}, KJ_BIND_METHOD(*this, setOutput),
-  //                              "filename", "Output to <filename>.");
+  //                              "<filename>", "Output to <filename>.");
   //
   // This option could be specified with an argument of "foo" in the following ways:
   //
@@ -312,11 +312,11 @@ public:
   // order of registration.
   //
   // For example, say you called:
-  //     builder.expectArg("foo", ...);
-  //     builder.expectOptionalArg("bar", ...);
-  //     builder.expectArg("baz", ...);
-  //     builder.expectZeroOrMoreArgs("qux", ...);
-  //     builder.expectArg("corge", ...);
+  //     builder.expectArg("<foo>", ...);
+  //     builder.expectOptionalArg("<bar>", ...);
+  //     builder.expectArg("<baz>", ...);
+  //     builder.expectZeroOrMoreArgs("<qux>", ...);
+  //     builder.expectArg("<corge>", ...);
   //
   // This command requires at least three arguments: foo, baz, and corge.  If four arguments are
   // given, the second is assigned to bar.  If five or more arguments are specified, then the

c++/src/kj/string.h

@@ -147,6 +147,8 @@ public:
     return StringPtr(*this).slice(start, end);
   }
 
+  inline Maybe<size_t> findFirst(char c) const { return StringPtr(*this).findFirst(c); }
+
 private:
   Array<char> content;
 };

compiler/src/c++-header.mustache

@@ -189,6 +189,8 @@ private:
   friend struct ::capnp::ToDynamic_;
   template <typename T, ::capnp::Kind k>
   friend struct ::capnp::_::PointerHelpers;
+  template <typename T, ::capnp::Kind k>
+  friend struct ::capnp::List;
   friend class ::capnp::MessageBuilder;
   friend class ::capnp::Orphanage;
   friend ::kj::String KJ_STRINGIFY({{typeFullName}}::Reader reader);