Propagate docstrings into binary schema

c++/src/capnp/compiler/capnp.c++

@@ -433,6 +433,12 @@ public:
       nodes.setWithCaveats(i, schemas[i].getProto());
     }
 
+    auto docs = compiler->getLoader().getAllDocs();
+    auto docnodes = request.initNodeDocs(docs.size());
+    for (size_t i = 0; i < docs.size(); i++) {
+      docnodes.setWithCaveats(i, docs[i]);
+    }
+
     auto requestedFiles = request.initRequestedFiles(sourceFiles.size());
     for (size_t i = 0; i < sourceFiles.size(); i++) {
       auto requestedFile = requestedFiles[i];

c++/src/capnp/compiler/compiler.c++

@@ -668,6 +668,9 @@ void Compiler::Node::traverse(uint eagerness, std::unordered_map<Node*, uint>& s
         }
       }
     }
+    KJ_IF_MAYBE(doc, content->translator->getDoc()) {
+      finalLoader.loadDoc(doc->getReader());
+    }
   }
 
   if (eagerness & PARENTS) {

c++/src/capnp/compiler/node-translator.c++

@@ -1377,10 +1377,16 @@ NodeTranslator::NodeTranslator(
       localBrand(kj::refcounted<BrandScope>(
           errorReporter, wipNodeParam.getReader().getId(),
           decl.getParameters().size(), resolver)),
-      wipNode(kj::mv(wipNodeParam)) {
+      wipNode(kj::mv(wipNodeParam)),
+      wipNodeDoc() {
   compileNode(decl, wipNode.get());
 }
 
+void NodeTranslator::addFieldDoc(uint codeOrder, ::capnp::Text::Reader docComment)
+{
+  fieldDocs.add(std::make_pair(codeOrder, docComment));
+}
+
 NodeTranslator::~NodeTranslator() noexcept(false) {}
 
 NodeTranslator::NodeSet NodeTranslator::getBootstrapNode() {
@@ -1467,6 +1473,23 @@ void NodeTranslator::compileNode(Declaration::Reader decl, schema::Node::Builder
   }
 
   builder.adoptAnnotations(compileAnnotationApplications(decl.getAnnotations(), targetsFlagName));
+
+  if (decl.hasDocComment() || !fieldDocs.empty()) {
+    Orphan<schema::NodeDoc> doc = orphanage.newOrphan<schema::NodeDoc>();
+    doc.get().setId(wipNode.getReader().getId());
+    if (decl.hasDocComment())
+      doc.get().setDocComment(decl.getDocComment());
+    if (!fieldDocs.empty()) {
+      auto fdocs = doc.get().initFieldDocs(fieldDocs.size());
+
+      for (size_t i = 0; i < fieldDocs.size(); i++) {
+        auto fdoc = fdocs[i];
+        fdoc.setCodeOrder(fieldDocs[i].first);
+        fdoc.setDocComment(fieldDocs[i].second);
+      }
+    }
+    wipNodeDoc = kj::mv(doc);
+  }
 }
 
 static kj::StringPtr getExpressionTargetName(Expression::Reader exp) {
@@ -1779,6 +1802,8 @@ private:
     // Information about the field declaration.  We don't use Declaration::Reader because it might
     // have come from a Declaration::Param instead.
 
+    kj::Maybe<::capnp::Text::Reader> docComment = nullptr;
+
     kj::Maybe<schema::Field::Builder> schema;
     // Schema for the field.  Initialized when getSchema() is first called.
 
@@ -1815,6 +1840,8 @@ private:
         hasDefaultValue = true;
         fieldDefaultValue = fieldDecl.getDefaultValue().getValue();
       }
+      if (decl.hasDocComment())
+        docComment = decl.getDocComment();
     }
     inline MemberInfo(MemberInfo& parent, uint codeOrder,
                       const Declaration::Param::Reader& decl,
@@ -1841,6 +1868,8 @@ private:
           startByte(decl.getStartByte()), endByte(decl.getEndByte()),
           node(node), unionScope(nullptr) {
       KJ_REQUIRE(decl.which() != Declaration::FIELD);
+      if (decl.hasDocComment())
+        docComment = decl.getDocComment();
     }
 
     schema::Field::Builder getSchema() {
@@ -2043,6 +2072,11 @@ private:
           // Ignore others.
           break;
       }
+      if (memberInfo) {
+        KJ_IF_MAYBE(mydoc, memberInfo->docComment) {
+          translator.addFieldDoc(memberInfo->codeOrder, *mydoc);
+        }
+      }
 
       KJ_IF_MAYBE(o, ordinal) {
         membersByOrdinal.insert(std::make_pair(*o, memberInfo));

c++/src/capnp/compiler/node-translator.h

@@ -33,6 +33,7 @@
 #include <kj/vector.h>
 #include <kj/one-of.h>
 #include "error-reporter.h"
+#include <map>
 
 namespace capnp {
 namespace compiler {
@@ -160,6 +161,9 @@ public:
   // `brandBuilder` may be used to construct a message which will fill in ResolvedDecl::brand in
   // the result.
 
+  kj::Maybe<Orphan<schema::NodeDoc>>& getDoc() { return wipNodeDoc; };
+  void addFieldDoc(uint codeOrder, ::capnp::Text::Reader docComment);
+
 private:
   class DuplicateNameDetector;
   class DuplicateOrdinalDetector;
@@ -175,7 +179,9 @@ private:
   kj::Own<BrandScope> localBrand;
 
   Orphan<schema::Node> wipNode;
-  // The work-in-progress schema node.
+  kj::Maybe<Orphan<schema::NodeDoc>> wipNodeDoc;
+  // The work-in-progress schema node and its docstring
+  kj::Vector<std::pair<uint, ::capnp::Text::Reader>> fieldDocs;
 
   kj::Vector<Orphan<schema::Node>> groups;
   // If this is a struct node and it contains groups, these are the nodes for those groups,  which

c++/src/capnp/schema-loader.c++

@@ -139,6 +139,8 @@ public:
   const _::RawBrandedSchema* getUnbound(const _::RawSchema* schema);
 
   kj::Array<Schema> getAllLoaded() const;
+  kj::Array<schema::NodeDoc::Reader> getAllDocs() const;
+  void loadDoc(schema::NodeDoc::Reader docReader);
 
   void requireStructSize(uint64_t id, uint dataWordCount, uint pointerCount);
   // Require any struct nodes loaded with this ID -- in the past and in the future -- to have at
@@ -157,6 +159,7 @@ private:
   std::unordered_map<uint64_t, _::RawSchema*> schemas;
   std::unordered_map<SchemaBindingsPair, _::RawBrandedSchema*, SchemaBindingsPairHash> brands;
   std::unordered_map<const _::RawSchema*, _::RawBrandedSchema*> unboundBrands;
+  std::unordered_map<uint64_t, schema::NodeDoc::Reader> docs;
 
   struct RequiredSize {
     uint16_t dataWordCount;
@@ -1846,6 +1849,25 @@ kj::Array<Schema> SchemaLoader::Impl::getAllLoaded() const {
   return result;
 }
 
+kj::Array<schema::NodeDoc::Reader> SchemaLoader::Impl::getAllDocs() const {
+  size_t count = 0;
+  for (auto& doc: docs) {
+    ++count;
+  }
+
+  kj::Array<schema::NodeDoc::Reader> result = kj::heapArray<schema::NodeDoc::Reader>(count);
+  size_t i = 0;
+  for (auto& doc: docs) {
+    result[i++] = doc.second;
+  }
+  return result;
+}
+
+void SchemaLoader::Impl::loadDoc(schema::NodeDoc::Reader docReader)
+{
+  docs[docReader.getId()] = docReader;
+}
+
 void SchemaLoader::Impl::requireStructSize(uint64_t id, uint dataWordCount, uint pointerCount) {
   auto& slot = structSizeRequirements[id];
   slot.dataWordCount = kj::max(slot.dataWordCount, dataWordCount);
@@ -2105,6 +2127,14 @@ kj::Array<Schema> SchemaLoader::getAllLoaded() const {
   return impl.lockShared()->get()->getAllLoaded();
 }
 
+kj::Array<schema::NodeDoc::Reader> SchemaLoader::getAllDocs() const {
+  return impl.lockShared()->get()->getAllDocs();
+}
+
+void SchemaLoader::loadDoc(schema::NodeDoc::Reader docReader) const {
+  return impl.lockExclusive()->get()->loadDoc(docReader);
+}
+
 void SchemaLoader::loadNative(const _::RawSchema* nativeSchema) {
   impl.lockExclusive()->get()->loadNative(nativeSchema);
 }

c++/src/capnp/schema-loader.h

@@ -152,6 +152,9 @@ public:
   // loadCompiledTypeAndDependencies<T>() in order to get a flat list of all of T's transitive
   // dependencies.
 
+  kj::Array<schema::NodeDoc::Reader> getAllDocs() const;
+  void loadDoc(schema::NodeDoc::Reader docReader) const;
+
 private:
   class Validator;
   class CompatibilityChecker;