Extend AsyncCapabilityStream to support sending FDs with a message.

Previously, FDs could only be sent separately from data, with the underlying implementation actually sending a dummy one-byte message.

For Cap'n Proto FD passing, it's better if we can attach the FDs to the RPC message they arrived with, because it avoids the need to pre-negotiate where FD passing is supported: if the sender sends FDs but the recipient doesn't arrange to receive them, the FDs will be discarded and closed automatically by the OS. Whereas, if we are sending separate one-byte messages, the recipient needs to know what to do with those.

c++/src/kj/async-io-test.c++

@@ -233,6 +233,41 @@ TEST(AsyncIo, CapabilityPipe) {
   EXPECT_EQ("bar", result);
   EXPECT_EQ("foo", result2);
 }
+
+TEST(AsyncIo, CapabilityPipeMultiStreamMessage) {
+  auto ioContext = setupAsyncIo();
+
+  auto pipe = ioContext.provider->newCapabilityPipe();
+  auto pipe2 = ioContext.provider->newCapabilityPipe();
+  auto pipe3 = ioContext.provider->newCapabilityPipe();
+
+  auto streams = heapArrayBuilder<Own<AsyncCapabilityStream>>(2);
+  streams.add(kj::mv(pipe2.ends[0]));
+  streams.add(kj::mv(pipe3.ends[0]));
+
+  ArrayPtr<const byte> secondBuf = "bar"_kj.asBytes();
+  pipe.ends[0]->writeWithStreams("foo"_kj.asBytes(), arrayPtr(&secondBuf, 1), streams.finish())
+      .wait(ioContext.waitScope);
+
+  char receiveBuffer[7];
+  Own<AsyncCapabilityStream> receiveStreams[3];
+  auto result = pipe.ends[1]->tryReadWithStreams(receiveBuffer, 6, 7, receiveStreams, 3)
+      .wait(ioContext.waitScope);
+
+  KJ_EXPECT(result.byteCount == 6);
+  receiveBuffer[6] = '\0';
+  KJ_EXPECT(kj::StringPtr(receiveBuffer) == "foobar");
+
+  KJ_ASSERT(result.capCount == 2);
+
+  receiveStreams[0]->write("baz", 3).wait(ioContext.waitScope);
+  receiveStreams[0] = nullptr;
+  KJ_EXPECT(pipe2.ends[1]->readAllText().wait(ioContext.waitScope) == "baz");
+
+  pipe3.ends[1]->write("qux", 3).wait(ioContext.waitScope);
+  pipe3.ends[1] = nullptr;
+  KJ_EXPECT(receiveStreams[1]->readAllText().wait(ioContext.waitScope) == "qux");
+}
 #endif
 
 TEST(AsyncIo, PipeThread) {

c++/src/kj/async-io-unix.c++

@@ -135,7 +135,29 @@ public:
   virtual ~AsyncStreamFd() noexcept(false) {}
 
   Promise<size_t> tryRead(void* buffer, size_t minBytes, size_t maxBytes) override {
-    return tryReadInternal(buffer, minBytes, maxBytes, 0);
+    return tryReadInternal(buffer, minBytes, maxBytes, nullptr, 0, {0,0})
+        .then([](ReadResult r) { return r.byteCount; });
+  }
+
+  Promise<ReadResult> tryReadWithFds(void* buffer, size_t minBytes, size_t maxBytes,
+                                     AutoCloseFd* fdBuffer, size_t maxFds) override {
+    return tryReadInternal(buffer, minBytes, maxBytes, fdBuffer, maxFds, {0,0});
+  }
+
+  Promise<ReadResult> tryReadWithStreams(
+      void* buffer, size_t minBytes, size_t maxBytes,
+      Own<AsyncCapabilityStream>* streamBuffer, size_t maxStreams) override {
+    auto fdBuffer = kj::heapArray<AutoCloseFd>(maxStreams);
+    auto promise = tryReadInternal(buffer, minBytes, maxBytes, fdBuffer.begin(), maxStreams, {0,0});
+
+    return promise.then([this, fdBuffer = kj::mv(fdBuffer), streamBuffer]
+                        (ReadResult result) mutable {
+      for (auto i: kj::zeroTo(result.capCount)) {
+        streamBuffer[i] = kj::heap<AsyncStreamFd>(eventPort, fdBuffer[i].release(),
+            LowLevelAsyncIoProvider::TAKE_OWNERSHIP | LowLevelAsyncIoProvider::ALREADY_CLOEXEC);
+      }
+      return result;
+    });
   }
 
   Promise<void> write(const void* buffer, size_t size) override {
@@ -173,12 +195,28 @@ public:
 
   Promise<void> write(ArrayPtr<const ArrayPtr<const byte>> pieces) override {
     if (pieces.size() == 0) {
-      return writeInternal(nullptr, nullptr);
+      return writeInternal(nullptr, nullptr, nullptr);
     } else {
-      return writeInternal(pieces[0], pieces.slice(1, pieces.size()));
+      return writeInternal(pieces[0], pieces.slice(1, pieces.size()), nullptr);
     }
   }
 
+  Promise<void> writeWithFds(ArrayPtr<const byte> data,
+                             ArrayPtr<const ArrayPtr<const byte>> moreData,
+                             ArrayPtr<const int> fds) override {
+    return writeInternal(data, moreData, fds);
+  }
+
+  Promise<void> writeWithStreams(ArrayPtr<const byte> data,
+                                 ArrayPtr<const ArrayPtr<const byte>> moreData,
+                                 Array<Own<AsyncCapabilityStream>> streams) override {
+    auto fds = KJ_MAP(stream, streams) {
+      return downcast<AsyncStreamFd>(*stream).fd;
+    };
+    auto promise = writeInternal(data, moreData, fds);
+    return promise.attach(kj::mv(fds));
+  }
+
   Promise<void> whenWriteDisconnected() override {
     KJ_IF_MAYBE(p, writeDisconnectedPromise) {
       return p->addBranch();
@@ -224,57 +262,6 @@ public:
     *length = socklen;
   }
 
-  kj::Promise<Maybe<Own<AsyncCapabilityStream>>> tryReceiveStream() override {
-    return tryReceiveFdImpl<Own<AsyncCapabilityStream>>();
-  }
-
-  kj::Promise<void> sendStream(Own<AsyncCapabilityStream> stream) override {
-    auto downcasted = stream.downcast<AsyncStreamFd>();
-    auto promise = sendFd(downcasted->fd);
-    return promise.attach(kj::mv(downcasted));
-  }
-
-  kj::Promise<kj::Maybe<AutoCloseFd>> tryReceiveFd() override {
-    return tryReceiveFdImpl<AutoCloseFd>();
-  }
-
-  kj::Promise<void> sendFd(int fdToSend) override {
-    struct msghdr msg;
-    struct iovec iov;
-    union {
-      struct cmsghdr cmsg;
-      char cmsgSpace[CMSG_LEN(sizeof(int))];
-    };
-    memset(&msg, 0, sizeof(msg));
-    memset(&iov, 0, sizeof(iov));
-    memset(cmsgSpace, 0, sizeof(cmsgSpace));
-
-    char c = 0;
-    iov.iov_base = &c;
-    iov.iov_len = 1;
-    msg.msg_iov = &iov;
-    msg.msg_iovlen = 1;
-
-    msg.msg_control = &cmsg;
-    msg.msg_controllen = sizeof(cmsgSpace);
-
-    cmsg.cmsg_len = sizeof(cmsgSpace);
-    cmsg.cmsg_level = SOL_SOCKET;
-    cmsg.cmsg_type = SCM_RIGHTS;
-    *reinterpret_cast<int*>(CMSG_DATA(&cmsg)) = fdToSend;
-
-    ssize_t n;
-    KJ_NONBLOCKING_SYSCALL(n = sendmsg(fd, &msg, 0));
-    if (n < 0) {
-      return observer.whenBecomesWritable().then([this,fdToSend]() {
-        return sendFd(fdToSend);
-      });
-    } else {
-      KJ_ASSERT(n == 1);
-      return kj::READY_NOW;
-    }
-  }
-
   Promise<void> waitConnected() {
     // Wait until initial connection has completed. This actually just waits until it is writable.
 
@@ -303,13 +290,15 @@ private:
   UnixEventPort::FdObserver observer;
   Maybe<ForkedPromise<void>> writeDisconnectedPromise;
 
-  Promise<size_t> tryReadInternal(void* buffer, size_t minBytes, size_t maxBytes,
-                                  size_t alreadyRead) {
+  Promise<ReadResult> tryReadInternal(void* buffer, size_t minBytes, size_t maxBytes,
+                                      AutoCloseFd* fdBuffer, size_t maxFds,
+                                      ReadResult alreadyRead) {
     // `alreadyRead` is the number of bytes we have already received via previous reads -- minBytes,
     // maxBytes, and buffer have already been adjusted to account for them, but this count must
     // be included in the final return value.
 
     ssize_t n;
+    if (maxFds == 0) {
       KJ_NONBLOCKING_SYSCALL(n = ::read(fd, buffer, maxBytes)) {
         // Error.
 
@@ -319,6 +308,84 @@ private:
         //   http://llvm.org/bugs/show_bug.cgi?id=12286
         goto error;
       }
+    } else {
+      struct msghdr msg;
+      memset(&msg, 0, sizeof(msg));
+
+      struct iovec iov;
+      memset(&iov, 0, sizeof(iov));
+      iov.iov_base = buffer;
+      iov.iov_len = maxBytes;
+      msg.msg_iov = &iov;
+      msg.msg_iovlen = 1;
+
+      // Allocate space to receive a cmsg.
+      size_t msgBytes = CMSG_SPACE(sizeof(int) * maxFds);
+      KJ_ASSERT(msgBytes % sizeof(void*) == 0);  // CMSG_SPACE guarantees alignment
+      KJ_STACK_ARRAY(void*, cmsgSpace, msgBytes / sizeof(void*), 16, 256);
+      auto cmsgBytes = cmsgSpace.asBytes();
+      memset(cmsgBytes.begin(), 0, cmsgBytes.size());
+      msg.msg_control = cmsgBytes.begin();
+      msg.msg_controllen = cmsgBytes.size();
+
+#ifdef MSG_CMSG_CLOEXEC
+      static constexpr int RECVMSG_FLAGS = MSG_CMSG_CLOEXEC;
+#else
+      static constexpr int RECVMSG_FLAGS = 0;
+#endif
+
+      KJ_NONBLOCKING_SYSCALL(n = ::recvmsg(fd, &msg, RECVMSG_FLAGS)) {
+        // Error.
+
+        // We can't "return kj::READY_NOW;" inside this block because it causes a memory leak due to
+        // a bug that exists in both Clang and GCC:
+        //   http://gcc.gnu.org/bugzilla/show_bug.cgi?id=33799
+        //   http://llvm.org/bugs/show_bug.cgi?id=12286
+        goto error;
+      }
+
+      if (n >= 0) {
+        // Process all messages.
+        //
+        // WARNING DANGER: We have to be VERY careful not to miss a file descriptor here, because
+        // if we do, then that FD will never be closed, and a malicious peer could exploit this to
+        // fill up our FD table, creating a DoS attack. Some things to keep in mind:
+        // - CMSG_SPACE() could have rounded up the space for alignment purposes, and this could
+        //   mean we permitted the kernel to deliver more file descriptors than `maxFds`. We need
+        //   to close the extras.
+        // - We can receive multiple ancillary messages at once. In particular, there is also
+        //   SCM_CREDENTIALS. The sender decides what to send. They could send SCM_CREDENTIALS
+        //   first followed by SCM_RIGHTS. We need to make sure we see both.
+        size_t nfds = 0;
+        for (struct cmsghdr* cmsg = CMSG_FIRSTHDR(&msg);
+            cmsg != nullptr; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
+          if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) {
+            auto data = arrayPtr(reinterpret_cast<int*>(CMSG_DATA(cmsg)),
+                                 (cmsg->cmsg_len - CMSG_LEN(0)) / sizeof(int));
+            kj::Vector<kj::AutoCloseFd> trashFds;
+            for (auto fd: data) {
+              kj::AutoCloseFd ownFd(fd);
+              if (nfds < maxFds) {
+                fdBuffer[nfds++] = kj::mv(ownFd);
+              } else {
+                trashFds.add(kj::mv(ownFd));
+              }
+            }
+          }
+        }
+
+#ifndef MSG_CMSG_CLOEXEC
+        for (size_t i = 0; i < nfds; i++) {
+          setCloseOnExec(fdBuffer[i]);
+        }
+#endif
+
+        alreadyRead.capCount += nfds;
+        fdBuffer += nfds;
+        maxFds -= nfds;
+      }
+    }
+
     if (false) {
     error:
       return alreadyRead;
@@ -327,21 +394,22 @@ private:
     if (n < 0) {
       // Read would block.
       return observer.whenBecomesReadable().then([=]() {
-        return tryReadInternal(buffer, minBytes, maxBytes, alreadyRead);
+        return tryReadInternal(buffer, minBytes, maxBytes, fdBuffer, maxFds, alreadyRead);
       });
     } else if (n == 0) {
       // EOF -OR- maxBytes == 0.
       return alreadyRead;
     } else if (implicitCast<size_t>(n) >= minBytes) {
       // We read enough to stop here.
-      return alreadyRead + n;
+      alreadyRead.byteCount += n;
+      return alreadyRead;
     } else {
       // The kernel returned fewer bytes than we asked for (and fewer than we need).
 
       buffer = reinterpret_cast<byte*>(buffer) + n;
       minBytes -= n;
       maxBytes -= n;
-      alreadyRead += n;
+      alreadyRead.byteCount += n;
 
       KJ_IF_MAYBE(atEnd, observer.atEndHint()) {
         if (*atEnd) {
@@ -357,20 +425,21 @@ private:
           // let's go ahead and skip calling read() here and instead go straight to waiting for
           // more input.
           return observer.whenBecomesReadable().then([=]() {
-            return tryReadInternal(buffer, minBytes, maxBytes, alreadyRead);
+            return tryReadInternal(buffer, minBytes, maxBytes, fdBuffer, maxFds, alreadyRead);
           });
         }
       } else {
         // The kernel has not indicated one way or the other whether we are likely to be at EOF.
         // In this case we *must* keep calling read() until we either get a return of zero or
         // EAGAIN.
-        return tryReadInternal(buffer, minBytes, maxBytes, alreadyRead);
+        return tryReadInternal(buffer, minBytes, maxBytes, fdBuffer, maxFds, alreadyRead);
       }
     }
   }
 
   Promise<void> writeInternal(ArrayPtr<const byte> firstPiece,
-                              ArrayPtr<const ArrayPtr<const byte>> morePieces) {
+                              ArrayPtr<const ArrayPtr<const byte>> morePieces,
+                              ArrayPtr<const int> fds) {
     const size_t iovmax = kj::miniposix::iovMax(1 + morePieces.size());
     // If there are more than IOV_MAX pieces, we'll only write the first IOV_MAX for now, and
     // then we'll loop later.
@@ -387,8 +456,14 @@ private:
       iovTotal += iov[i].iov_len;
     }
 
-    ssize_t writeResult;
-    KJ_NONBLOCKING_SYSCALL(writeResult = ::writev(fd, iov.begin(), iov.size())) {
+    if (iovTotal == 0) {
+      KJ_REQUIRE(fds.size() == 0, "can't write FDs without bytes");
+      return kj::READY_NOW;
+    }
+
+    ssize_t n;
+    if (fds.size() == 0) {
+      KJ_NONBLOCKING_SYSCALL(n = ::writev(fd, iov.begin(), iov.size())) {
         // Error.
 
         // We can't "return kj::READY_NOW;" inside this block because it causes a memory leak due to
@@ -397,13 +472,64 @@ private:
         //   http://llvm.org/bugs/show_bug.cgi?id=12286
         goto error;
       }
+    } else {
+      struct msghdr msg;
+      memset(&msg, 0, sizeof(msg));
+      msg.msg_iov = iov.begin();
+      msg.msg_iovlen = iov.size();
+
+      // Allocate space to receive a cmsg.
+      size_t msgBytes = CMSG_SPACE(sizeof(int) * fds.size());
+      KJ_ASSERT(msgBytes % sizeof(void*) == 0);  // CMSG_SPACE guarantees alignment
+      KJ_STACK_ARRAY(void*, cmsgSpace, msgBytes / sizeof(void*), 16, 256);
+      auto cmsgBytes = cmsgSpace.asBytes();
+      memset(cmsgBytes.begin(), 0, cmsgBytes.size());
+      msg.msg_control = cmsgBytes.begin();
+      msg.msg_controllen = cmsgBytes.size();
+
+      struct cmsghdr* cmsg = CMSG_FIRSTHDR(&msg);
+      cmsg->cmsg_level = SOL_SOCKET;
+      cmsg->cmsg_type = SCM_RIGHTS;
+      cmsg->cmsg_len = CMSG_LEN(sizeof(int) * fds.size());
+      memcpy(CMSG_DATA(cmsg), fds.begin(), fds.asBytes().size());
+
+      KJ_NONBLOCKING_SYSCALL(n = ::sendmsg(fd, &msg, 0)) {
+        // Error.
+
+        // We can't "return kj::READY_NOW;" inside this block because it causes a memory leak due to
+        // a bug that exists in both Clang and GCC:
+        //   http://gcc.gnu.org/bugzilla/show_bug.cgi?id=33799
+        //   http://llvm.org/bugs/show_bug.cgi?id=12286
+        goto error;
+      }
+    }
+
     if (false) {
     error:
       return kj::READY_NOW;
     }
 
-    // A negative result means EAGAIN, which we can treat the same as having written zero bytes.
-    size_t n = writeResult < 0 ? 0 : writeResult;
+    if (n < 0) {
+      // Got EAGAIN. Nothing was written.
+      return observer.whenBecomesWritable().then([=]() {
+        return writeInternal(firstPiece, morePieces, fds);
+      });
+    } else if (n == 0) {
+      // Why would a sendmsg() with a non-empty message ever return 0 when writing to a stream
+      // socket? If there's no room in the send buffer, it should fail with EAGAIN. If the
+      // connection is closed, it should fail with EPIPE. Various documents and forum posts around
+      // the internet claim this can happen but no one seems to know when. My guess is it can only
+      // happen if we try to send an empty message -- which we didn't. So I think this is
+      // impossible. If it is possible, we need to figure out how to correctly handle it, which
+      // depends on what caused it.
+      //
+      // Note in particular that if 0 is a valid return here, and we sent an SCM_RIGHTS message,
+      // we need to know whether the message was sent or not, in order to decide whether to retry
+      // sending it!
+      KJ_FAIL_ASSERT("non-empty sendmsg() returned 0");
+    }
+
+    // Non-zero bytes were written. This also implies that *all* FDs were written.
 
     // Discard all data that was written, then issue a new write for what's left (if any).
     for (;;) {
@@ -414,11 +540,11 @@ private:
 
         if (iovTotal == 0) {
           // Oops, what actually happened is that we hit the IOV_MAX limit. Don't wait.
-          return writeInternal(firstPiece, morePieces);
+          return writeInternal(firstPiece, morePieces, nullptr);
         }
 
         return observer.whenBecomesWritable().then([=]() {
-          return writeInternal(firstPiece, morePieces);
+          return writeInternal(firstPiece, morePieces, nullptr);
         });
       } else if (morePieces.size() == 0) {
         // First piece was fully-consumed and there are no more pieces, so we're done.

c++/src/kj/async-io.c++

@@ -1804,6 +1804,25 @@ Tee newTee(Own<AsyncInputStream> input, uint64_t limit) {
   return { { mv(branch1), mv(branch2) } };
 }
 
+Promise<void> AsyncCapabilityStream::writeWithFds(
+    ArrayPtr<const byte> data, ArrayPtr<const ArrayPtr<const byte>> moreData,
+    ArrayPtr<const AutoCloseFd> fds) {
+  // HACK: AutoCloseFd actually contains an `int` under the hood. We can reinterpret_cast to avoid
+  //   unnecessary memory allocation.
+  static_assert(sizeof(AutoCloseFd) == sizeof(int));
+  auto intArray = arrayPtr(reinterpret_cast<const int*>(fds.begin()), fds.size());
+
+  // Be extra-paranoid about aliasing rules by injecting a compiler barrier here. Probably
+  // not necessary but also probably doesn't hurt.
+#if _MSC_VER
+  _ReadWriteBarrier();
+#else
+  __asm__ __volatile__("": : :"memory");
+#endif
+
+  return writeWithFds(data, moreData, intArray);
+}
+
 Promise<Own<AsyncCapabilityStream>> AsyncCapabilityStream::receiveStream() {
   return tryReceiveStream()
       .then([](Maybe<Own<AsyncCapabilityStream>>&& result)
@@ -1816,6 +1835,35 @@ Promise<Own<AsyncCapabilityStream>> AsyncCapabilityStream::receiveStream() {
   });
 }
 
+kj::Promise<Maybe<Own<AsyncCapabilityStream>>> AsyncCapabilityStream::tryReceiveStream() {
+  struct ResultHolder {
+    byte b;
+    Own<AsyncCapabilityStream> stream;
+  };
+  auto result = kj::heap<ResultHolder>();
+  auto promise = tryReadWithStreams(&result->b, 1, 1, &result->stream, 1);
+  return promise.then([result = kj::mv(result)](ReadResult actual) mutable
+                      -> Maybe<Own<AsyncCapabilityStream>> {
+    if (actual.byteCount == 0) {
+      return nullptr;
+    }
+
+    KJ_REQUIRE(actual.capCount == 1,
+        "expected to receive a capability (e.g. file descirptor via SCM_RIGHTS), but didn't") {
+      return nullptr;
+    }
+
+    return kj::mv(result->stream);
+  });
+}
+
+Promise<void> AsyncCapabilityStream::sendStream(Own<AsyncCapabilityStream> stream) {
+  static constexpr byte b = 0;
+  auto streams = kj::heapArray<Own<AsyncCapabilityStream>>(1);
+  streams[0] = kj::mv(stream);
+  return writeWithStreams(arrayPtr(&b, 1), nullptr, kj::mv(streams));
+}
+
 Promise<AutoCloseFd> AsyncCapabilityStream::receiveFd() {
   return tryReceiveFd().then([](Maybe<AutoCloseFd>&& result) -> Promise<AutoCloseFd> {
     KJ_IF_MAYBE(r, result) {
@@ -1825,11 +1873,35 @@ Promise<AutoCloseFd> AsyncCapabilityStream::receiveFd() {
     }
   });
 }
-Promise<Maybe<AutoCloseFd>> AsyncCapabilityStream::tryReceiveFd() {
-  return KJ_EXCEPTION(UNIMPLEMENTED, "this stream cannot receive file descriptors");
+
+kj::Promise<kj::Maybe<AutoCloseFd>> AsyncCapabilityStream::tryReceiveFd() {
+  struct ResultHolder {
+    byte b;
+    AutoCloseFd fd;
+  };
+  auto result = kj::heap<ResultHolder>();
+  auto promise = tryReadWithFds(&result->b, 1, 1, &result->fd, 1);
+  return promise.then([result = kj::mv(result)](ReadResult actual) mutable
+                      -> Maybe<AutoCloseFd> {
+    if (actual.byteCount == 0) {
+      return nullptr;
+    }
+
+    KJ_REQUIRE(actual.capCount == 1,
+        "expected to receive a file descriptor (e.g. via SCM_RIGHTS), but didn't") {
+      return nullptr;
+    }
+
+    return kj::mv(result->fd);
+  });
 }
+
 Promise<void> AsyncCapabilityStream::sendFd(int fd) {
-  return KJ_EXCEPTION(UNIMPLEMENTED, "this stream cannot send file descriptors");
+  static constexpr byte b = 0;
+  auto fds = kj::heapArray<int>(1);
+  fds[0] = fd;
+  auto promise = writeWithFds(arrayPtr(&b, 1), nullptr, fds);
+  return promise.attach(kj::mv(fds));
 }
 
 void AsyncIoStream::getsockopt(int level, int option, void* value, uint* length) {

c++/src/kj/async-io.h

@@ -175,15 +175,57 @@ class AsyncCapabilityStream: public AsyncIoStream {
   // broker, or in terms of direct handle passing if at least one process trusts the other.
 
 public:
+  virtual Promise<void> writeWithFds(ArrayPtr<const byte> data,
+                                     ArrayPtr<const ArrayPtr<const byte>> moreData,
+                                     ArrayPtr<const int> fds) = 0;
+  Promise<void> writeWithFds(ArrayPtr<const byte> data,
+                             ArrayPtr<const ArrayPtr<const byte>> moreData,
+                             ArrayPtr<const AutoCloseFd> fds);
+  // Write some data to the stream with some file descirptors attached to it.
+  //
+  // The maximum number of FDs that can be sent at a time is usually subject to an OS-imposed
+  // limit. On Linux, this is 253. In practice, sending more than a handful of FDs at once is
+  // probably a bad idea.
+
+  struct ReadResult {
+    size_t byteCount;
+    size_t capCount;
+  };
+
+  virtual Promise<ReadResult> tryReadWithFds(void* buffer, size_t minBytes, size_t maxBytes,
+                                             AutoCloseFd* fdBuffer, size_t maxFds) = 0;
+  // Read data from the stream that may have file descriptors attached. Any attached descriptors
+  // will be added to `fds`. If multiple bundles of FDs are encountered in the course of reading
+  // the amount of data requested by minBytes/maxBytes, then they will be concatenated. If more FDs
+  // are received than fit in the buffer, then the excess will be discarded and closed -- this
+  // behavior, while ugly, is important to defend against denial-of-service attacks that may fill
+  // up the FD table with garbage. Applications must think carefully about how many FDs they really
+  // need to receive at once and set a well-defined limit.
+
+  virtual Promise<void> writeWithStreams(ArrayPtr<const byte> data,
+                                         ArrayPtr<const ArrayPtr<const byte>> moreData,
+                                         Array<Own<AsyncCapabilityStream>> streams) = 0;
+  virtual Promise<ReadResult> tryReadWithStreams(
+      void* buffer, size_t minBytes, size_t maxBytes,
+      Own<AsyncCapabilityStream>* streamBuffer, size_t maxStreams) = 0;
+  // Like above, but passes AsyncCapabilityStream objects. The stream implementations must be from
+  // the same AsyncIoProvider.
+
+  // ---------------------------------------------------------------------------
+  // Helpers for sending individual capabilities.
+  //
+  // These are equivalent to the above methods with the constraint that only one FD is
+  // sent/received at a time and the corresponding data is a single zero-valued byte.
+
   Promise<Own<AsyncCapabilityStream>> receiveStream();
-  virtual Promise<Maybe<Own<AsyncCapabilityStream>>> tryReceiveStream() = 0;
-  virtual Promise<void> sendStream(Own<AsyncCapabilityStream> stream) = 0;
-  // Transfer a stream.
+  Promise<Maybe<Own<AsyncCapabilityStream>>> tryReceiveStream();
+  Promise<void> sendStream(Own<AsyncCapabilityStream> stream);
+  // Transfer a single stream.
 
   Promise<AutoCloseFd> receiveFd();
-  virtual Promise<Maybe<AutoCloseFd>> tryReceiveFd();
-  virtual Promise<void> sendFd(int fd);
-  // Transfer a raw file descriptor. Default implementation throws UNIMPLEMENTED.
+  Promise<Maybe<AutoCloseFd>> tryReceiveFd();
+  Promise<void> sendFd(int fd);
+  // Transfer a single raw file descriptor.
 };
 
 struct OneWayPipe {