Add asBytes() and asChars() methods to array classes to reinterpret-cast to…

Add asBytes() and asChars() methods to array classes to reinterpret-cast to bytes / chars, since this happens all the time and is otherwise a huge pain.

Use the new methods in a bunch of places.

c++/src/capnp/blob.h

@@ -100,7 +100,7 @@ public:
   inline Builder(decltype(nullptr)): ArrayPtr<byte>(nullptr) {}
   inline Builder(byte* value, size_t size): ArrayPtr<byte>(value, size) {}
   inline Builder(kj::Array<byte>& value): ArrayPtr<byte>(value) {}
-  inline Builder(ArrayPtr<byte>& value): ArrayPtr<byte>(value) {}
+  inline Builder(ArrayPtr<byte> value): ArrayPtr<byte>(value) {}
 
   inline Data::Reader asReader() const { return Data::Reader(*this); }
   inline operator Reader() const { return asReader(); }
@@ -124,6 +124,8 @@ public:
   inline kj::ArrayPtr<char> asArray();
   inline operator kj::ArrayPtr<const char>() const;
   inline kj::ArrayPtr<const char> asArray() const;
+  inline kj::ArrayPtr<byte> asBytes() { return asArray().asBytes(); }
+  inline kj::ArrayPtr<const byte> asBytes() const { return asArray().asBytes(); }
   // Result does not include NUL terminator.
 
   inline operator kj::StringPtr() const;

c++/src/capnp/compiler/capnp.c++

@@ -1040,8 +1040,7 @@ public:
       for (;;) {
         auto buf = input.tryGetReadBuffer();
         if (buf.size() == 0) break;
-        allText.addAll(reinterpret_cast<const char*>(buf.begin()),
-                       reinterpret_cast<const char*>(buf.end()));
+        allText.addAll(buf.asChars());
         input.skip(buf.size());
       }
     }

c++/src/capnp/compiler/lexer.c++

@@ -181,11 +181,9 @@ Lexer::Lexer(Orphanage orphanageParam, ErrorReporter& errorReporter)
             return t;
           }),
       p::transformWithLocation(p::doubleQuotedHexBinary,
-          [this](Location loc, kj::Array<char> data) -> Orphan<Token> {
+          [this](Location loc, kj::Array<byte> data) -> Orphan<Token> {
             auto t = orphanage.newOrphan<Token>();
-            kj::ArrayPtr<byte> dataPtr(reinterpret_cast<byte*>(data.begin()),
-                                       reinterpret_cast<byte*>(data.end()));
-            initTok(t, loc).setBinaryLiteral(dataPtr);
+            initTok(t, loc).setBinaryLiteral(data);
             return t;
           }),
       p::transformWithLocation(p::integer,

c++/src/capnp/compiler/md5.h

@@ -44,7 +44,7 @@ public:
 
   void update(kj::ArrayPtr<const kj::byte> data);
   inline void update(kj::ArrayPtr<const char> data) {
-    return update(kj::arrayPtr(reinterpret_cast<const kj::byte*>(data.begin()), data.size()));
+    return update(data.asBytes());
   }
   inline void update(kj::StringPtr data) {
     return update(data.asArray());

c++/src/capnp/compiler/node-translator.c++

@@ -2717,8 +2717,7 @@ Orphan<DynamicValue> ValueTranslator::compileValueInner(Expression::Reader src,
     case Expression::STRING:
       if (type.isData()) {
         Text::Reader text = src.getString();
-        return orphanage.newOrphanCopy(Data::Reader(
-            reinterpret_cast<const byte*>(text.begin()), text.size()));
+        return orphanage.newOrphanCopy(Data::Reader(text.asBytes()));
       } else {
         return orphanage.newOrphanCopy(src.getString());
       }

c++/src/capnp/dynamic.c++

@@ -1749,8 +1749,7 @@ PipelineFor<DynamicCapability> DynamicValue::Pipeline::AsImpl<DynamicCapability>
 Data::Reader DynamicValue::Reader::AsImpl<Data>::apply(const Reader& reader) {
   if (reader.type == TEXT) {
     // Coerce text to data.
-    return Data::Reader(reinterpret_cast<const byte*>(reader.textValue.begin()),
-                        reader.textValue.size());
+    return reader.textValue.asBytes();
   }
   KJ_REQUIRE(reader.type == DATA, "Value type mismatch.") {
     return Data::Reader();
@@ -1760,8 +1759,7 @@ Data::Reader DynamicValue::Reader::AsImpl<Data>::apply(const Reader& reader) {
 Data::Builder DynamicValue::Builder::AsImpl<Data>::apply(Builder& builder) {
   if (builder.type == TEXT) {
     // Coerce text to data.
-    return Data::Builder(reinterpret_cast<byte*>(builder.textValue.begin()),
-                         builder.textValue.size());
+    return builder.textValue.asBytes();
   }
   KJ_REQUIRE(builder.type == DATA, "Value type mismatch.") {
     return BuilderFor<Data>();

c++/src/capnp/orphan-test.c++

@@ -899,7 +899,7 @@ TEST(Orphans, ReferenceExternalData) {
   {
     auto segments = builder.getSegmentsForOutput();
     ASSERT_EQ(2, segments.size());
-    EXPECT_EQ(data, reinterpret_cast<const byte*>(segments[1].begin()));
+    EXPECT_EQ(data, segments[1].asBytes().begin());
     EXPECT_EQ((sizeof(data) + 7) / 8, segments[1].size());
   }
 

c++/src/capnp/schema-loader.c++

@@ -1706,8 +1706,7 @@ kj::ArrayPtr<const T> SchemaLoader::Impl::copyDeduped(kj::ArrayPtr<const T> valu
     return kj::arrayPtr(kj::implicitCast<const T*>(nullptr), 0);
   }
 
-  auto bytes = kj::arrayPtr(
-      reinterpret_cast<const byte*>(values.begin()), values.size() * sizeof(T));
+  auto bytes = values.asBytes();
 
   auto iter = dedupTable.find(bytes);
   if (iter != dedupTable.end()) {
@@ -1718,8 +1717,7 @@ kj::ArrayPtr<const T> SchemaLoader::Impl::copyDeduped(kj::ArrayPtr<const T> valu
   auto copy = arena.allocateArray<T>(values.size());
   memcpy(copy.begin(), values.begin(), values.size() * sizeof(T));
 
-  bytes = kj::arrayPtr(reinterpret_cast<const byte*>(copy.begin()), values.size() * sizeof(T));
-  KJ_ASSERT(dedupTable.insert(bytes).second);
+  KJ_ASSERT(dedupTable.insert(copy.asBytes()).second);
 
   return copy;
 }

c++/src/capnp/serialize-async.c++

@@ -205,12 +205,10 @@ kj::Promise<void> writeMessage(kj::AsyncOutputStream& output,
   }
 
   arrays.pieces = kj::heapArray<kj::ArrayPtr<const byte>>(segments.size() + 1);
-  arrays.pieces[0] = kj::arrayPtr(reinterpret_cast<byte*>(arrays.table.begin()),
-                                  arrays.table.size() * sizeof(arrays.table[0]));
+  arrays.pieces[0] = arrays.table.asBytes();
 
   for (uint i = 0; i < segments.size(); i++) {
-    arrays.pieces[i + 1] = kj::arrayPtr(reinterpret_cast<const byte*>(segments[i].begin()),
-                                        reinterpret_cast<const byte*>(segments[i].end()));
+    arrays.pieces[i + 1] = segments[i].asBytes();
   }
 
   auto promise = output.write(arrays.pieces);

c++/src/capnp/serialize-packed-test.c++

@@ -106,8 +106,7 @@ std::ostream& operator<<(std::ostream& os, const DisplayByteArray& bytes) {
   return os;
 }
 
-void expectPacksTo(std::initializer_list<uint8_t> unpacked,
-                   std::initializer_list<uint8_t> packed) {
+void expectPacksTo(kj::ArrayPtr<const byte> unpacked, kj::ArrayPtr<const byte> packed) {
   TestPipe pipe;
 
   // -----------------------------------------------------------------
@@ -119,7 +118,7 @@ void expectPacksTo(std::initializer_list<uint8_t> unpacked,
     packedOut.write(unpacked.begin(), unpacked.size());
   }
 
-  if (pipe.getData() != std::string(reinterpret_cast<const char*>(packed.begin()), packed.size())) {
+  if (pipe.getData() != std::string(packed.asChars().begin(), packed.asChars().size())) {
     ADD_FAILURE()
         << "Tried to pack: " << DisplayByteArray(unpacked) << "\n"
         << "Expected:      " << DisplayByteArray(packed) << "\n"

c++/src/capnp/serialize-test.c++

@@ -131,8 +131,8 @@ TEST(Serialize, FlatArrayEvenSegmentCount) {
 class TestInputStream: public kj::InputStream {
 public:
   TestInputStream(kj::ArrayPtr<const word> data, bool lazy)
-      : pos(reinterpret_cast<const char*>(data.begin())),
-        end(reinterpret_cast<const char*>(data.end())),
+      : pos(data.asChars().begin()),
+        end(data.asChars().end()),
         lazy(lazy) {}
   ~TestInputStream() {}
 
@@ -246,7 +246,7 @@ public:
 
   const bool dataEquals(kj::ArrayPtr<const word> other) {
     return data ==
-        std::string(reinterpret_cast<const char*>(other.begin()), other.size() * sizeof(word));
+        std::string(other.asChars().begin(), other.asChars().size());
   }
 
 private:

c++/src/capnp/serialize.c++

@@ -199,7 +199,7 @@ InputStreamMessageReader::InputStreamMessageReader(
   if (segmentCount == 1) {
     inputStream.read(scratchSpace.begin(), totalWords * sizeof(word));
   } else if (segmentCount > 1) {
-    readPos = reinterpret_cast<byte*>(scratchSpace.begin());
+    readPos = scratchSpace.asBytes().begin();
     readPos += inputStream.read(readPos, segment0Size * sizeof(word), totalWords * sizeof(word));
   }
 }
@@ -256,11 +256,10 @@ void writeMessage(kj::OutputStream& output, kj::ArrayPtr<const kj::ArrayPtr<cons
   }
 
   KJ_STACK_ARRAY(kj::ArrayPtr<const byte>, pieces, segments.size() + 1, 4, 32);
-  pieces[0] = kj::arrayPtr(reinterpret_cast<byte*>(table.begin()), table.size() * sizeof(table[0]));
+  pieces[0] = table.asBytes();
 
   for (uint i = 0; i < segments.size(); i++) {
-    pieces[i + 1] = kj::arrayPtr(reinterpret_cast<const byte*>(segments[i].begin()),
-                                 reinterpret_cast<const byte*>(segments[i].end()));
+    pieces[i + 1] = segments[i].asBytes();
   }
 
   output.write(pieces);

c++/src/capnp/serialize.h

@@ -75,6 +75,14 @@ private:
 
 kj::Array<word> messageToFlatArray(MessageBuilder& builder);
 // Constructs a flat array containing the entire content of the given message.
+//
+// To output the message as bytes, use `.toBytes()` on the returned word array. Keep in mind that
+// toBytes() returns an ArrayPtr, so you have to save the Array as well to prevent it from being
+// deleted. For example:
+//
+//     kj::Array<capnp::word> words = messageToFlatArray(myMessage);
+//     kj::Array<kj::byte> bytes = words.toBytes();
+//     write(fd, bytes.begin(), bytes.size());
 
 kj::Array<word> messageToFlatArray(kj::ArrayPtr<const kj::ArrayPtr<const word>> segments);
 // Version of messageToFlatArray that takes a raw segment array.

c++/src/capnp/stringify.c++

@@ -144,8 +144,7 @@ static kj::StringTree print(const DynamicValue::Reader& value,
       // TODO(now): Data should be printed as binary literal.
       kj::ArrayPtr<const char> chars;
       if (value.getType() == DynamicValue::DATA) {
-        auto reader = value.as<Data>();
-        chars = kj::arrayPtr(reinterpret_cast<const char*>(reader.begin()), reader.size());
+        chars = value.as<Data>().asChars();
       } else {
         chars = value.as<Text>();
       }

c++/src/capnp/test-util.h

@@ -52,10 +52,10 @@
 namespace capnp {
 
 inline std::ostream& operator<<(std::ostream& os, const Data::Reader& value) {
-  return os.write(reinterpret_cast<const char*>(value.begin()), value.size());
+  return os.write(value.asChars().begin(), value.asChars().size());
 }
 inline std::ostream& operator<<(std::ostream& os, const Data::Builder& value) {
-  return os.write(reinterpret_cast<const char*>(value.begin()), value.size());
+  return os.write(value.asChars().begin(), value.asChars().size());
 }
 inline std::ostream& operator<<(std::ostream& os, const Text::Reader& value) {
   return os.write(value.begin(), value.size());

c++/src/kj/array.h

@@ -186,6 +186,11 @@ public:
     return ArrayPtr<const T>(ptr + start, end - start);
   }
 
+  inline ArrayPtr<const byte> asBytes() const { return asPtr().asBytes(); }
+  inline ArrayPtr<PropagateConst<T, byte>> asBytes() { return asPtr().asBytes(); }
+  inline ArrayPtr<const char> asChars() const { return asPtr().asChars(); }
+  inline ArrayPtr<PropagateConst<T, char>> asChars() { return asPtr().asChars(); }
+
   inline bool operator==(decltype(nullptr)) const { return size_ == 0; }
   inline bool operator!=(decltype(nullptr)) const { return size_ != 0; }
 

c++/src/kj/common-test.c++

@@ -20,6 +20,7 @@
 // THE SOFTWARE.
 
 #include "common.h"
+#include <inttypes.h>
 #include <gtest/gtest.h>
 
 namespace kj {
@@ -324,5 +325,124 @@ TEST(Common, CanConvert) {
   static_assert(!canConvert<const void*, void*>(), "failure");
 }
 
+TEST(Common, ArrayAsBytes) {
+  uint32_t raw[] = { 0x12345678u, 0x9abcdef0u };
+
+  ArrayPtr<uint32_t> array = raw;
+  ASSERT_EQ(2, array.size());
+  EXPECT_EQ(0x12345678u, array[0]);
+  EXPECT_EQ(0x9abcdef0u, array[1]);
+
+  {
+    ArrayPtr<byte> bytes = array.asBytes();
+    ASSERT_EQ(8, bytes.size());
+
+    if (bytes[0] == '\x12') {
+      // big-endian
+      EXPECT_EQ(0x12u, bytes[0]);
+      EXPECT_EQ(0x34u, bytes[1]);
+      EXPECT_EQ(0x56u, bytes[2]);
+      EXPECT_EQ(0x78u, bytes[3]);
+      EXPECT_EQ(0x9au, bytes[4]);
+      EXPECT_EQ(0xbcu, bytes[5]);
+      EXPECT_EQ(0xdeu, bytes[6]);
+      EXPECT_EQ(0xf0u, bytes[7]);
+    } else {
+      // little-endian
+      EXPECT_EQ(0x12u, bytes[3]);
+      EXPECT_EQ(0x34u, bytes[2]);
+      EXPECT_EQ(0x56u, bytes[1]);
+      EXPECT_EQ(0x78u, bytes[0]);
+      EXPECT_EQ(0x9au, bytes[7]);
+      EXPECT_EQ(0xbcu, bytes[6]);
+      EXPECT_EQ(0xdeu, bytes[5]);
+      EXPECT_EQ(0xf0u, bytes[4]);
+    }
+  }
+
+  {
+    ArrayPtr<char> chars = array.asChars();
+    ASSERT_EQ(8, chars.size());
+
+    if (chars[0] == '\x12') {
+      // big-endian
+      EXPECT_EQ('\x12', chars[0]);
+      EXPECT_EQ('\x34', chars[1]);
+      EXPECT_EQ('\x56', chars[2]);
+      EXPECT_EQ('\x78', chars[3]);
+      EXPECT_EQ('\x9a', chars[4]);
+      EXPECT_EQ('\xbc', chars[5]);
+      EXPECT_EQ('\xde', chars[6]);
+      EXPECT_EQ('\xf0', chars[7]);
+    } else {
+      // little-endian
+      EXPECT_EQ('\x12', chars[3]);
+      EXPECT_EQ('\x34', chars[2]);
+      EXPECT_EQ('\x56', chars[1]);
+      EXPECT_EQ('\x78', chars[0]);
+      EXPECT_EQ('\x9a', chars[7]);
+      EXPECT_EQ('\xbc', chars[6]);
+      EXPECT_EQ('\xde', chars[5]);
+      EXPECT_EQ('\xf0', chars[4]);
+    }
+  }
+
+  ArrayPtr<const uint32_t> constArray = array;
+
+  {
+    ArrayPtr<const byte> bytes = constArray.asBytes();
+    ASSERT_EQ(8, bytes.size());
+
+    if (bytes[0] == '\x12') {
+      // big-endian
+      EXPECT_EQ(0x12u, bytes[0]);
+      EXPECT_EQ(0x34u, bytes[1]);
+      EXPECT_EQ(0x56u, bytes[2]);
+      EXPECT_EQ(0x78u, bytes[3]);
+      EXPECT_EQ(0x9au, bytes[4]);
+      EXPECT_EQ(0xbcu, bytes[5]);
+      EXPECT_EQ(0xdeu, bytes[6]);
+      EXPECT_EQ(0xf0u, bytes[7]);
+    } else {
+      // little-endian
+      EXPECT_EQ(0x12u, bytes[3]);
+      EXPECT_EQ(0x34u, bytes[2]);
+      EXPECT_EQ(0x56u, bytes[1]);
+      EXPECT_EQ(0x78u, bytes[0]);
+      EXPECT_EQ(0x9au, bytes[7]);
+      EXPECT_EQ(0xbcu, bytes[6]);
+      EXPECT_EQ(0xdeu, bytes[5]);
+      EXPECT_EQ(0xf0u, bytes[4]);
+    }
+  }
+
+  {
+    ArrayPtr<const char> chars = constArray.asChars();
+    ASSERT_EQ(8, chars.size());
+
+    if (chars[0] == '\x12') {
+      // big-endian
+      EXPECT_EQ('\x12', chars[0]);
+      EXPECT_EQ('\x34', chars[1]);
+      EXPECT_EQ('\x56', chars[2]);
+      EXPECT_EQ('\x78', chars[3]);
+      EXPECT_EQ('\x9a', chars[4]);
+      EXPECT_EQ('\xbc', chars[5]);
+      EXPECT_EQ('\xde', chars[6]);
+      EXPECT_EQ('\xf0', chars[7]);
+    } else {
+      // little-endian
+      EXPECT_EQ('\x12', chars[3]);
+      EXPECT_EQ('\x34', chars[2]);
+      EXPECT_EQ('\x56', chars[1]);
+      EXPECT_EQ('\x78', chars[0]);
+      EXPECT_EQ('\x9a', chars[7]);
+      EXPECT_EQ('\xbc', chars[6]);
+      EXPECT_EQ('\xde', chars[5]);
+      EXPECT_EQ('\xf0', chars[4]);
+    }
+  }
+}
+
 }  // namespace
 }  // namespace kj

c++/src/kj/common.h

@@ -396,6 +396,13 @@ template <typename T> struct IsReference_ { static constexpr bool value = false;
 template <typename T> struct IsReference_<T&> { static constexpr bool value = true; };
 template <typename T> constexpr bool isReference() { return IsReference_<T>::value; }
 
+template <typename From, typename To>
+struct PropagateConst_ { typedef To Type; };
+template <typename From, typename To>
+struct PropagateConst_<const From, To> { typedef const To Type; };
+template <typename From, typename To>
+using PropagateConst = typename PropagateConst_<From, To>::Type;
+
 namespace _ {  // private
 
 template <typename T>
@@ -1138,6 +1145,17 @@ public:
     return ArrayPtr(ptr + start, end - start);
   }
 
+  inline ArrayPtr<PropagateConst<T, byte>> asBytes() const {
+    // Reinterpret the array as a byte array. This is explicitly legal under C++ aliasing
+    // rules.
+    return { reinterpret_cast<PropagateConst<T, byte>*>(ptr), size_ * sizeof(T) };
+  }
+  inline ArrayPtr<PropagateConst<T, char>> asChars() const {
+    // Reinterpret the array as a char array. This is explicitly legal under C++ aliasing
+    // rules.
+    return { reinterpret_cast<PropagateConst<T, char>*>(ptr), size_ * sizeof(T) };
+  }
+
   inline bool operator==(decltype(nullptr)) const { return size_ == 0; }
   inline bool operator!=(decltype(nullptr)) const { return size_ != 0; }
 

c++/src/kj/parse/char.h

@@ -294,6 +294,12 @@ struct ParseHexEscape {
   }
 };
 
+struct ParseHexByte {
+  inline byte operator()(char first, char second) const {
+    return (parseDigit(first) << 4) | parseDigit(second);
+  }
+};
+
 struct ParseOctEscape {
   inline char operator()(char first, Maybe<char> second, Maybe<char> third) const {
     char result = first - '0';
@@ -332,10 +338,10 @@ constexpr auto singleQuotedString = charsToString(sequence(
 
 constexpr auto doubleQuotedHexBinary = sequence(
     exactChar<'0'>(), exactChar<'x'>(), exactChar<'\"'>(),
-    oneOrMore(transform(sequence(discardWhitespace, hexDigit, hexDigit), _::ParseHexEscape())),
+    oneOrMore(transform(sequence(discardWhitespace, hexDigit, hexDigit), _::ParseHexByte())),
     discardWhitespace,
     exactChar<'\"'>());
-// Parses a double-quoted hex binary literal.
+// Parses a double-quoted hex binary literal. Returns Array<byte>.
 
 }  // namespace parse
 }  // namespace kj

c++/src/kj/string.h

@@ -77,6 +77,7 @@ public:
 
   inline operator ArrayPtr<const char>() const;
   inline ArrayPtr<const char> asArray() const;
+  inline ArrayPtr<const byte> asBytes() const { return asArray().asBytes(); }
   // Result does not include NUL terminator.
 
   inline const char* cStr() const { return content.begin(); }
@@ -143,6 +144,8 @@ public:
   inline operator ArrayPtr<const char>() const;
   inline ArrayPtr<char> asArray();
   inline ArrayPtr<const char> asArray() const;
+  inline ArrayPtr<byte> asBytes() { return asArray().asBytes(); }
+  inline ArrayPtr<const byte> asBytes() const { return asArray().asBytes(); }
   // Result does not include NUL terminator.
 
   inline const char* cStr() const;