More benchmark updates, delete unused descriptor code.

c++/src/capnproto/benchmark/capnproto-carsales.c++

@@ -93,6 +93,8 @@ void randomCar(Car::Builder car) {
   engine.setHorsepower(100 * fastRand(400));
   engine.setCylinders(4 + 2 * fastRand(3));
   engine.setCc(800 + fastRand(10000));
+  engine.setUsesGas(true);
+  engine.setUsesElectric(fastRand(2));
 
   car.setFuelCapacity(10.0 + fastRandDouble(30.0));
   car.setFuelLevel(fastRandDouble(car.getFuelCapacity()));

c++/src/capnproto/benchmark/capnproto-common.h

@@ -27,7 +27,9 @@
 #include "common.h"
 #include <capnproto/serialize.h>
 #include <capnproto/serialize-packed.h>
+#if HAVE_SNAPPY
 #include <capnproto/serialize-snappy.h>
+#endif  // HAVE_SNAPPY
 #include <thread>
 
 namespace capnproto {
@@ -96,6 +98,7 @@ struct Packed {
   }
 };
 
+#if HAVE_SNAPPY
 static byte snappyReadBuffer[SNAPPY_BUFFER_SIZE];
 static byte snappyWriteBuffer[SNAPPY_BUFFER_SIZE];
 static byte snappyCompressedBuffer[SNAPPY_COMPRESSED_BUFFER_SIZE];
@@ -120,6 +123,7 @@ struct SnappyCompressed {
         arrayPtr(snappyCompressedBuffer, SNAPPY_COMPRESSED_BUFFER_SIZE));
   }
 };
+#endif  // HAVE_SNAPPY
 
 // =======================================================================================
 
@@ -395,7 +399,9 @@ struct BenchmarkMethods {
 struct BenchmarkTypes {
   typedef capnp::Uncompressed Uncompressed;
   typedef capnp::Packed Packed;
+#if HAVE_SNAPPY
   typedef capnp::SnappyCompressed SnappyCompressed;
+#endif  // HAVE_SNAPPY
 
   typedef capnp::UseScratch ReusableResources;
   typedef capnp::NoScratch SingleUseResources;

c++/src/capnproto/benchmark/common.h

@@ -254,9 +254,11 @@ uint64_t doBenchmark3(const std::string& mode, const std::string& reuse,
   } else if (compression == "packed") {
     return doBenchmark2<BenchmarkTypes, TestCase, typename BenchmarkTypes::Packed>(
         mode, reuse, iters);
+#if HAVE_SNAPPY
   } else if (compression == "snappy") {
     return doBenchmark2<BenchmarkTypes, TestCase, typename BenchmarkTypes::SnappyCompressed>(
         mode, reuse, iters);
+#endif  // HAVE_SNAPPY
   } else {
     fprintf(stderr, "Unknown compression mode: %s\n", compression.c_str());
     exit(1);

c++/src/capnproto/benchmark/null-carsales.c++

@@ -27,7 +27,7 @@ namespace capnproto {
 namespace benchmark {
 namespace null {
 
-enum class Color {
+enum class Color: uint8_t {
   BLACK,
   WHITE,
   RED,
@@ -41,38 +41,55 @@ enum class Color {
 constexpr uint COLOR_RANGE = static_cast<uint>(Color::SILVER) + 1;
 
 struct Wheel {
-  uint16_t diameter;
   float airPressure;
+  uint16_t diameter;
   bool snowTires;
 };
 
 struct Engine {
+  uint32_t cc;
   uint16_t horsepower;
   uint8_t cylinders;
-  uint32_t cc;
-  bool usesGas;
-  bool usesElectric;
+  uint8_t bits;
+  inline bool usesGas()      const { return bits & 1; }
+  inline bool usesElectric() const { return bits & 2; }
+
+  inline void setBits(bool usesGas, bool usesElectric) {
+    bits = (uint8_t)usesGas | ((uint8_t)usesElectric << 1);
+  }
 };
 
 struct Car {
+  // SORT FIELDS BY SIZE since we need "theoretical best" memory usage
+  Engine engine;
+  List<Wheel> wheels;
   const char* make;
   const char* model;
-  Color color;
-  uint8_t seats;
-  uint8_t doors;
-  List<Wheel> wheels;
+  float fuelCapacity;
+  float fuelLevel;
+  uint32_t weight;
   uint16_t length;
   uint16_t width;
   uint16_t height;
-  uint32_t weight;
-  Engine engine;
-  float fuelCapacity;
-  float fuelLevel;
-  bool hasPowerWindows;
-  bool hasPowerSteering;
-  bool hasCruiseControl;
+  Color color;
+  uint8_t seats;
+  uint8_t doors;
   uint8_t cupHolders;
-  bool hasNavSystem;
+
+  uint8_t bits;
+
+  inline bool hasPowerWindows()  const { return bits & 1; }
+  inline bool hasPowerSteering() const { return bits & 2; }
+  inline bool hasCruiseControl() const { return bits & 4; }
+  inline bool hasNavSystem()     const { return bits & 8; }
+
+  inline void setBits(bool hasPowerWindows, bool hasPowerSteering,
+                      bool hasCruiseControl, bool hasNavSystem) {
+    bits = (uint8_t)hasPowerWindows
+         | ((uint8_t)hasPowerSteering << 1)
+         | ((uint8_t)hasCruiseControl << 2)
+         | ((uint8_t)hasNavSystem << 3);
+  }
 };
 
 
@@ -92,8 +109,8 @@ uint64_t carValue(const Car& car) {
 
   auto engine = car.engine;
   result += engine.horsepower * 40;
-  if (engine.usesElectric) {
-    if (engine.usesGas) {
+  if (engine.usesElectric()) {
+    if (engine.usesGas()) {
       // hybrid
       result += 5000;
     } else {
@@ -101,10 +118,10 @@ uint64_t carValue(const Car& car) {
     }
   }
 
-  result += car.hasPowerWindows ? 100 : 0;
-  result += car.hasPowerSteering ? 200 : 0;
-  result += car.hasCruiseControl ? 400 : 0;
-  result += car.hasNavSystem ? 2000 : 0;
+  result += car.hasPowerWindows() ? 100 : 0;
+  result += car.hasPowerSteering() ? 200 : 0;
+  result += car.hasCruiseControl() ? 400 : 0;
+  result += car.hasNavSystem() ? 2000 : 0;
 
   result += car.cupHolders * 25;
 
@@ -138,14 +155,16 @@ void randomCar(Car* car) {
   car->engine.horsepower = 100 * fastRand(400);
   car->engine.cylinders = 4 + 2 * fastRand(3);
   car->engine.cc = 800 + fastRand(10000);
+  car->engine.setBits(true, fastRand(2));
 
   car->fuelCapacity = 10.0 + fastRandDouble(30.0);
   car->fuelLevel = fastRandDouble(car->fuelCapacity);
-  car->hasPowerWindows = fastRand(2);
-  car->hasPowerSteering = fastRand(2);
-  car->hasCruiseControl = fastRand(2);
+  bool hasPowerWindows = fastRand(2);
+  bool hasPowerSteering = fastRand(2);
+  bool hasCruiseControl = fastRand(2);
   car->cupHolders = fastRand(12);
-  car->hasNavSystem = fastRand(2);
+  bool hasNavSystem = fastRand(2);
+  car->setBits(hasPowerWindows, hasPowerSteering, hasCruiseControl, hasNavSystem);
 }
 
 class CarSalesTestCase {

c++/src/capnproto/benchmark/null-common.h

@@ -65,24 +65,35 @@ struct List {
 // =======================================================================================
 
 struct SingleUseObjects {
-  template <typename ObjectType>
-  struct Object {
-    struct Reusable {};
-    struct SingleUse {
-      ObjectType value;
-      inline SingleUse(Reusable&) {}
-    };
+  class ObjectSizeCounter {
+  public:
+    ObjectSizeCounter(uint64_t iters): counter(0) {}
+
+    void add(uint64_t wordCount) {
+      counter += wordCount;
+    }
+
+    uint64_t get() { return counter; }
+
+  private:
+    uint64_t counter;
   };
 };
 
 struct ReusableObjects {
-  template <typename ObjectType>
-  struct Object {
-    typedef ObjectType Reusable;
-    struct SingleUse {
-      ObjectType& value;
-      inline SingleUse(Reusable& reusable): value(reusable) {}
-    };
+  class ObjectSizeCounter {
+  public:
+    ObjectSizeCounter(uint64_t iters): iters(iters), maxSize(0) {}
+
+    void add(size_t wordCount) {
+      maxSize = std::max(wordCount, maxSize);
+    }
+
+    uint64_t get() { return iters * maxSize; }
+
+  private:
+    uint64_t iters;
+    size_t maxSize;
   };
 };
 
@@ -120,7 +131,7 @@ struct BenchmarkMethods {
   }
 
   static uint64_t passByObject(uint64_t iters, bool countObjectSize) {
-    uint64_t throughput = 0;
+    typename ReuseStrategy::ObjectSizeCounter sizeCounter(iters);
 
     for (; iters > 0; --iters) {
       arenaPos = arena;
@@ -134,10 +145,10 @@ struct BenchmarkMethods {
         throw std::logic_error("Incorrect response.");
       }
 
-      throughput += (arenaPos - arena) * sizeof(arena[0]);
+      sizeCounter.add((arenaPos - arena) * sizeof(arena[0]));
     }
 
-    return throughput;
+    return sizeCounter.get();
   }
 
   static uint64_t passByBytes(uint64_t iters) {
@@ -149,10 +160,12 @@ struct BenchmarkMethods {
 struct BenchmarkTypes {
   typedef void Uncompressed;
   typedef void Packed;
+#if HAVE_SNAPPY
   typedef void SnappyCompressed;
+#endif  // HAVE_SNAPPY
 
-  typedef void ReusableResources;
-  typedef void SingleUseResources;
+  typedef ReusableObjects ReusableResources;
+  typedef SingleUseObjects SingleUseResources;
 
   template <typename TestCase, typename ReuseStrategy, typename Compression>
   struct BenchmarkMethods: public null::BenchmarkMethods<TestCase, ReuseStrategy, Compression> {};

c++/src/capnproto/benchmark/protobuf-carsales.c++

@@ -92,6 +92,8 @@ void randomCar(Car* car) {
   engine->set_horsepower(100 * fastRand(400));
   engine->set_cylinders(4 + 2 * fastRand(3));
   engine->set_cc(800 + fastRand(10000));
+  engine->set_uses_gas(true);
+  engine->set_uses_electric(fastRand(2));
 
   car->set_fuel_capacity(10.0 + fastRandDouble(30.0));
   car->set_fuel_level(fastRandDouble(car->fuel_capacity()));

c++/src/capnproto/benchmark/protobuf-common.h

@@ -24,8 +24,10 @@
 #include "common.h"
 #include <google/protobuf/io/zero_copy_stream_impl.h>
 #include <thread>
+#if HAVE_SNAPPY
 #include <snappy/snappy.h>
 #include <snappy/snappy-sinksource.h>
+#endif  // HAVE_SNAPPY
 
 namespace capnproto {
 namespace benchmark {
@@ -121,6 +123,8 @@ struct Uncompressed {
 // arrays in some static scratch space.  This probably gives protobufs an edge that it doesn't
 // deserve.
 
+#if HAVE_SNAPPY
+
 static char scratch[1 << 20];
 static char scratch2[1 << 20];
 
@@ -158,6 +162,8 @@ struct SnappyCompressed {
   static void flush(OutputStream*) {}
 };
 
+#endif  // HAVE_SNAPPY
+
 // =======================================================================================
 
 #define REUSABLE(type) \
@@ -337,7 +343,9 @@ struct BenchmarkMethods {
 struct BenchmarkTypes {
   typedef protobuf::Uncompressed Uncompressed;
   typedef protobuf::Uncompressed Packed;
+#if HAVE_SNAPPY
   typedef protobuf::SnappyCompressed SnappyCompressed;
+#endif  // HAVE_SNAPPY
 
   typedef protobuf::ReusableMessages ReusableResources;
   typedef protobuf::SingleUseMessages SingleUseResources;

c++/src/capnproto/descriptor-test.c++

-// Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-// 1. Redistributions of source code must retain the above copyright notice, this
-//    list of conditions and the following disclaimer.
-// 2. Redistributions in binary form must reproduce the above copyright notice,
-//    this list of conditions and the following disclaimer in the documentation
-//    and/or other materials provided with the distribution.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#include "descriptor.h"
-#include <gtest/gtest.h>
-
-namespace capnproto {
-namespace internal {
-namespace {
-
-const int READONLY_SEGMENT_START = 123;
-
-const FieldDescriptor TEST_FIELDS[2] = {
-    { 1*WORDS, 0*REFERENCES, FieldSize::FOUR_BYTES, 0*ELEMENTS, 0*BYTES, 1, 0, 0*BYTES, 0*BITS, nullptr, nullptr },
-    { 1*WORDS, 1*REFERENCES, FieldSize::REFERENCE , 0*ELEMENTS, 0*BYTES, 1, 0, 0*BYTES, 0*BITS, nullptr, nullptr }
-};
-
-extern const StructDescriptor TEST_STRUCT;
-
-const AlignedData<1> TEST_STRUCT_DEFAULT_VALUE = {
-    { 0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0 }
-};
-
-const StructDescriptor TEST_STRUCT = {
-    { Descriptor::Kind::STRUCT },
-    FieldNumber(2),
-    1 * WORDS,
-    1 * REFERENCES,
-    TEST_FIELDS,
-    TEST_STRUCT_DEFAULT_VALUE.words
-};
-
-const int READONLY_SEGMENT_END = 321;
-
-TEST(Descriptors, InReadOnlySegment) {
-  // It's extremely important that statically-initialized descriptors end up in the read-only
-  // segment, proving that they will not require any dynamic initialization at startup.  We hackily
-  // assume that variables will be placed in each segment in the order that they appear, therefore
-  // if our test declarations above do in fact land in the read-only segment, they should appear
-  // between READONLY_SEGMENT_START and READONLY_SEGMENT_END.
-
-  EXPECT_LE((const void*)&READONLY_SEGMENT_START, (const void*)&TEST_FIELDS);
-  EXPECT_GE((const void*)&READONLY_SEGMENT_END  , (const void*)&TEST_FIELDS);
-  EXPECT_LE((const void*)&READONLY_SEGMENT_START, (const void*)&TEST_STRUCT_DEFAULT_VALUE);
-  EXPECT_GE((const void*)&READONLY_SEGMENT_END  , (const void*)&TEST_STRUCT_DEFAULT_VALUE);
-  EXPECT_LE((const void*)&READONLY_SEGMENT_START, (const void*)&TEST_STRUCT);
-  EXPECT_GE((const void*)&READONLY_SEGMENT_END  , (const void*)&TEST_STRUCT);
-}
-
-}  // namespace
-}  // namespace internal
-}  // namespace capnproto

c++/src/capnproto/descriptor.c++

-// Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-// 1. Redistributions of source code must retain the above copyright notice, this
-//    list of conditions and the following disclaimer.
-// 2. Redistributions in binary form must reproduce the above copyright notice,
-//    this list of conditions and the following disclaimer in the documentation
-//    and/or other materials provided with the distribution.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#include "descriptor.h"
-
-namespace capnproto {
-namespace internal {
-
-}  // namespace internal
-}  // namespace capnproto

c++/src/capnproto/generated-header-support.h

@@ -29,6 +29,4 @@
 #include "wire-format.h"
 #include "list.h"
 
-#include "descriptor.h"    // TODO:  Eliminate this.
-
 #endif  // CAPNPROTO_GENERATED_HEADER_SUPPORT_H_

c++/src/capnproto/list.h

@@ -25,7 +25,6 @@
 #define CAPNPROTO_LIST_H_
 
 #include "wire-format.h"
-#include "descriptor.h"     // only for FieldSize; TODO:  Eliminate this
 #include <initializer_list>
 
 namespace capnproto {

c++/src/capnproto/wire-format-test.c++

@@ -22,7 +22,6 @@
 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "wire-format.h"
-#include "descriptor.h"
 #include "message.h"
 #include "arena.h"
 #include <gtest/gtest.h>

c++/src/capnproto/wire-format.c++

@@ -23,7 +23,6 @@
 
 #include "wire-format.h"
 #include "arena.h"
-#include "descriptor.h"
 #include <string.h>
 #include <limits>
 

c++/src/capnproto/wire-format.h

@@ -37,10 +37,6 @@
 namespace capnproto {
 namespace internal {
 
-class FieldDescriptor;
-typedef Id<uint8_t, FieldDescriptor> FieldNumber;
-enum class FieldSize: uint8_t;
-
 class StructBuilder;
 class StructReader;
 class ListBuilder;
@@ -50,6 +46,82 @@ struct WireHelpers;
 class SegmentReader;
 class SegmentBuilder;
 
+class FieldDescriptor;
+typedef Id<uint8_t, FieldDescriptor> FieldNumber;
+enum class FieldSize: uint8_t;
+
+enum class FieldSize: uint8_t {
+  // TODO:  Rename to FieldLayout or maybe ValueLayout.
+
+  VOID = 0,
+  BIT = 1,
+  BYTE = 2,
+  TWO_BYTES = 3,
+  FOUR_BYTES = 4,
+  EIGHT_BYTES = 5,
+
+  REFERENCE = 6,  // Indicates that the field lives in the reference segment, not the data segment.
+
+  INLINE_COMPOSITE = 7
+  // A composite type of fixed width.  This serves two purposes:
+  // 1) For lists of composite types where all the elements would have the exact same width,
+  //    allocating a list of references which in turn point at the elements would waste space.  We
+  //    can avoid a layer of indirection by placing all the elements in a flat sequence, and only
+  //    indicating the element properties (e.g. field count for structs) once.
+  //
+  //    Specifically, a list reference indicating INLINE_COMPOSITE element size actually points to
+  //    a "tag" describing one element.  This tag is formatted like a wire reference, but the
+  //    "offset" instead stores the element count of the list.  The flat list of elements appears
+  //    immediately after the tag.  In the list reference itself, the element count is replaced with
+  //    a word count for the whole list (excluding tag).  This allows the tag and elements to be
+  //    precached in a single step rather than two sequential steps.
+  //
+  //    It is NOT intended to be possible to substitute an INLINE_COMPOSITE list for a REFERENCE
+  //    list or vice-versa without breaking recipients.  Recipients expect one or the other
+  //    depending on the message definition.
+  //
+  //    However, it IS allowed to substitute an INLINE_COMPOSITE list -- specifically, of structs --
+  //    when a list was expected, or vice versa, with the assumption that the first field of the
+  //    struct (field number zero) correspond to the element type.  This allows a list of
+  //    primitives to be upgraded to a list of structs, avoiding the need to use parallel arrays
+  //    when you realize that you need to attach some extra information to each element of some
+  //    primitive list.
+  //
+  // 2) For struct fields of composite types where the field's total size is known at compile time,
+  //    we can embed the field directly into the parent struct to avoid indirection through a
+  //    reference.  However, this means that the field size can never change -- e.g. if it is a
+  //    struct, new fields cannot be added to it.  It's unclear if this is really useful so at this
+  //    time it is not supported.
+};
+
+typedef decltype(BITS / ELEMENTS) BitsPerElement;
+
+namespace internal {
+  static constexpr BitsPerElement BITS_PER_ELEMENT_TABLE[8] = {
+      0 * BITS / ELEMENTS,
+      1 * BITS / ELEMENTS,
+      8 * BITS / ELEMENTS,
+      16 * BITS / ELEMENTS,
+      32 * BITS / ELEMENTS,
+      64 * BITS / ELEMENTS,
+      64 * BITS / ELEMENTS,
+      0 * BITS / ELEMENTS
+  };
+}
+
+inline constexpr BitsPerElement bitsPerElement(FieldSize size) {
+  return internal::BITS_PER_ELEMENT_TABLE[static_cast<int>(size)];
+}
+
+template <int wordCount>
+union AlignedData {
+  // Useful for declaring static constant data blobs as an array of bytes, but forcing those
+  // bytes to be word-aligned.
+
+  uint8_t bytes[wordCount * sizeof(word)];
+  word words[wordCount];
+};
+
 // -------------------------------------------------------------------
 
 template <typename T>