Merge pull request #707 from capnproto/preallocated-str

Add functions for async-signal-safe stringification

c++/src/kj/exception.c++

@@ -336,6 +336,42 @@ String stringifyStackTraceAddresses(ArrayPtr<void* const> trace) {
 #endif
 }
 
+StringPtr stringifyStackTraceAddresses(ArrayPtr<void* const> trace, ArrayPtr<char> scratch) {
+  // Version which writes into a pre-allocated buffer. This is safe for signal handlers to the
+  // extent that dladdr() is safe.
+  //
+  // TODO(cleanup): We should improve the KJ stringification framework so that there's a way to
+  //   write this string directly into a larger message buffer with strPreallocated().
+
+#if KJ_HAS_LIBDL
+  char* ptr = scratch.begin();
+  char* limit = scratch.end() - 1;
+
+  for (auto addr: trace) {
+    Dl_info info;
+    // Shared libraries are mapped near the end of the address space while the executable is mapped
+    // near the beginning. We want to print addresses in the executable as raw addresses, not
+    // offsets, since that's what addr2line expects for executables. For shared libraries it
+    // expects offsets. In any case, most frames are likely to be in the main executable so it
+    // makes the output cleaner if we don't repeatedly write its name.
+    if (reinterpret_cast<uintptr_t>(addr) >= 0x400000000000ull && dladdr(addr, &info)) {
+      uintptr_t offset = reinterpret_cast<uintptr_t>(addr) -
+                         reinterpret_cast<uintptr_t>(info.dli_fbase);
+      ptr = _::fillLimited(ptr, limit, kj::StringPtr(info.dli_fname), "@0x"_kj, hex(offset));
+    } else {
+      ptr = _::fillLimited(ptr, limit, toCharSequence(addr));
+    }
+
+    ptr = _::fillLimited(ptr, limit, " "_kj);
+  }
+  *ptr = '\0';
+  return StringPtr(scratch.begin(), ptr);
+#else
+  // TODO(someday): Support other platforms.
+  return kj::strPreallocated(scratch, kj::delimited(trace, " "));
+#endif
+}
+
 String getStackTrace() {
   void* space[32];
   auto trace = getStackTrace(space, 2);

c++/src/kj/exception.h

@@ -354,6 +354,7 @@ String stringifyStackTrace(ArrayPtr<void* const>);
 // suprocesses.
 
 String stringifyStackTraceAddresses(ArrayPtr<void* const> trace);
+StringPtr stringifyStackTraceAddresses(ArrayPtr<void* const> trace, ArrayPtr<char> scratch);
 // Construct a string containing just enough information about a stack trace to be able to convert
 // it to file and line numbers later using offline tools. This produces a sequence of
 // space-separated code location identifiers. Each identifier may be an absolute address

c++/src/kj/string-test.c++

@@ -35,6 +35,12 @@ TEST(String, Str) {
   EXPECT_EQ("foo", str('f', 'o', 'o'));
   EXPECT_EQ("123 234 -123 e7",
             str((int8_t)123, " ", (uint8_t)234, " ", (int8_t)-123, " ", hex((uint8_t)0xe7)));
+  EXPECT_EQ("-128 -32768 -2147483648 -9223372036854775808",
+      str((signed char)-128, ' ', (signed short)-32768, ' ',
+          ((int)-2147483647) - 1, ' ', ((long long)-9223372036854775807ll) - 1))
+  EXPECT_EQ("ff ffff ffffffff ffffffffffffffff",
+      str(hex((uint8_t)0xff), ' ', hex((uint16_t)0xffff), ' ', hex((uint32_t)0xffffffffu), ' ',
+          hex((uint64_t)0xffffffffffffffffull)));
 
   char buf[3] = {'f', 'o', 'o'};
   ArrayPtr<char> a = buf;
@@ -191,6 +197,25 @@ KJ_TEST("string literals with _kj suffix") {
   KJ_EXPECT(kj::str(ARR) == "foo");
 }
 
+KJ_TEST("kj::delimited() and kj::strPreallocated()") {
+  int rawArray[] = {1, 23, 456, 78};
+  ArrayPtr<int> array = rawArray;
+  KJ_EXPECT(str(delimited(array, "::")) == "1::23::456::78");
+
+  {
+    char buffer[256];
+    KJ_EXPECT(strPreallocated(buffer, delimited(array, "::"), 'x')
+        == "1::23::456::78x");
+    KJ_EXPECT(strPreallocated(buffer, "foo", 123, true) == "foo123true");
+  }
+
+  {
+    char buffer[5];
+    KJ_EXPECT(strPreallocated(buffer, delimited(array, "::"), 'x') == "1::2");
+    KJ_EXPECT(strPreallocated(buffer, "foo", 123, true) == "foo1");
+  }
+}
+
 }  // namespace
 }  // namespace _ (private)
 }  // namespace kj

c++/src/kj/string-tree.c++

@@ -53,11 +53,21 @@ String StringTree::flatten() const {
   return result;
 }
 
-void StringTree::flattenTo(char* __restrict__ target) const {
+char* StringTree::flattenTo(char* __restrict__ target) const {
   visit([&target](ArrayPtr<const char> text) {
     memcpy(target, text.begin(), text.size());
     target += text.size();
   });
+  return target;
+}
+
+char* StringTree::flattenTo(char* __restrict__ target, char* limit) const {
+  visit([&target,limit](ArrayPtr<const char> text) {
+    size_t size = kj::min(text.size(), limit - target);
+    memcpy(target, text.begin(), size);
+    target += size;
+  });
+  return target;
 }
 
 }  // namespace kj

c++/src/kj/string-tree.h

@@ -62,8 +62,10 @@ public:
   // TODO(someday):  flatten() when *this is an rvalue and when branches.size() == 0 could simply
   //   return `kj::mv(text)`.  Requires reference qualifiers (Clang 3.3 / GCC 4.8).
 
-  void flattenTo(char* __restrict__ target) const;
-  // Copy the contents to the given character array.  Does not add a NUL terminator.
+  char* flattenTo(char* __restrict__ target) const;
+  char* flattenTo(char* __restrict__ target, char* limit) const;
+  // Copy the contents to the given character array.  Does not add a NUL terminator. Returns a
+  // pointer just past the end of what was filled.
 
 private:
   size_t size_;
@@ -124,6 +126,14 @@ char* fill(char* __restrict__ target, const StringTree& first, Rest&&... rest) {
   return fill(target + first.size(), kj::fwd<Rest>(rest)...);
 }
 
+template <typename... Rest>
+char* fillLimited(char* __restrict__ target, char* limit, const StringTree& first, Rest&&... rest) {
+  // Make str() work with stringifiers that return StringTree by patching fill().
+
+  target = first.flattenTo(target, limit);
+  return fillLimited(target + first.size(), limit, kj::fwd<Rest>(rest)...);
+}
+
 template <typename T> constexpr bool isStringTree() { return false; }
 template <> constexpr bool isStringTree<StringTree>() { return true; }
 

c++/src/kj/string.c++

@@ -25,6 +25,7 @@
 #include <float.h>
 #include <errno.h>
 #include <stdlib.h>
+#include <stdint.h>
 
 namespace kj {
 
@@ -118,18 +119,39 @@ String heapString(const char* value, size_t size) {
   return String(buffer, size, _::HeapArrayDisposer::instance);
 }
 
-#define HEXIFY_INT(type, format) \
+template <typename T>
+static CappedArray<char, sizeof(T) * 2 + 1> hexImpl(T i) {
+  // We don't use sprintf() because it's not async-signal-safe (for strPreallocated()).
+  CappedArray<char, sizeof(T) * 2 + 1> result;
+  uint8_t reverse[sizeof(T) * 2];
+  uint8_t* p = reverse;
+  if (i == 0) {
+    *p++ = 0;
+  } else {
+    while (i > 0) {
+      *p++ = i % 16;
+      i /= 16;
+    }
+  }
+
+  char* p2 = result.begin();
+  while (p > reverse) {
+    *p2++ = "0123456789abcdef"[*--p];
+  }
+  result.setSize(p2 - result.begin());
+  return result;
+}
+
+#define HEXIFY_INT(type) \
 CappedArray<char, sizeof(type) * 2 + 1> hex(type i) { \
-  CappedArray<char, sizeof(type) * 2 + 1> result; \
-  result.setSize(sprintf(result.begin(), format, i)); \
-  return result; \
+  return hexImpl<type>(i); \
 }
 
-HEXIFY_INT(unsigned char, "%x");
-HEXIFY_INT(unsigned short, "%x");
-HEXIFY_INT(unsigned int, "%x");
-HEXIFY_INT(unsigned long, "%lx");
-HEXIFY_INT(unsigned long long, "%llx");
+HEXIFY_INT(unsigned char);
+HEXIFY_INT(unsigned short);
+HEXIFY_INT(unsigned int);
+HEXIFY_INT(unsigned long);
+HEXIFY_INT(unsigned long long);
 
 #undef HEXIFY_INT
 
@@ -143,27 +165,54 @@ StringPtr Stringifier::operator*(bool b) const {
   return b ? StringPtr("true") : StringPtr("false");
 }
 
-#define STRINGIFY_INT(type, format) \
+template <typename T, typename Unsigned>
+static CappedArray<char, sizeof(T) * 3 + 2> stringifyImpl(T i) {
+  // We don't use sprintf() because it's not async-signal-safe (for strPreallocated()).
+  CappedArray<char, sizeof(T) * 3 + 2> result;
+  bool negative = i < 0;
+  Unsigned u = negative ? -i : i;
+  uint8_t reverse[sizeof(T) * 3 + 1];
+  uint8_t* p = reverse;
+  if (u == 0) {
+    *p++ = 0;
+  } else {
+    while (u > 0) {
+      *p++ = u % 10;
+      u /= 10;
+    }
+  }
+
+  char* p2 = result.begin();
+  if (negative) *p2++ = '-';
+  while (p > reverse) {
+    *p2++ = '0' + *--p;
+  }
+  result.setSize(p2 - result.begin());
+  return result;
+}
+
+#define STRINGIFY_INT(type, unsigned) \
 CappedArray<char, sizeof(type) * 3 + 2> Stringifier::operator*(type i) const { \
-  CappedArray<char, sizeof(type) * 3 + 2> result; \
-  result.setSize(sprintf(result.begin(), format, i)); \
-  return result; \
+  return stringifyImpl<type, unsigned>(i); \
 }
 
-STRINGIFY_INT(signed char, "%d");
-STRINGIFY_INT(unsigned char, "%u");
-STRINGIFY_INT(short, "%d");
-STRINGIFY_INT(unsigned short, "%u");
-STRINGIFY_INT(int, "%d");
-STRINGIFY_INT(unsigned int, "%u");
-STRINGIFY_INT(long, "%ld");
-STRINGIFY_INT(unsigned long, "%lu");
-STRINGIFY_INT(long long, "%lld");
-STRINGIFY_INT(unsigned long long, "%llu");
-STRINGIFY_INT(const void*, "%p");
+STRINGIFY_INT(signed char, uint);
+STRINGIFY_INT(unsigned char, uint);
+STRINGIFY_INT(short, uint);
+STRINGIFY_INT(unsigned short, uint);
+STRINGIFY_INT(int, uint);
+STRINGIFY_INT(unsigned int, uint);
+STRINGIFY_INT(long, unsigned long);
+STRINGIFY_INT(unsigned long, unsigned long);
+STRINGIFY_INT(long long, unsigned long long);
+STRINGIFY_INT(unsigned long long, unsigned long long);
 
 #undef STRINGIFY_INT
 
+CappedArray<char, sizeof(const void*) * 2 + 1> Stringifier::operator*(const void* i) const { \
+  return hexImpl<uintptr_t>(reinterpret_cast<uintptr_t>(i));
+}
+
 namespace {
 
 // ----------------------------------------------------------------------

c++/src/kj/string.h

@@ -264,9 +264,12 @@ inline size_t sum(std::initializer_list<size_t> nums) {
 }
 
 inline char* fill(char* ptr) { return ptr; }
+inline char* fillLimited(char* ptr, char* limit) { return ptr; }
 
 template <typename... Rest>
 char* fill(char* __restrict__ target, const StringTree& first, Rest&&... rest);
+template <typename... Rest>
+char* fillLimited(char* __restrict__ target, char* limit, const StringTree& first, Rest&&... rest);
 // Make str() work with stringifiers that return StringTree by patching fill().
 //
 // Defined in string-tree.h.
@@ -295,6 +298,27 @@ inline String concat(String&& arr) {
   return kj::mv(arr);
 }
 
+template <typename First, typename... Rest>
+char* fillLimited(char* __restrict__ target, char* limit, const First& first, Rest&&... rest) {
+  auto i = first.begin();
+  auto end = first.end();
+  while (i != end) {
+    if (target == limit) return target;
+    *target++ = *i++;
+  }
+  return fillLimited(target, limit, kj::fwd<Rest>(rest)...);
+}
+
+template <typename T>
+class Delimited;
+// Delimits a sequence of type T with a string delimiter. Implements kj::delimited().
+
+template <typename T, typename... Rest>
+char* fill(char* __restrict__ target, Delimited<T> first, Rest&&... rest);
+template <typename T, typename... Rest>
+char* fillLimited(char* __restrict__ target, char* limit, Delimited<T> first,Rest&&... rest);
+// As with StringTree, we special-case Delimited<T>.
+
 struct Stringifier {
   // This is a dummy type with only one instance: STR (below).  To make an arbitrary type
   // stringifiable, define `operator*(Stringifier, T)` to return an iterable container of `char`.
@@ -344,12 +368,12 @@ struct Stringifier {
   CappedArray<char, sizeof(unsigned long long) * 3 + 2> operator*(unsigned long long i) const;
   CappedArray<char, 24> operator*(float f) const;
   CappedArray<char, 32> operator*(double f) const;
-  CappedArray<char, sizeof(const void*) * 3 + 2> operator*(const void* s) const;
+  CappedArray<char, sizeof(const void*) * 2 + 1> operator*(const void* s) const;
 
   template <typename T>
-  String operator*(ArrayPtr<T> arr) const;
+  _::Delimited<ArrayPtr<T>> operator*(ArrayPtr<T> arr) const;
   template <typename T>
-  String operator*(const Array<T>& arr) const;
+  _::Delimited<ArrayPtr<const T>> operator*(const Array<T>& arr) const;
 
 #if KJ_COMPILER_SUPPORTS_STL_STRING_INTEROP  // supports expression SFINAE?
   template <typename T, typename Result = decltype(instance<T>().toString())>
@@ -393,6 +417,10 @@ String str(Params&&... params) {
 inline String str(String&& s) { return mv(s); }
 // Overload to prevent redundant allocation.
 
+template <typename T>
+_::Delimited<T> delimited(T&& arr, kj::StringPtr delim);
+// Use to stringify an array.
+
 template <typename T>
 String strArray(T&& arr, const char* delim) {
   size_t delimLen = strlen(delim);
@@ -416,16 +444,40 @@ String strArray(T&& arr, const char* delim) {
   return result;
 }
 
+template <typename... Params>
+StringPtr strPreallocated(ArrayPtr<char> buffer, Params&&... params) {
+  // Like str() but writes into a preallocated buffer. If the buffer is not long enough, the result
+  // is truncated (but still NUL-terminated).
+  //
+  // This can be used like:
+  //
+  //     char buffer[256];
+  //     StringPtr text = strPreallocated(buffer, params...);
+  //
+  // This is useful for optimization. It can also potentially be used safely in async signal
+  // handlers. HOWEVER, to use in an async signal handler, all of the stringifiers for the inputs
+  // must also be signal-safe. KJ guarantees signal safety when stringifying any built-in integer
+  // type (but NOT floating-points), basic char/byte sequences (ArrayPtr<byte>, String, etc.), as
+  // well as Array<T> as long as T can also be stringified safely. To safely stringify a delimited
+  // array, you must use kj::delimited(arr, delim) rather than the deprecated
+  // kj::strArray(arr, delim).
+
+  char* end = _::fillLimited(buffer.begin(), buffer.end() - 1,
+      toCharSequence(kj::fwd<Params>(params))...);
+  *end = '\0';
+  return StringPtr(buffer.begin(), end);
+}
+
 namespace _ {  // private
 
 template <typename T>
-inline String Stringifier::operator*(ArrayPtr<T> arr) const {
-  return strArray(arr, ", ");
+inline _::Delimited<ArrayPtr<T>> Stringifier::operator*(ArrayPtr<T> arr) const {
+  return _::Delimited<ArrayPtr<T>>(arr, ", ");
 }
 
 template <typename T>
-inline String Stringifier::operator*(const Array<T>& arr) const {
-  return strArray(arr, ", ");
+inline _::Delimited<ArrayPtr<const T>> Stringifier::operator*(const Array<T>& arr) const {
+  return _::Delimited<ArrayPtr<const T>>(arr, ", ");
 }
 
 }  // namespace _ (private)
@@ -548,6 +600,101 @@ inline String heapString(ArrayPtr<const char> value) {
   return heapString(value.begin(), value.size());
 }
 
+namespace _ {  // private
+
+template <typename T>
+class Delimited {
+public:
+  Delimited(T array, kj::StringPtr delimiter)
+      : array(kj::fwd<T>(array)), delimiter(delimiter) {}
+
+  // TODO(someday): In theory we should support iteration as a character sequence, but the iterator
+  //   will be pretty complicated.
+
+  size_t size() {
+    ensureStringifiedInitialized();
+
+    size_t result = 0;
+    bool first = true;
+    for (auto& e: stringified) {
+      if (first) {
+        first = false;
+      } else {
+        result += delimiter.size();
+      }
+      result += e.size();
+    }
+    return result;
+  }
+
+  char* flattenTo(char* __restrict__ target) {
+    ensureStringifiedInitialized();
+
+    bool first = true;
+    for (auto& elem: stringified) {
+      if (first) {
+        first = false;
+      } else {
+        target = fill(target, delimiter);
+      }
+      target = fill(target, elem);
+    }
+    return target;
+  }
+
+  char* flattenTo(char* __restrict__ target, char* limit) {
+    // This is called in the strPreallocated(). We want to avoid allocation. size() will not have
+    // been called in this case, so hopefully `stringified` is still uninitialized. We will
+    // stringify each item and immediately use it.
+    bool first = true;
+    for (auto&& elem: array) {
+      if (target == limit) return target;
+      if (first) {
+        first = false;
+      } else {
+        target = fillLimited(target, limit, delimiter);
+      }
+      target = fillLimited(target, limit, kj::toCharSequence(elem));
+    }
+    return target;
+  }
+
+private:
+  typedef decltype(toCharSequence(*instance<T>().begin())) StringifiedItem;
+  T array;
+  kj::StringPtr delimiter;
+  Array<StringifiedItem> stringified;
+
+  void ensureStringifiedInitialized() {
+    if (array.size() > 0 && stringified.size() == 0) {
+      stringified = KJ_MAP(e, array) { return toCharSequence(e); };
+    }
+  }
+};
+
+template <typename T, typename... Rest>
+char* fill(char* __restrict__ target, Delimited<T> first, Rest&&... rest) {
+  target = first.flattenTo(target);
+  return fill(target, kj::fwd<Rest>(rest)...);
+}
+template <typename T, typename... Rest>
+char* fillLimited(char* __restrict__ target, char* limit, Delimited<T> first, Rest&&... rest) {
+  target = first.flattenTo(target, limit);
+  return fillLimited(target, limit, kj::fwd<Rest>(rest)...);
+}
+
+template <typename T>
+inline Delimited<T>&& KJ_STRINGIFY(Delimited<T>&& delimited) { return kj::mv(delimited); }
+template <typename T>
+inline const Delimited<T>& KJ_STRINGIFY(const Delimited<T>& delimited) { return delimited; }
+
+}  // namespace _ (private)
+
+template <typename T>
+_::Delimited<T> delimited(T&& arr, kj::StringPtr delim) {
+  return _::Delimited<T>(kj::fwd<T>(arr), delim);
+}
+
 }  // namespace kj
 
 constexpr kj::StringPtr operator "" _kj(const char* str, size_t n) {