Skip to content

C
capnproto
Commit f8b5d71d authored by Kenton Varda's avatar Kenton Varda
Browse files
Options

Fancy benchmark harness.

parent 50c229bb
Show whitespace changes
Inline Side-by-side
Showing with 1010 additions and 24 deletions
c++/src/capnproto/benchmark/benchmark-capnproto.c++
View file @ f8b5d71d
......@@ -122,7 +122,7 @@ int32_t makeExpression(Expression::Builder exp, int depth) {
// TODO: Operation_MAX or something.
exp.setOp((Operation)(rand() % (int)Operation::MODULUS + 1));
int left, right;
uint32_t left, right;
if (rand() % 8 < depth) {
exp.setLeftIsValue(true);
......@@ -156,7 +156,7 @@ int32_t makeExpression(Expression::Builder exp, int depth) {
}
int32_t evaluateExpression(Expression::Reader exp) {
int left, right;
int32_t left, right;
if (exp.getLeftIsValue()) {
left = exp.getLeftValue();
......@@ -379,24 +379,79 @@ struct NoScratch {
public:
inline MessageBuilder(ScratchSpace& scratch): MallocMessageBuilder() {}
};
class ObjectSizeCounter {
public:
ObjectSizeCounter(uint64_t iters): counter(0) {}
template <typename RequestBuilder, typename ResponseBuilder>
void add(RequestBuilder& request, ResponseBuilder& response) {
for (auto segment: request.getSegmentsForOutput()) {
counter += segment.size() * sizeof(word);
}
for (auto segment: response.getSegmentsForOutput()) {
counter += segment.size() * sizeof(word);
}
}
uint64_t get() { return counter; }
private:
uint64_t counter;
};
};
template <typename Compression, size_t size>
constexpr size_t SCRATCH_SIZE = 128 * 1024;
word scratchSpace[4 * SCRATCH_SIZE];
int scratchCounter = 0;
template <typename Compression>
struct UseScratch {
struct ScratchSpace {
word words[size];
word* words;
ScratchSpace() {
CAPNPROTO_ASSERT(scratchCounter < 4, "Too many scratch spaces needed at once.");
words = scratchSpace + scratchCounter++ * SCRATCH_SIZE;
}
~ScratchSpace() {
--scratchCounter;
}
};
class MessageReader: public Compression::MessageReader {
public:
inline MessageReader(int fd, ScratchSpace& scratch)
: Compression::MessageReader(fd, ReaderOptions(), arrayPtr(scratch.words, size)) {}
: Compression::MessageReader(fd, ReaderOptions(), arrayPtr(scratch.words, SCRATCH_SIZE)) {}
};
class MessageBuilder: public MallocMessageBuilder {
public:
inline MessageBuilder(ScratchSpace& scratch)
: MallocMessageBuilder(arrayPtr(scratch.words, size)) {}
: MallocMessageBuilder(arrayPtr(scratch.words, SCRATCH_SIZE)) {}
};
class ObjectSizeCounter {
public:
ObjectSizeCounter(uint64_t iters): iters(iters), maxSize(0) {}
template <typename RequestBuilder, typename ResponseBuilder>
void add(RequestBuilder& request, ResponseBuilder& response) {
size_t counter = 0;
for (auto segment: request.getSegmentsForOutput()) {
counter += segment.size() * sizeof(word);
}
for (auto segment: response.getSegmentsForOutput()) {
counter += segment.size() * sizeof(word);
}
maxSize = std::max(counter, maxSize);
}
uint64_t get() { return iters * maxSize; }
private:
uint64_t iters;
size_t maxSize;
};
};
......@@ -490,10 +545,12 @@ uint64_t server(int inputFd, int outputFd, uint64_t iters) {
}
template <typename TestCase, typename ReuseStrategy, typename Compression>
uint64_t passByObject(uint64_t iters) {
uint64_t passByObject(uint64_t iters, bool countObjectSize) {
typename ReuseStrategy::ScratchSpace requestScratch;
typename ReuseStrategy::ScratchSpace responseScratch;
typename ReuseStrategy::ObjectSizeCounter counter(iters);
for (; iters > 0; --iters) {
typename ReuseStrategy::MessageBuilder requestMessage(requestScratch);
auto request = requestMessage.template initRoot<typename TestCase::Request>();
......@@ -506,9 +563,13 @@ uint64_t passByObject(uint64_t iters) {
if (!TestCase::checkResponse(response.asReader(), expected)) {
throw std::logic_error("Incorrect response.");
}
if (countObjectSize) {
counter.add(requestMessage, responseMessage);
}
}
return 0;
return counter.get();
}
template <typename TestCase, typename ReuseStrategy, typename Compression>
......@@ -592,7 +653,9 @@ uint64_t doBenchmark(const std::string& mode, uint64_t iters) {
return server<TestCase, ReuseStrategy, Compression>(
STDIN_FILENO, STDOUT_FILENO, iters);
} else if (mode == "object") {
return passByObject<TestCase, ReuseStrategy, Compression>(iters);
return passByObject<TestCase, ReuseStrategy, Compression>(iters, false);
} else if (mode == "object-size") {
return passByObject<TestCase, ReuseStrategy, Compression>(iters, true);
} else if (mode == "bytes") {
return passByBytes<TestCase, ReuseStrategy, Compression>(iters);
} else if (mode == "pipe") {
......@@ -610,7 +673,7 @@ uint64_t doBenchmark(const std::string& mode, uint64_t iters) {
template <typename TestCase, typename Compression>
uint64_t doBenchmark2(const std::string& mode, const std::string& reuse, uint64_t iters) {
if (reuse == "reuse") {
return doBenchmark<TestCase, UseScratch<Compression, 1024>, Compression>(mode, iters);
return doBenchmark<TestCase, UseScratch<Compression>, Compression>(mode, iters);
} else if (reuse == "no-reuse") {
return doBenchmark<TestCase, NoScratch<Compression>, Compression>(mode, iters);
} else {
......@@ -634,15 +697,14 @@ uint64_t doBenchmark3(const std::string& mode, const std::string& reuse,
int main(int argc, char* argv[]) {
if (argc != 6) {
std::cerr << "USAGE: " << argv[0] << " MODE REUSE COMPRESSION ITERATION_COUNT" << std::endl;
std::cerr << "USAGE: " << argv[0]
<< " TEST_CASE MODE REUSE COMPRESSION ITERATION_COUNT" << std::endl;
return 1;
}
uint64_t iters = strtoull(argv[5], nullptr, 0);
srand(123);
std::cerr << "Doing " << iters << " iterations..." << std::endl;
uint64_t throughput;
std::string testcase = argv[1];
......@@ -655,7 +717,7 @@ int main(int argc, char* argv[]) {
return 1;
}
std::cerr << "Average messages size = " << (throughput / iters) << std::endl;
std::cout << throughput << std::endl;
return 0;
}
......
c++/src/capnproto/benchmark/benchmark-null.c++ 0 → 100644
View file @ f8b5d71d
// Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <inttypes.h>
#include <iostream>
#include <string>
#include <stddef.h>
#include <limits.h>
#include <memory>
#include <stdexcept>
#include <algorithm>
#include <string.h>
namespace capnproto {
namespace benchmark {
namespace null {
uint64_t arena[1024*1024];
uint64_t* arenaPos = arena;
template <typename T>
T* allocate(int count = 1) {
T* result = reinterpret_cast<T*>(arenaPos);
arenaPos += (sizeof(T) * count + 7) / 8;
if (arenaPos > arena + sizeof(arena) / sizeof(arena[0])) {
throw std::bad_alloc();
}
return result;
}
char* copyString(const char* str) {
size_t len = strlen(str);
char* result = allocate<char>(len);
memcpy(result, str, len + 1);
return result;
}
// =======================================================================================
enum class Operation {
ADD,
SUBTRACT,
MULTIPLY,
DIVIDE,
MODULUS
};
uint Operation_MAX = static_cast<uint>(Operation::MODULUS) + 1;
struct Expression {
Operation op;
bool leftIsValue;
bool rightIsValue;
union {
int32_t leftValue;
Expression* leftExpression;
};
union {
int32_t rightValue;
Expression* rightExpression;
};
};
inline int32_t div(int32_t a, int32_t b) {
if (b == 0) return INT_MAX;
// INT_MIN / -1 => SIGFPE. Who knew?
if (a == INT_MIN && b == -1) return INT_MAX;
return a / b;
}
inline int32_t mod(int32_t a, int32_t b) {
if (b == 0) return INT_MAX;
// INT_MIN % -1 => SIGFPE. Who knew?
if (a == INT_MIN && b == -1) return INT_MAX;
return a % b;
}
int32_t makeExpression(Expression* exp, int depth) {
exp->op = (Operation)(rand() % Operation_MAX);
int32_t left, right;
if (rand() % 8 < depth) {
exp->leftIsValue = true;
left = rand() % 128 + 1;
exp->leftValue = left;
} else {
exp->leftIsValue = false;
exp->leftExpression = allocate<Expression>();
left = makeExpression(exp->leftExpression, depth + 1);
}
if (rand() % 8 < depth) {
exp->rightIsValue = true;
right = rand() % 128 + 1;
exp->rightValue = right;
} else {
exp->rightIsValue = false;
exp->rightExpression = allocate<Expression>();
right = makeExpression(exp->rightExpression, depth + 1);
}
switch (exp->op) {
case Operation::ADD:
return left + right;
case Operation::SUBTRACT:
return left - right;
case Operation::MULTIPLY:
return left * right;
case Operation::DIVIDE:
return div(left, right);
case Operation::MODULUS:
return mod(left, right);
}
throw std::logic_error("Can't get here.");
}
int32_t evaluateExpression(const Expression& exp) {
uint32_t left, right;
if (exp.leftIsValue) {
left = exp.leftValue;
} else {
left = evaluateExpression(*exp.leftExpression);
}
if (exp.rightIsValue) {
right = exp.rightValue;
} else {
right = evaluateExpression(*exp.rightExpression);
}
switch (exp.op) {
case Operation::ADD:
return left + right;
case Operation::SUBTRACT:
return left - right;
case Operation::MULTIPLY:
return left * right;
case Operation::DIVIDE:
return div(left, right);
case Operation::MODULUS:
return mod(left, right);
}
throw std::logic_error("Can't get here.");
}
class ExpressionTestCase {
public:
typedef Expression Request;
typedef int32_t Response;
typedef int32_t Expectation;
static inline int32_t setupRequest(Expression* request) {
return makeExpression(request, 0);
}
static inline void handleRequest(const Expression& request, int32_t* response) {
*response = evaluateExpression(request);
}
static inline bool checkResponse(int32_t response, int32_t expected) {
return response == expected;
}
static size_t spaceUsed(const Expression& expression) {
return sizeof(Expression) +
(expression.leftExpression == nullptr ? 0 : spaceUsed(*expression.leftExpression)) +
(expression.rightExpression == nullptr ? 0 : spaceUsed(*expression.rightExpression));
}
};
// =======================================================================================
// Test case: Cat Rank
//
// The server receives a list of candidate search results with scores. It promotes the ones that
// mention "cat" in their snippet and demotes the ones that mention "dog", sorts the results by
// descending score, and returns.
//
// The promotion multiplier is large enough that all the results mentioning "cat" but not "dog"
// should end up at the front ofthe list, which is how we verify the result.
static const char* WORDS[] = {
"foo ", "bar ", "baz ", "qux ", "quux ", "corge ", "grault ", "garply ", "waldo ", "fred ",
"plugh ", "xyzzy ", "thud "
};
constexpr size_t WORDS_COUNT = sizeof(WORDS) / sizeof(WORDS[0]);
template <typename T>
struct List {
size_t size;
T* items;
inline T* begin() const { return items; }
inline T* end() const { return items + size; }
};
struct SearchResult {
const char* url;
double score;
const char* snippet;
};
struct ScoredResult {
double score;
const SearchResult* result;
ScoredResult() = default;
ScoredResult(double score, const SearchResult* result): score(score), result(result) {}
inline bool operator<(const ScoredResult& other) const { return score > other.score; }
};
class CatRankTestCase {
public:
typedef List<SearchResult> Request;
typedef List<SearchResult> Response;
typedef int Expectation;
static int setupRequest(List<SearchResult>* request) {
int count = rand() % 1000;
int goodCount = 0;
request->size = count;
request->items = allocate<SearchResult>(count);
for (int i = 0; i < count; i++) {
SearchResult& result = request->items[i];
result.score = 1000 - i;
char* pos = reinterpret_cast<char*>(arenaPos);
result.url = pos;
strcpy(pos, "http://example.com/");
pos += strlen("http://example.com/");
int urlSize = rand() % 100;
for (int j = 0; j < urlSize; j++) {
*pos++ = 'a' + rand() % 26;
}
*pos++ = '\0';
// Retroactively allocate the space we used.
if (allocate<char>(pos - result.url) != result.url) {
throw std::bad_alloc();
}
bool isCat = rand() % 8 == 0;
bool isDog = rand() % 8 == 0;
goodCount += isCat && !isDog;
pos = reinterpret_cast<char*>(arenaPos);
result.snippet = pos;
*pos++ = ' ';
int prefix = rand() % 20;
for (int j = 0; j < prefix; j++) {
const char* word = WORDS[rand() % WORDS_COUNT];
size_t len = strlen(word);
memcpy(pos, word, len);
pos += len;
}
if (isCat) {
strcpy(pos, "cat ");
pos += 4;
}
if (isDog) {
strcpy(pos, "dog ");
pos += 4;
}
int suffix = rand() % 20;
for (int j = 0; j < suffix; j++) {
const char* word = WORDS[rand() % WORDS_COUNT];
size_t len = strlen(word);
memcpy(pos, word, len);
pos += len;
}
*pos++ = '\0';
// Retroactively allocate the space we used.
if (allocate<char>(pos - result.snippet) != result.snippet) {
throw std::bad_alloc();
}
}
return goodCount;
}
static inline void handleRequest(
const List<SearchResult>& request, List<SearchResult>* response) {
std::vector<ScoredResult> scoredResults;
scoredResults.reserve(request.size);
for (auto& result: request) {
double score = result.score;
if (strstr(result.snippet, " cat ") != nullptr) {
score *= 10000;
}
if (strstr(result.snippet, " dog ") != nullptr) {
score /= 10000;
}
scoredResults.emplace_back(score, &result);
}
std::sort(scoredResults.begin(), scoredResults.end());
response->size = scoredResults.size();
response->items = allocate<SearchResult>(scoredResults.size());
SearchResult* dst = response->items;
for (auto& result: scoredResults) {
dst->url = copyString(result.result->url);
dst->score = result.score;
dst->snippet = copyString(result.result->snippet);
++dst;
}
}
static inline bool checkResponse(
const List<SearchResult>& response, int expectedGoodCount) {
int goodCount = 0;
for (auto& result: response) {
if (result.score > 1001) {
++goodCount;
} else {
break;
}
}
return goodCount == expectedGoodCount;
}
};
// =======================================================================================
struct SingleUseObjects {
template <typename ObjectType>
struct Object {
struct Reusable {};
struct SingleUse {
ObjectType value;
inline SingleUse(Reusable&) {}
};
};
};
struct ReusableObjects {
template <typename ObjectType>
struct Object {
typedef ObjectType Reusable;
struct SingleUse {
ObjectType& value;
inline SingleUse(Reusable& reusable): value(reusable) {}
};
};
};
// =======================================================================================
template <typename TestCase>
uint64_t passByObject(uint64_t iters) {
uint64_t throughput = 0;
for (; iters > 0; --iters) {
arenaPos = arena;
typename TestCase::Request request;
typename TestCase::Expectation expected = TestCase::setupRequest(&request);
typename TestCase::Response response;
TestCase::handleRequest(request, &response);
if (!TestCase::checkResponse(response, expected)) {
throw std::logic_error("Incorrect response.");
}
throughput += (arenaPos - arena) * sizeof(arena[0]);
}
return throughput;
}
template <typename TestCase>
uint64_t doBenchmark(const std::string& mode, uint64_t iters) {
if (mode == "object") {
return passByObject<TestCase>(iters);
} else {
std::cerr << "Unknown mode: " << mode << std::endl;
exit(1);
}
}
int main(int argc, char* argv[]) {
if (argc != 6) {
std::cerr << "USAGE: " << argv[0]
<< " TEST_CASE MODE REUSE COMPRESSION ITERATION_COUNT" << std::endl;
return 1;
}
uint64_t iters = strtoull(argv[5], nullptr, 0);
srand(123);
uint64_t throughput;
std::string testcase = argv[1];
if (testcase == "eval") {
throughput = doBenchmark<ExpressionTestCase>(argv[2], iters);
} else if (testcase == "catrank") {
throughput = doBenchmark<CatRankTestCase>(argv[2], iters);
} else {
std::cerr << "Unknown test case: " << testcase << std::endl;
return 1;
}
std::cout << throughput << std::endl;
return 0;
}
} // namespace null
} // namespace benchmark
} // namespace capnproto
int main(int argc, char* argv[]) {
return capnproto::benchmark::null::main(argc, argv);
}
c++/src/capnproto/benchmark/benchmark-protobuf.c++
View file @ f8b5d71d
......@@ -121,7 +121,7 @@ inline int32_t mod(int32_t a, int32_t b) {
int32_t makeExpression(Expression* exp, int depth) {
exp->set_op((Operation)(rand() % Operation_MAX + 1));
int left, right;
int32_t left, right;
if (rand() % 8 < depth) {
left = rand() % 128 + 1;
......@@ -153,7 +153,7 @@ int32_t makeExpression(Expression* exp, int depth) {
}
int32_t evaluateExpression(const Expression& exp) {
int left, right;
uint32_t left, right;
if (exp.has_left_value()) {
left = exp.left_value();
......@@ -566,7 +566,9 @@ uint64_t server(int inputFd, int outputFd, uint64_t iters) {
}
template <typename TestCase, typename ReuseStrategy, typename Compression>
uint64_t passByObject(uint64_t iters) {
uint64_t passByObject(uint64_t iters, bool countObjectSize) {
uint64_t throughput = 0;
REUSABLE(Request) reusableRequest;
REUSABLE(Response) reusableResponse;
......@@ -581,9 +583,14 @@ uint64_t passByObject(uint64_t iters) {
throw std::logic_error("Incorrect response.");
}
ReuseStrategy::doneWith(response);
if (countObjectSize) {
throughput += request.SpaceUsed();
throughput += response.SpaceUsed();
}
}
return 0;
return throughput;
}
template <typename TestCase, typename ReuseStrategy, typename Compression>
......@@ -679,7 +686,9 @@ uint64_t doBenchmark(const std::string& mode, uint64_t iters) {
return server<TestCase, ReuseStrategy, Compression>(
STDIN_FILENO, STDOUT_FILENO, iters);
} else if (mode == "object") {
return passByObject<TestCase, ReuseStrategy, Compression>(iters);
return passByObject<TestCase, ReuseStrategy, Compression>(iters, false);
} else if (mode == "object-size") {
return passByObject<TestCase, ReuseStrategy, Compression>(iters, true);
} else if (mode == "bytes") {
return passByBytes<TestCase, ReuseStrategy, Compression>(iters);
} else if (mode == "pipe") {
......@@ -729,8 +738,6 @@ int main(int argc, char* argv[]) {
uint64_t iters = strtoull(argv[5], nullptr, 0);
srand(123);
std::cerr << "Doing " << iters << " iterations..." << std::endl;
uint64_t throughput;
std::string testcase = argv[1];
......@@ -743,7 +750,7 @@ int main(int argc, char* argv[]) {
return 1;
}
std::cerr << "Average messages size = " << (throughput / iters) << std::endl;
std::cout << throughput << std::endl;
return 0;
}
......
c++/src/capnproto/benchmark/benchmark-runner.c++ 0 → 100644
View file @ f8b5d71d
// Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <sys/types.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <sys/wait.h>
#include <sys/stat.h>
#include <inttypes.h>
#include <string>
#include <stdio.h>
#include <unistd.h>
#include <string.h>
#include <iostream>
#include <iomanip>
using namespace std;
namespace capnproto {
namespace benchmark {
namespace runner {
struct Times {
uint64_t real;
uint64_t user;
uint64_t sys;
uint64_t cpu() { return user + sys; }
Times operator-(const Times& other) {
Times result;
result.real = real - other.real;
result.user = user - other.user;
result.sys = sys - other.sys;
return result;
}
};
uint64_t asNanosecs(const struct timeval& tv) {
return (uint64_t)tv.tv_sec * 1000000000 + (uint64_t)tv.tv_usec * 1000;
}
Times currentTimes() {
Times result;
struct rusage self, children;
getrusage(RUSAGE_SELF, &self);
getrusage(RUSAGE_CHILDREN, &children);
struct timeval real;
gettimeofday(&real, nullptr);
result.real = asNanosecs(real);
result.user = asNanosecs(self.ru_utime) + asNanosecs(children.ru_utime);
result.sys = asNanosecs(self.ru_stime) + asNanosecs(children.ru_stime);
return result;
}
struct TestResult {
uint64_t throughput;
Times time;
};
enum class Product {
CAPNPROTO,
PROTOBUF,
NULLCASE
};
enum class TestCase {
EVAL,
CATRANK
};
enum class Mode {
OBJECTS,
OBJECT_SIZE,
BYTES,
PIPE_SYNC,
PIPE_ASYNC
};
enum class Reuse {
YES,
NO
};
enum class Compression {
SNAPPY,
NONE
};
TestResult runTest(Product product, TestCase testCase, Mode mode, Reuse reuse,
Compression compression, uint64_t iters) {
char* argv[7];
switch (product) {
case Product::CAPNPROTO:
argv[0] = strdup("benchmark-capnproto");
break;
case Product::PROTOBUF:
argv[0] = strdup("benchmark-protobuf");
break;
case Product::NULLCASE:
argv[0] = strdup("benchmark-null");
break;
}
switch (testCase) {
case TestCase::EVAL:
argv[1] = strdup("eval");
break;
case TestCase::CATRANK:
argv[1] = strdup("catrank");
break;
}
switch (mode) {
case Mode::OBJECTS:
argv[2] = strdup("object");
break;
case Mode::OBJECT_SIZE:
argv[2] = strdup("object-size");
break;
case Mode::BYTES:
argv[2] = strdup("bytes");
break;
case Mode::PIPE_SYNC:
argv[2] = strdup("pipe");
break;
case Mode::PIPE_ASYNC:
argv[2] = strdup("pipe-async");
break;
}
switch (reuse) {
case Reuse::YES:
argv[3] = strdup("reuse");
break;
case Reuse::NO:
argv[3] = strdup("no-reuse");
break;
}
switch (compression) {
case Compression::SNAPPY:
argv[4] = strdup("snappy");
break;
case Compression::NONE:
argv[4] = strdup("none");
break;
}
char itersStr[64];
sprintf(itersStr, "%llu", (long long unsigned int)iters);
argv[5] = itersStr;
argv[6] = nullptr;
// Make pipe for child to write throughput.
int childPipe[2];
if (pipe(childPipe) < 0) {
perror("pipe");
exit(1);
}
// Spawn the child process.
struct timeval start, end;
gettimeofday(&start, nullptr);
pid_t child = fork();
if (child == 0) {
close(childPipe[0]);
dup2(childPipe[1], STDOUT_FILENO);
close(childPipe[1]);
execv(argv[0], argv);
exit(1);
}
close(childPipe[1]);
for (int i = 0; i < 5; i++) {
free(argv[i]);
}
// Read throughput number written to child's stdout.
FILE* input = fdopen(childPipe[0], "r");
long long unsigned int throughput;
if (fscanf(input, "%lld", &throughput) != 1) {
fprintf(stderr, "Child didn't write throughput to stdout.");
}
char buffer[1024];
while (fgets(buffer, sizeof(buffer), input) != nullptr) {
// Loop until EOF.
}
fclose(input);
// Wait for child exit.
int status;
struct rusage usage;
wait4(child, &status, 0, &usage);
gettimeofday(&end, nullptr);
// Calculate results.
TestResult result;
result.throughput = throughput;
result.time.real = asNanosecs(end) - asNanosecs(start);
result.time.user = asNanosecs(usage.ru_utime);
result.time.sys = asNanosecs(usage.ru_stime);
return result;
}
void reportTableHeader() {
cout << setw(50) << right << "obj size or"
<< endl;
cout << setw(40) << left << "Test"
<< setw(10) << right << "I/O bytes"
<< setw(10) << right << "wall ns"
<< setw(10) << right << "user ns"
<< setw(10) << right << "sys ns"
<< endl;
cout << setfill('=') << setw(80) << "" << setfill(' ') << endl;
}
void reportResults(const char* name, uint64_t iters, TestResult results) {
cout << setw(40) << left << name
<< setw(10) << right << (results.throughput / iters)
<< setw(10) << right << (results.time.real / iters)
<< setw(10) << right << (results.time.user / iters)
<< setw(10) << right << (results.time.sys / iters)
<< endl;
}
void reportComparisonHeader() {
cout << setw(35) << left << "Overhead type"
<< setw(15) << right << "Protobuf"
<< setw(15) << right << "Cap'n Proto"
<< setw(15) << right << "Improvement"
<< endl;
cout << setfill('=') << setw(80) << "" << setfill(' ') << endl;
}
class Gain {
public:
Gain(double oldValue, double newValue)
: amount(oldValue < newValue ? newValue / oldValue : -oldValue / newValue) {}
void writeTo(std::ostream& os) {
if (-2 < amount && amount < 2) {
double percent = (amount > 0 ? amount - 1 : amount + 1) * 100;
os << (int)(percent + 0.5) << "%";
} else {
os << fixed << setprecision(2) << amount << "x";
}
}
private:
double amount;
};
ostream& operator<<(ostream& os, Gain gain) {
gain.writeTo(os);
return os;
}
void reportComparison(const char* name, double base, double protobuf, double capnproto,
double iters) {
cout << setw(35) << left << name
<< setw(14) << right << Gain(base, protobuf)
<< setw(14) << right << Gain(base, capnproto);
// Since smaller is better, the "improvement" is the "gain" from capnproto to protobuf.
cout << setw(14) << right << Gain(capnproto - base, protobuf - base) << endl;
}
void reportComparison(const char* name, const char* unit, double protobuf, double capnproto,
double iters) {
cout << setw(35) << left << name
<< setw(15-strlen(unit)) << right << setprecision(2) << (protobuf / iters) << unit
<< setw(15-strlen(unit)) << right << setprecision(2) << (capnproto / iters) << unit;
// Since smaller is better, the "improvement" is the "gain" from capnproto to protobuf.
cout << setw(14) << right << Gain(capnproto, protobuf) << endl;
}
size_t fileSize(const char* name) {
struct stat stats;
if (stat(name, &stats) < 0) {
perror(name);
exit(1);
}
return stats.st_size;
}
int main(int argc, char* argv[]) {
char* path = argv[0];
char* slashpos = strrchr(path, '/');
if (slashpos != nullptr) {
*slashpos = '\0';
if (chdir(path) < 0) {
perror("chdir");
return 1;
}
*slashpos = '/';
}
TestCase testCase = TestCase::CATRANK;
Mode mode = Mode::PIPE_SYNC;
Reuse reuse = Reuse::YES;
Compression compression = Compression::NONE;
uint64_t iters = 1;
for (int i = 1; i < argc; i++) {
string arg = argv[i];
if (isdigit(argv[i][0])) {
iters = strtoul(argv[i], nullptr, 0);
} else if (arg == "async") {
mode = Mode::PIPE_ASYNC;
} else if (arg == "inmem") {
mode = Mode::BYTES;
} else if (arg == "eval") {
testCase = TestCase::EVAL;
} else if (arg == "no-reuse") {
reuse = Reuse::NO;
} else if (arg == "snappy") {
compression = Compression::SNAPPY;
}
}
switch (testCase) {
case TestCase::EVAL:
iters *= 100000;
break;
case TestCase::CATRANK:
iters *= 1000;
break;
}
cout << "Running " << iters << " iterations of ";
switch (testCase) {
case TestCase::EVAL:
cout << "calculator";
break;
case TestCase::CATRANK:
cout << "CatRank";
break;
}
cout << " example case with:" << endl;
switch (mode) {
case Mode::OBJECTS:
case Mode::OBJECT_SIZE:
// Can't happen.
break;
case Mode::BYTES:
cout << "* client and server in the same process (passing bytes in memory)" << endl;
break;
case Mode::PIPE_SYNC:
cout << "* client and server passing messages over pipes" << endl;
cout << "* client sending one request at a time" << endl;
break;
case Mode::PIPE_ASYNC:
cout << "* client and server passing messages over pipes" << endl;
cout << "* client saturating pipe with requests without waiting for responses" << endl;
break;
}
switch (reuse) {
case Reuse::YES:
cout << "* ideal object reuse" << endl;
break;
case Reuse::NO:
cout << "* no object reuse" << endl;
break;
}
switch (compression) {
case Compression::SNAPPY:
cout << "* Snappy compression" << endl;
break;
case Compression::NONE:
cout << "* no compression" << endl;
break;
}
cout << endl;
reportTableHeader();
TestResult nullCase = runTest(
Product::NULLCASE, testCase, Mode::OBJECTS, reuse, compression, iters);
reportResults("Theoretical best pass-by-object", iters, nullCase);
TestResult protobufBase = runTest(
Product::PROTOBUF, testCase, Mode::OBJECTS, reuse, compression, iters);
protobufBase.throughput = runTest(
Product::PROTOBUF, testCase, Mode::OBJECT_SIZE, reuse, compression, iters).throughput;
reportResults("Protobuf pass-by-object", iters, protobufBase);
TestResult protobuf = runTest(
Product::PROTOBUF, testCase, mode, reuse, compression, iters);
reportResults("Protobuf end-to-end", iters, protobuf);
TestResult capnpBase = runTest(
Product::CAPNPROTO, testCase, Mode::OBJECTS, reuse, compression, iters);
capnpBase.throughput = runTest(
Product::CAPNPROTO, testCase, Mode::OBJECT_SIZE, reuse, compression, iters).throughput;
reportResults("Cap'n Proto pass-by-object", iters, capnpBase);
TestResult capnp = runTest(
Product::CAPNPROTO, testCase, mode, reuse, compression, iters);
reportResults("Cap'n Proto end-to-end", iters, capnp);
cout << endl;
reportComparisonHeader();
reportComparison("memory",
nullCase.throughput, protobufBase.throughput, capnpBase.throughput, iters);
reportComparison("object manipulation",
nullCase.time.cpu(), protobufBase.time.cpu(), capnpBase.time.cpu(), iters);
reportComparison("I/O overhead", "us",
(protobuf.time.cpu() - protobufBase.time.cpu()) / 1000.0,
(capnp.time.cpu() - capnpBase.time.cpu()) / 1000.0, iters);
reportComparison("binary size", "kB",
fileSize("benchmark-protobuf") / 1024.0, fileSize("benchmark-capnproto") / 1024.0, 1);
reportComparison("generated code size", "kB",
fileSize("benchmark.pb.cc") / 1024.0 + fileSize("benchmark.pb.h") / 1024.0,
fileSize("benchmark.capnp.c++") / 1024.0 + fileSize("benchmark.capnp.h") / 1024.0, 1);
reportComparison("generated obj size", "kB",
fileSize("benchmark.pb.o") / 1024.0, fileSize("benchmark.capnp.o") / 1024.0, 1);
return 0;
}
} // namespace runner
} // namespace benchmark
} // namespace capnproto
int main(int argc, char* argv[]) {
return capnproto::benchmark::runner::main(argc, argv);
}
c++/src/capnproto/message.c++
View file @ f8b5d71d
......@@ -204,7 +204,16 @@ MallocMessageBuilder::MallocMessageBuilder(
ownFirstSegment(false), firstSegment(firstSegment.begin()) {}
MallocMessageBuilder::~MallocMessageBuilder() {
if (ownFirstSegment) free(firstSegment);
if (ownFirstSegment) {
free(firstSegment);
} else {
ArrayPtr<const ArrayPtr<const word>> segments = getSegmentsForOutput();
if (segments.size() > 0) {
CAPNPROTO_ASSERT(segments[0].begin() == firstSegment,
"First segment in getSegmentsForOutput() is not the first segment allocated?");
memset(firstSegment, 0, segments[0].size() * sizeof(word));
}
}
if (moreSegments != nullptr) {
for (void* ptr: moreSegments->segments) {
free(ptr);
......@@ -218,7 +227,6 @@ ArrayPtr<word> MallocMessageBuilder::allocateSegment(uint minimumSize) {
firstSegment = nullptr;
ownFirstSegment = true;
if (result.size() >= minimumSize) {
memset(result.begin(), 0, result.size() * sizeof(word));
return result;
}
// If the provided first segment wasn't big enough, we discard it and proceed to allocate
......
c++/src/capnproto/message.h
View file @ f8b5d71d
......@@ -268,6 +268,9 @@ public:
// This version always returns the given array for the first segment, and then proceeds with the
// allocation strategy. This is useful for optimization when building lots of small messages in
// a tight loop: you can reuse the space for the first segment.
//
// firstSegment MUST be zero-initialized. MallocMessageBuilder's destructor will write new zeros
// over any space that was used so that it can be reused.
CAPNPROTO_DISALLOW_COPY(MallocMessageBuilder);
virtual ~MallocMessageBuilder();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment