Skip to content

C
capnproto
Commit de95af09 authored by Kenton Varda's avatar Kenton Varda
Browse files
Options

Delete snappy code. It relied on an unsubmitted patch to Snappy's interface and… 

Delete snappy code.  It relied on an unsubmitted patch to Snappy's interface and it turns out LZ4 is better anyway so we'll probably integrate with that instead, eventually.
parent e0ac5f02
Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 590 deletions
c++/src/capnp/serialize-snappy-test.c++ deleted 100644 → 0
View file @ e0ac5f02
// Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "serialize-snappy.h"
#include <kj/debug.h>
#include "test.capnp.h"
#include <gtest/gtest.h>
#include <string>
#include <stdlib.h>
#include "test-util.h"
namespace capnp {
namespace _ { // private
namespace {
class TestMessageBuilder: public MallocMessageBuilder {
// A MessageBuilder that tries to allocate an exact number of total segments, by allocating
// minimum-size segments until it reaches the number, then allocating one large segment to
// finish.
public:
explicit TestMessageBuilder(uint desiredSegmentCount)
: MallocMessageBuilder(0, AllocationStrategy::FIXED_SIZE),
desiredSegmentCount(desiredSegmentCount) {}
~TestMessageBuilder() {
EXPECT_EQ(0u, desiredSegmentCount);
}
kj::ArrayPtr<word> allocateSegment(uint minimumSize) override {
if (desiredSegmentCount <= 1) {
if (desiredSegmentCount < 1) {
ADD_FAILURE() << "Allocated more segments than desired.";
} else {
--desiredSegmentCount;
}
return MallocMessageBuilder::allocateSegment(SUGGESTED_FIRST_SEGMENT_WORDS);
} else {
--desiredSegmentCount;
return MallocMessageBuilder::allocateSegment(minimumSize);
}
}
private:
uint desiredSegmentCount;
};
class TestPipe: public kj::BufferedInputStream, public kj::OutputStream {
public:
TestPipe()
: preferredReadSize(kj::maxValue), readPos(0) {}
explicit TestPipe(size_t preferredReadSize)
: preferredReadSize(preferredReadSize), readPos(0) {}
~TestPipe() {}
const std::string& getData() { return data; }
std::string getUnreadData() { return data.substr(readPos); }
std::string::size_type getReadPos() { return readPos; }
void resetRead(size_t preferredReadSize = kj::maxValue) {
readPos = 0;
this->preferredReadSize = preferredReadSize;
}
bool allRead() {
return readPos == data.size();
}
void clear(size_t preferredReadSize = kj::maxValue) {
resetRead(preferredReadSize);
data.clear();
}
void write(const void* buffer, size_t size) override {
data.append(reinterpret_cast<const char*>(buffer), size);
}
size_t tryRead(void* buffer, size_t minBytes, size_t maxBytes) override {
KJ_ASSERT(maxBytes <= data.size() - readPos, "Overran end of stream.");
size_t amount = std::min(maxBytes, std::max(minBytes, preferredReadSize));
memcpy(buffer, data.data() + readPos, amount);
readPos += amount;
return amount;
}
void skip(size_t bytes) override {
KJ_ASSERT(bytes <= data.size() - readPos, "Overran end of stream.");
readPos += bytes;
}
kj::ArrayPtr<const byte> tryGetReadBuffer() override {
size_t amount = std::min(data.size() - readPos, preferredReadSize);
return kj::arrayPtr(reinterpret_cast<const byte*>(data.data() + readPos), amount);
}
private:
size_t preferredReadSize;
std::string data;
std::string::size_type readPos;
};
//struct DisplayByteArray {
// DisplayByteArray(const std::string& str)
// : data(reinterpret_cast<const uint8_t*>(str.data())), size(str.size()) {}
// DisplayByteArray(const std::initializer_list<uint8_t>& list)
// : data(list.begin()), size(list.size()) {}
//
// const uint8_t* data;
// size_t size;
//};
//
//std::ostream& operator<<(std::ostream& os, const DisplayByteArray& bytes) {
// os << "{ ";
// for (size_t i = 0; i < bytes.size; i++) {
// if (i > 0) {
// os << ", ";
// }
// os << (uint)bytes.data[i];
// }
// os << " }";
//
// return os;
//}
TEST(Snappy, RoundTrip) {
TestMessageBuilder builder(1);
initTestMessage(builder.initRoot<TestAllTypes>());
TestPipe pipe;
writeSnappyPackedMessage(pipe, builder);
SnappyPackedMessageReader reader(pipe);
checkTestMessage(reader.getRoot<TestAllTypes>());
EXPECT_TRUE(pipe.allRead());
}
TEST(Snappy, RoundTripScratchSpace) {
TestMessageBuilder builder(1);
initTestMessage(builder.initRoot<TestAllTypes>());
TestPipe pipe;
writeSnappyPackedMessage(pipe, builder);
word scratch[1024];
SnappyPackedMessageReader reader(pipe, ReaderOptions(), kj::ArrayPtr<word>(scratch, 1024));
checkTestMessage(reader.getRoot<TestAllTypes>());
EXPECT_TRUE(pipe.allRead());
}
TEST(Snappy, RoundTripLazy) {
TestMessageBuilder builder(1);
initTestMessage(builder.initRoot<TestAllTypes>());
TestPipe pipe(1);
writeSnappyPackedMessage(pipe, builder);
SnappyPackedMessageReader reader(pipe);
checkTestMessage(reader.getRoot<TestAllTypes>());
EXPECT_TRUE(pipe.allRead());
}
TEST(Snappy, RoundTripOddSegmentCount) {
TestMessageBuilder builder(7);
initTestMessage(builder.initRoot<TestAllTypes>());
TestPipe pipe;
writeSnappyPackedMessage(pipe, builder);
SnappyPackedMessageReader reader(pipe);
checkTestMessage(reader.getRoot<TestAllTypes>());
EXPECT_TRUE(pipe.allRead());
}
TEST(Snappy, RoundTripOddSegmentCountLazy) {
TestMessageBuilder builder(7);
initTestMessage(builder.initRoot<TestAllTypes>());
TestPipe pipe(1);
writeSnappyPackedMessage(pipe, builder);
SnappyPackedMessageReader reader(pipe);
checkTestMessage(reader.getRoot<TestAllTypes>());
EXPECT_TRUE(pipe.allRead());
}
TEST(Snappy, RoundTripEvenSegmentCount) {
TestMessageBuilder builder(10);
initTestMessage(builder.initRoot<TestAllTypes>());
TestPipe pipe;
writeSnappyPackedMessage(pipe, builder);
SnappyPackedMessageReader reader(pipe);
checkTestMessage(reader.getRoot<TestAllTypes>());
EXPECT_TRUE(pipe.allRead());
}
TEST(Snappy, RoundTripEvenSegmentCountLazy) {
TestMessageBuilder builder(10);
initTestMessage(builder.initRoot<TestAllTypes>());
TestPipe pipe(1);
writeSnappyPackedMessage(pipe, builder);
SnappyPackedMessageReader reader(pipe);
checkTestMessage(reader.getRoot<TestAllTypes>());
EXPECT_TRUE(pipe.allRead());
}
TEST(Snappy, RoundTripTwoMessages) {
TestMessageBuilder builder(1);
initTestMessage(builder.initRoot<TestAllTypes>());
TestMessageBuilder builder2(1);
builder2.initRoot<TestAllTypes>().setTextField("Second message.");
TestPipe pipe(1);
writeSnappyPackedMessage(pipe, builder);
size_t firstSize = pipe.getData().size();
writeSnappyPackedMessage(pipe, builder2);
{
SnappyPackedMessageReader reader(pipe);
checkTestMessage(reader.getRoot<TestAllTypes>());
}
EXPECT_EQ(firstSize, pipe.getReadPos());
{
SnappyPackedMessageReader reader(pipe);
EXPECT_EQ("Second message.", reader.getRoot<TestAllTypes>().getTextField());
}
EXPECT_TRUE(pipe.allRead());
}
// TODO(test): Test error cases.
} // namespace
} // namespace _ (private)
} // namespace capnp
c++/src/capnp/serialize-snappy.c++ deleted 100644 → 0
View file @ e0ac5f02
// Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "serialize-snappy.h"
#include <kj/debug.h>
#include "layout.h"
#include <snappy/snappy.h>
#include <snappy/snappy-sinksource.h>
#include <vector>
namespace capnp {
class SnappyInputStream::InputStreamSnappySource: public snappy::Source {
public:
inline InputStreamSnappySource(BufferedInputStream& inputStream)
: inputStream(inputStream) {}
inline ~InputStreamSnappySource() noexcept {}
bool atEnd() {
return inputStream.getReadBuffer().size() == 0;
}
// implements snappy::Source ---------------------------------------
size_t Available() const override {
KJ_FAIL_ASSERT("Snappy doesn't actually call this.");
return 0;
}
const char* Peek(size_t* len) override {
kj::ArrayPtr<const byte> buffer = inputStream.getReadBuffer();
*len = buffer.size();
return reinterpret_cast<const char*>(buffer.begin());
}
void Skip(size_t n) override {
inputStream.skip(n);
}
private:
BufferedInputStream& inputStream;
};
SnappyInputStream::SnappyInputStream(BufferedInputStream& inner, kj::ArrayPtr<byte> buffer)
: inner(inner) {
if (buffer.size() < SNAPPY_BUFFER_SIZE) {
ownedBuffer = kj::heapArray<byte>(SNAPPY_BUFFER_SIZE);
buffer = ownedBuffer;
}
this->buffer = buffer;
}
SnappyInputStream::~SnappyInputStream() noexcept(false) {}
kj::ArrayPtr<const byte> SnappyInputStream::tryGetReadBuffer() {
if (bufferAvailable.size() == 0) {
refill();
}
return bufferAvailable;
}
size_t SnappyInputStream::tryRead(void* dst, size_t minBytes, size_t maxBytes) {
size_t total = 0;
while (minBytes > bufferAvailable.size()) {
memcpy(dst, bufferAvailable.begin(), bufferAvailable.size());
dst = reinterpret_cast<byte*>(dst) + bufferAvailable.size();
total += bufferAvailable.size();
minBytes -= bufferAvailable.size();
maxBytes -= bufferAvailable.size();
if (!refill()) {
return total;
}
}
// Serve from current buffer.
size_t n = std::min(bufferAvailable.size(), maxBytes);
memcpy(dst, bufferAvailable.begin(), n);
bufferAvailable = bufferAvailable.slice(n, bufferAvailable.size());
return total + n;
}
void SnappyInputStream::skip(size_t bytes) {
while (bytes > bufferAvailable.size()) {
bytes -= bufferAvailable.size();
KJ_REQUIRE(refill(), "Premature EOF");
}
bufferAvailable = bufferAvailable.slice(bytes, bufferAvailable.size());
}
bool SnappyInputStream::refill() {
uint32_t length = 0;
InputStreamSnappySource snappySource(inner);
if (snappySource.atEnd()) {
return false;
}
KJ_REQUIRE(
snappy::RawUncompress(
&snappySource, reinterpret_cast<char*>(buffer.begin()), buffer.size(), &length),
"Snappy decompression failed.") {
return false;
}
bufferAvailable = buffer.slice(0, length);
return true;
}
// =======================================================================================
SnappyOutputStream::SnappyOutputStream(
OutputStream& inner, kj::ArrayPtr<byte> buffer, kj::ArrayPtr<byte> compressedBuffer)
: inner(inner) {
KJ_DASSERT(SNAPPY_COMPRESSED_BUFFER_SIZE >= snappy::MaxCompressedLength(snappy::kBlockSize),
"snappy::MaxCompressedLength() changed?");
if (buffer.size() < SNAPPY_BUFFER_SIZE) {
ownedBuffer = kj::heapArray<byte>(SNAPPY_BUFFER_SIZE);
buffer = ownedBuffer;
}
this->buffer = buffer;
bufferPos = buffer.begin();
if (compressedBuffer.size() < SNAPPY_COMPRESSED_BUFFER_SIZE) {
ownedCompressedBuffer = kj::heapArray<byte>(SNAPPY_COMPRESSED_BUFFER_SIZE);
compressedBuffer = ownedCompressedBuffer;
}
this->compressedBuffer = compressedBuffer;
}
SnappyOutputStream::~SnappyOutputStream() noexcept(false) {
if (bufferPos > buffer.begin()) {
unwindDetector.catchExceptionsIfUnwinding([&]() {
flush();
});
}
}
void SnappyOutputStream::flush() {
if (bufferPos > buffer.begin()) {
snappy::ByteArraySource source(
reinterpret_cast<char*>(buffer.begin()), bufferPos - buffer.begin());
snappy::UncheckedByteArraySink sink(reinterpret_cast<char*>(compressedBuffer.begin()));
size_t n = snappy::Compress(&source, &sink);
KJ_ASSERT(n <= compressedBuffer.size(),
"Critical security bug: Snappy compression overran its output buffer.");
inner.write(compressedBuffer.begin(), n);
bufferPos = buffer.begin();
}
}
kj::ArrayPtr<byte> SnappyOutputStream::getWriteBuffer() {
return kj::arrayPtr(bufferPos, buffer.end());
}
void SnappyOutputStream::write(const void* src, size_t size) {
if (src == bufferPos) {
// Oh goody, the caller wrote directly into our buffer.
bufferPos += size;
} else {
for (;;) {
size_t available = buffer.end() - bufferPos;
if (size < available) break;
memcpy(bufferPos, src, available);
size -= available;
src = reinterpret_cast<const byte*>(src) + available;
bufferPos = buffer.end();
flush();
}
memcpy(bufferPos, src, size);
bufferPos += size;
}
}
// =======================================================================================
SnappyPackedMessageReader::SnappyPackedMessageReader(
BufferedInputStream& inputStream, ReaderOptions options,
kj::ArrayPtr<word> scratchSpace, kj::ArrayPtr<byte> buffer)
: SnappyInputStream(inputStream, buffer),
PackedMessageReader(static_cast<SnappyInputStream&>(*this), options, scratchSpace) {}
SnappyPackedMessageReader::~SnappyPackedMessageReader() noexcept(false) {}
void writeSnappyPackedMessage(kj::OutputStream& output,
kj::ArrayPtr<const kj::ArrayPtr<const word>> segments,
kj::ArrayPtr<byte> buffer, kj::ArrayPtr<byte> compressedBuffer) {
SnappyOutputStream snappyOut(output, buffer, compressedBuffer);
writePackedMessage(snappyOut, segments);
}
} // namespace capnp
c++/src/capnp/serialize-snappy.h deleted 100644 → 0
View file @ e0ac5f02
// Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef CAPNP_SERIALIZE_SNAPPY_H_
#define CAPNP_SERIALIZE_SNAPPY_H_
#include "serialize.h"
#include "serialize-packed.h"
namespace capnp {
constexpr size_t SNAPPY_BUFFER_SIZE = 65536;
constexpr size_t SNAPPY_COMPRESSED_BUFFER_SIZE = 76490;
class SnappyInputStream: public kj::BufferedInputStream {
public:
explicit SnappyInputStream(BufferedInputStream& inner, kj::ArrayPtr<byte> buffer = nullptr);
KJ_DISALLOW_COPY(SnappyInputStream);
~SnappyInputStream() noexcept(false);
// implements BufferedInputStream ----------------------------------
kj::ArrayPtr<const byte> tryGetReadBuffer() override;
size_t tryRead(void* buffer, size_t minBytes, size_t maxBytes) override;
void skip(size_t bytes) override;
private:
class InputStreamSnappySource;
BufferedInputStream& inner;
kj::Array<byte> ownedBuffer;
kj::ArrayPtr<byte> buffer;
kj::ArrayPtr<byte> bufferAvailable;
bool refill();
};
class SnappyOutputStream: public kj::BufferedOutputStream {
public:
explicit SnappyOutputStream(OutputStream& inner,
kj::ArrayPtr<byte> buffer = nullptr,
kj::ArrayPtr<byte> compressedBuffer = nullptr);
KJ_DISALLOW_COPY(SnappyOutputStream);
~SnappyOutputStream() noexcept(false);
void flush();
// Force the stream to write any remaining bytes in its buffer to the inner stream. This will
// hurt compression, of course, by forcing the current block to end prematurely.
// implements BufferedOutputStream ---------------------------------
kj::ArrayPtr<byte> getWriteBuffer() override;
void write(const void* buffer, size_t size) override;
private:
OutputStream& inner;
kj::Array<byte> ownedBuffer;
kj::ArrayPtr<byte> buffer;
byte* bufferPos;
kj::Array<byte> ownedCompressedBuffer;
kj::ArrayPtr<byte> compressedBuffer;
kj::UnwindDetector unwindDetector;
};
class SnappyPackedMessageReader: private SnappyInputStream, public PackedMessageReader {
public:
SnappyPackedMessageReader(
BufferedInputStream& inputStream, ReaderOptions options = ReaderOptions(),
kj::ArrayPtr<word> scratchSpace = nullptr, kj::ArrayPtr<byte> buffer = nullptr);
~SnappyPackedMessageReader() noexcept(false);
};
void writeSnappyPackedMessage(kj::OutputStream& output, MessageBuilder& builder,
kj::ArrayPtr<byte> buffer = nullptr,
kj::ArrayPtr<byte> compressedBuffer = nullptr);
void writeSnappyPackedMessage(kj::OutputStream& output,
kj::ArrayPtr<const kj::ArrayPtr<const word>> segments,
kj::ArrayPtr<byte> buffer = nullptr,
kj::ArrayPtr<byte> compressedBuffer = nullptr);
// =======================================================================================
// inline stuff
inline void writeSnappyPackedMessage(kj::OutputStream& output, MessageBuilder& builder,
kj::ArrayPtr<byte> buffer,
kj::ArrayPtr<byte> compressedBuffer) {
writeSnappyPackedMessage(output, builder.getSegmentsForOutput(), buffer, compressedBuffer);
}
} // namespace capnp
#endif // CAPNP_SERIALIZE_SNAPPY_H_
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment