serialize.h 6.96 KB
Newer Older
Kenton Varda's avatar
Kenton Varda committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42
// Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
//    list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
//    this list of conditions and the following disclaimer in the documentation
//    and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// This file implements a simple serialization format for Cap'n Proto messages.  The format
// is as follows:
//
// * 32-bit little-endian segment count (4 bytes).
// * 32-bit little-endian size of each segment (4*(segment count) bytes).
// * Padding so that subsequent data is 64-bit-aligned (0 or 4 bytes).  (I.e., if there are an even
//     number of segments, there are 4 bytes of zeros here, otherwise there is no padding.)
// * Data from each segment, in order (8*sum(segment sizes) bytes)
//
// This format has some important properties:
// - It is self-delimiting, so multiple messages may be written to a stream without any external
//   delimiter.
// - The total size and position of each segment can be determined by reading only the first part
//   of the message, allowing lazy and random-access reading of the segment data.
// - A message is always at least 8 bytes.
// - A single-segment message can be read entirely in two system calls with no buffering.
// - A multi-segment message can be read entirely in three system calls with no buffering.
// - The format is appropriate for mmap()ing since all data is aligned.

Kenton Varda's avatar
Kenton Varda committed
43 44
#ifndef CAPNP_SERIALIZE_H_
#define CAPNP_SERIALIZE_H_
Kenton Varda's avatar
Kenton Varda committed
45 46

#include "message.h"
47
#include <kj/io.h>
Kenton Varda's avatar
Kenton Varda committed
48

49
namespace capnp {
Kenton Varda's avatar
Kenton Varda committed
50 51 52 53 54 55

class FlatArrayMessageReader: public MessageReader {
  // Parses a message from a flat array.  Note that it makes sense to use this together with mmap()
  // for extremely fast parsing.

public:
56
  FlatArrayMessageReader(kj::ArrayPtr<const word> array, ReaderOptions options = ReaderOptions());
Kenton Varda's avatar
Kenton Varda committed
57 58
  // The array must remain valid until the MessageReader is destroyed.

59
  kj::ArrayPtr<const word> getSegment(uint id) override;
Kenton Varda's avatar
Kenton Varda committed
60 61 62

private:
  // Optimize for single-segment case.
63 64
  kj::ArrayPtr<const word> segment0;
  kj::Array<kj::ArrayPtr<const word>> moreSegments;
Kenton Varda's avatar
Kenton Varda committed
65 66
};

67
kj::Array<word> messageToFlatArray(MessageBuilder& builder);
Kenton Varda's avatar
Kenton Varda committed
68 69
// Constructs a flat array containing the entire content of the given message.

70
kj::Array<word> messageToFlatArray(kj::ArrayPtr<const kj::ArrayPtr<const word>> segments);
Kenton Varda's avatar
Kenton Varda committed
71 72 73 74 75 76
// Version of messageToFlatArray that takes a raw segment array.

// =======================================================================================

class InputStreamMessageReader: public MessageReader {
public:
77
  InputStreamMessageReader(kj::InputStream& inputStream,
Kenton Varda's avatar
Kenton Varda committed
78
                           ReaderOptions options = ReaderOptions(),
79
                           kj::ArrayPtr<word> scratchSpace = nullptr);
80
  ~InputStreamMessageReader() noexcept(false);
81

Kenton Varda's avatar
Kenton Varda committed
82
  // implements MessageReader ----------------------------------------
83
  kj::ArrayPtr<const word> getSegment(uint id) override;
Kenton Varda's avatar
Kenton Varda committed
84 85

private:
86
  kj::InputStream& inputStream;
87
  byte* readPos;
Kenton Varda's avatar
Kenton Varda committed
88 89

  // Optimize for single-segment case.
90 91
  kj::ArrayPtr<const word> segment0;
  kj::Array<kj::ArrayPtr<const word>> moreSegments;
Kenton Varda's avatar
Kenton Varda committed
92

93
  kj::Array<word> ownedSpace;
94
  // Only if scratchSpace wasn't big enough.
95 96

  kj::UnwindDetector unwindDetector;
Kenton Varda's avatar
Kenton Varda committed
97 98
};

99
void writeMessage(kj::OutputStream& output, MessageBuilder& builder);
Kenton Varda's avatar
Kenton Varda committed
100 101
// Write the message to the given output stream.

102
void writeMessage(kj::OutputStream& output, kj::ArrayPtr<const kj::ArrayPtr<const word>> segments);
Kenton Varda's avatar
Kenton Varda committed
103 104 105 106 107
// Write the segment array to the given output stream.

// =======================================================================================
// Specializations for reading from / writing to file descriptors.

108
class StreamFdMessageReader: private kj::FdInputStream, public InputStreamMessageReader {
Kenton Varda's avatar
Kenton Varda committed
109 110 111 112 113
  // A MessageReader that reads from a steam-based file descriptor.  For seekable file descriptors
  // (e.g. actual disk files), FdFileMessageReader is better, but this will still work.

public:
  StreamFdMessageReader(int fd, ReaderOptions options = ReaderOptions(),
114
                        kj::ArrayPtr<word> scratchSpace = nullptr)
115
      : FdInputStream(fd), InputStreamMessageReader(*this, options, scratchSpace) {}
Kenton Varda's avatar
Kenton Varda committed
116 117
  // Read message from a file descriptor, without taking ownership of the descriptor.

118
  StreamFdMessageReader(kj::AutoCloseFd fd, ReaderOptions options = ReaderOptions(),
119
                        kj::ArrayPtr<word> scratchSpace = nullptr)
Kenton Varda's avatar
Kenton Varda committed
120
      : FdInputStream(kj::mv(fd)), InputStreamMessageReader(*this, options, scratchSpace) {}
Kenton Varda's avatar
Kenton Varda committed
121 122
  // Read a message from a file descriptor, taking ownership of the descriptor.

123
  ~StreamFdMessageReader() noexcept(false);
Kenton Varda's avatar
Kenton Varda committed
124 125 126 127 128 129 130 131 132
};

void writeMessageToFd(int fd, MessageBuilder& builder);
// Write the message to the given file descriptor.
//
// This function throws an exception on any I/O error.  If your code is not exception-safe, be sure
// you catch this exception at the call site.  If throwing an exception is not acceptable, you
// can implement your own OutputStream with arbitrary error handling and then use writeMessage().

133
void writeMessageToFd(int fd, kj::ArrayPtr<const kj::ArrayPtr<const word>> segments);
Kenton Varda's avatar
Kenton Varda committed
134 135 136 137 138 139 140 141 142
// Write the segment array to the given file descriptor.
//
// This function throws an exception on any I/O error.  If your code is not exception-safe, be sure
// you catch this exception at the call site.  If throwing an exception is not acceptable, you
// can implement your own OutputStream with arbitrary error handling and then use writeMessage().

// =======================================================================================
// inline stuff

143
inline kj::Array<word> messageToFlatArray(MessageBuilder& builder) {
Kenton Varda's avatar
Kenton Varda committed
144 145 146
  return messageToFlatArray(builder.getSegmentsForOutput());
}

147
inline void writeMessage(kj::OutputStream& output, MessageBuilder& builder) {
Kenton Varda's avatar
Kenton Varda committed
148 149 150 151 152 153 154
  writeMessage(output, builder.getSegmentsForOutput());
}

inline void writeMessageToFd(int fd, MessageBuilder& builder) {
  writeMessageToFd(fd, builder.getSegmentsForOutput());
}

155
}  // namespace capnp
Kenton Varda's avatar
Kenton Varda committed
156 157

#endif  // SERIALIZE_H_