serialize.h 10.5 KB
Newer Older
Kenton Varda's avatar
Kenton Varda committed
1 2
// Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors
// Licensed under the MIT License:
Kenton Varda's avatar
Kenton Varda committed
3
//
Kenton Varda's avatar
Kenton Varda committed
4 5 6 7 8 9
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
Kenton Varda's avatar
Kenton Varda committed
10
//
Kenton Varda's avatar
Kenton Varda committed
11 12
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
Kenton Varda's avatar
Kenton Varda committed
13
//
Kenton Varda's avatar
Kenton Varda committed
14 15 16 17 18 19 20
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
Kenton Varda's avatar
Kenton Varda committed
21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40

// This file implements a simple serialization format for Cap'n Proto messages.  The format
// is as follows:
//
// * 32-bit little-endian segment count (4 bytes).
// * 32-bit little-endian size of each segment (4*(segment count) bytes).
// * Padding so that subsequent data is 64-bit-aligned (0 or 4 bytes).  (I.e., if there are an even
//     number of segments, there are 4 bytes of zeros here, otherwise there is no padding.)
// * Data from each segment, in order (8*sum(segment sizes) bytes)
//
// This format has some important properties:
// - It is self-delimiting, so multiple messages may be written to a stream without any external
//   delimiter.
// - The total size and position of each segment can be determined by reading only the first part
//   of the message, allowing lazy and random-access reading of the segment data.
// - A message is always at least 8 bytes.
// - A single-segment message can be read entirely in two system calls with no buffering.
// - A multi-segment message can be read entirely in three system calls with no buffering.
// - The format is appropriate for mmap()ing since all data is aligned.

Kenton Varda's avatar
Kenton Varda committed
41 42
#ifndef CAPNP_SERIALIZE_H_
#define CAPNP_SERIALIZE_H_
Kenton Varda's avatar
Kenton Varda committed
43

44 45 46 47
#if defined(__GNUC__) && !CAPNP_HEADER_WARNINGS
#pragma GCC system_header
#endif

Kenton Varda's avatar
Kenton Varda committed
48
#include "message.h"
49
#include <kj/io.h>
Kenton Varda's avatar
Kenton Varda committed
50

51
namespace capnp {
Kenton Varda's avatar
Kenton Varda committed
52 53 54 55 56 57

class FlatArrayMessageReader: public MessageReader {
  // Parses a message from a flat array.  Note that it makes sense to use this together with mmap()
  // for extremely fast parsing.

public:
58
  FlatArrayMessageReader(kj::ArrayPtr<const word> array, ReaderOptions options = ReaderOptions());
Kenton Varda's avatar
Kenton Varda committed
59 60
  // The array must remain valid until the MessageReader is destroyed.

61
  kj::ArrayPtr<const word> getSegment(uint id) override;
Kenton Varda's avatar
Kenton Varda committed
62

63 64 65 66 67 68
  const word* getEnd() const { return end; }
  // Get a pointer just past the end of the message as determined by reading the message header.
  // This could actually be before the end of the input array.  This pointer is useful e.g. if
  // you know that the input array has extra stuff appended after the message and you want to
  // get at it.

Kenton Varda's avatar
Kenton Varda committed
69 70
private:
  // Optimize for single-segment case.
71 72
  kj::ArrayPtr<const word> segment0;
  kj::Array<kj::ArrayPtr<const word>> moreSegments;
73
  const word* end;
Kenton Varda's avatar
Kenton Varda committed
74 75
};

76 77 78 79 80 81 82 83 84 85 86 87 88 89 90
kj::ArrayPtr<const word> initMessageBuilderFromFlatArrayCopy(
    kj::ArrayPtr<const word> array, MessageBuilder& target,
    ReaderOptions options = ReaderOptions());
// Convenience function which reads a message using `FlatArrayMessageReader` then copies the
// content into the target `MessageBuilder`, verifying that the message structure is valid
// (although not necessarily that it matches the desired schema).
//
// Returns an ArrayPtr containing any words left over in the array after consuming the whole
// message. This is useful when reading multiple messages that have been concatenated. See also
// FlatArrayMessageReader::getEnd().
//
// (Note that it's also possible to initialize a `MessageBuilder` directly without a copy using one
// of `MessageBuilder`'s constructors. However, this approach skips the validation step and is not
// safe to use on untrusted input. Therefore, we do not provide a convenience method for it.)

91
kj::Array<word> messageToFlatArray(MessageBuilder& builder);
Kenton Varda's avatar
Kenton Varda committed
92
// Constructs a flat array containing the entire content of the given message.
93
//
Kenton Varda's avatar
Kenton Varda committed
94 95
// To output the message as bytes, use `.asBytes()` on the returned word array. Keep in mind that
// `asBytes()` returns an ArrayPtr, so you have to save the Array as well to prevent it from being
96 97 98
// deleted. For example:
//
//     kj::Array<capnp::word> words = messageToFlatArray(myMessage);
Kenton Varda's avatar
Kenton Varda committed
99
//     kj::ArrayPtr<kj::byte> bytes = words.asBytes();
100
//     write(fd, bytes.begin(), bytes.size());
Kenton Varda's avatar
Kenton Varda committed
101

102
kj::Array<word> messageToFlatArray(kj::ArrayPtr<const kj::ArrayPtr<const word>> segments);
Kenton Varda's avatar
Kenton Varda committed
103 104
// Version of messageToFlatArray that takes a raw segment array.

105 106 107 108 109 110
size_t computeSerializedSizeInWords(MessageBuilder& builder);
// Returns the size, in words, that will be needed to serialize the message, including the header.

size_t computeSerializedSizeInWords(kj::ArrayPtr<const kj::ArrayPtr<const word>> segments);
// Version of computeSerializedSizeInWords that takes a raw segment array.

Kenton Varda's avatar
Kenton Varda committed
111 112 113
// =======================================================================================

class InputStreamMessageReader: public MessageReader {
114 115 116
  // A MessageReader that reads from an abstract kj::InputStream. See also StreamFdMessageReader
  // for a subclass specific to file descriptors.

Kenton Varda's avatar
Kenton Varda committed
117
public:
118
  InputStreamMessageReader(kj::InputStream& inputStream,
Kenton Varda's avatar
Kenton Varda committed
119
                           ReaderOptions options = ReaderOptions(),
120
                           kj::ArrayPtr<word> scratchSpace = nullptr);
121
  ~InputStreamMessageReader() noexcept(false);
122

Kenton Varda's avatar
Kenton Varda committed
123
  // implements MessageReader ----------------------------------------
124
  kj::ArrayPtr<const word> getSegment(uint id) override;
Kenton Varda's avatar
Kenton Varda committed
125 126

private:
127
  kj::InputStream& inputStream;
128
  byte* readPos;
Kenton Varda's avatar
Kenton Varda committed
129 130

  // Optimize for single-segment case.
131 132
  kj::ArrayPtr<const word> segment0;
  kj::Array<kj::ArrayPtr<const word>> moreSegments;
Kenton Varda's avatar
Kenton Varda committed
133

134
  kj::Array<word> ownedSpace;
135
  // Only if scratchSpace wasn't big enough.
136 137

  kj::UnwindDetector unwindDetector;
Kenton Varda's avatar
Kenton Varda committed
138 139
};

140 141 142 143 144 145 146 147 148 149 150
void readMessageCopy(kj::InputStream& input, MessageBuilder& target,
                     ReaderOptions options = ReaderOptions(),
                     kj::ArrayPtr<word> scratchSpace = nullptr);
// Convenience function which reads a message using `InputStreamMessageReader` then copies the
// content into the target `MessageBuilder`, verifying that the message structure is valid
// (although not necessarily that it matches the desired schema).
//
// (Note that it's also possible to initialize a `MessageBuilder` directly without a copy using one
// of `MessageBuilder`'s constructors. However, this approach skips the validation step and is not
// safe to use on untrusted input. Therefore, we do not provide a convenience method for it.)

151
void writeMessage(kj::OutputStream& output, MessageBuilder& builder);
Kenton Varda's avatar
Kenton Varda committed
152 153
// Write the message to the given output stream.

154
void writeMessage(kj::OutputStream& output, kj::ArrayPtr<const kj::ArrayPtr<const word>> segments);
Kenton Varda's avatar
Kenton Varda committed
155 156 157 158 159
// Write the segment array to the given output stream.

// =======================================================================================
// Specializations for reading from / writing to file descriptors.

160
class StreamFdMessageReader: private kj::FdInputStream, public InputStreamMessageReader {
161
  // A MessageReader that reads from a steam-based file descriptor.
Kenton Varda's avatar
Kenton Varda committed
162 163 164

public:
  StreamFdMessageReader(int fd, ReaderOptions options = ReaderOptions(),
165
                        kj::ArrayPtr<word> scratchSpace = nullptr)
166
      : FdInputStream(fd), InputStreamMessageReader(*this, options, scratchSpace) {}
Kenton Varda's avatar
Kenton Varda committed
167 168
  // Read message from a file descriptor, without taking ownership of the descriptor.

169
  StreamFdMessageReader(kj::AutoCloseFd fd, ReaderOptions options = ReaderOptions(),
170
                        kj::ArrayPtr<word> scratchSpace = nullptr)
Kenton Varda's avatar
Kenton Varda committed
171
      : FdInputStream(kj::mv(fd)), InputStreamMessageReader(*this, options, scratchSpace) {}
Kenton Varda's avatar
Kenton Varda committed
172 173
  // Read a message from a file descriptor, taking ownership of the descriptor.

174
  ~StreamFdMessageReader() noexcept(false);
Kenton Varda's avatar
Kenton Varda committed
175 176
};

177 178 179 180 181 182 183 184 185 186 187
void readMessageCopyFromFd(int fd, MessageBuilder& target,
                           ReaderOptions options = ReaderOptions(),
                           kj::ArrayPtr<word> scratchSpace = nullptr);
// Convenience function which reads a message using `StreamFdMessageReader` then copies the
// content into the target `MessageBuilder`, verifying that the message structure is valid
// (although not necessarily that it matches the desired schema).
//
// (Note that it's also possible to initialize a `MessageBuilder` directly without a copy using one
// of `MessageBuilder`'s constructors. However, this approach skips the validation step and is not
// safe to use on untrusted input. Therefore, we do not provide a convenience method for it.)

Kenton Varda's avatar
Kenton Varda committed
188 189 190 191 192 193 194
void writeMessageToFd(int fd, MessageBuilder& builder);
// Write the message to the given file descriptor.
//
// This function throws an exception on any I/O error.  If your code is not exception-safe, be sure
// you catch this exception at the call site.  If throwing an exception is not acceptable, you
// can implement your own OutputStream with arbitrary error handling and then use writeMessage().

195
void writeMessageToFd(int fd, kj::ArrayPtr<const kj::ArrayPtr<const word>> segments);
Kenton Varda's avatar
Kenton Varda committed
196 197 198 199 200 201 202 203 204
// Write the segment array to the given file descriptor.
//
// This function throws an exception on any I/O error.  If your code is not exception-safe, be sure
// you catch this exception at the call site.  If throwing an exception is not acceptable, you
// can implement your own OutputStream with arbitrary error handling and then use writeMessage().

// =======================================================================================
// inline stuff

205
inline kj::Array<word> messageToFlatArray(MessageBuilder& builder) {
Kenton Varda's avatar
Kenton Varda committed
206 207 208
  return messageToFlatArray(builder.getSegmentsForOutput());
}

209 210 211 212
inline size_t computeSerializedSizeInWords(MessageBuilder& builder) {
  return computeSerializedSizeInWords(builder.getSegmentsForOutput());
}

213
inline void writeMessage(kj::OutputStream& output, MessageBuilder& builder) {
Kenton Varda's avatar
Kenton Varda committed
214 215 216 217 218 219 220
  writeMessage(output, builder.getSegmentsForOutput());
}

inline void writeMessageToFd(int fd, MessageBuilder& builder) {
  writeMessageToFd(fd, builder.getSegmentsForOutput());
}

221
}  // namespace capnp
Kenton Varda's avatar
Kenton Varda committed
222 223

#endif  // SERIALIZE_H_