Merge pull request #247 from pqu/serialize-text

Add TextCodec to expose an API for the schema's text format.

c++/Makefile.am

@@ -172,6 +172,7 @@ includecapnp_HEADERS =                                         \
   src/capnp/serialize.h                                        \
   src/capnp/serialize-async.h                                  \
   src/capnp/serialize-packed.h                                 \
+  src/capnp/serialize-text.h                                   \
   src/capnp/pointer-helpers.h                                  \
   src/capnp/generated-header-support.h                         \
   src/capnp/rpc-prelude.h                                      \
@@ -284,7 +285,8 @@ libcapnpc_la_SOURCES=                                          \
   src/capnp/compiler/node-translator.c++                       \
   src/capnp/compiler/compiler.h                                \
   src/capnp/compiler/compiler.c++                              \
-  src/capnp/schema-parser.c++
+  src/capnp/schema-parser.c++                                  \
+  src/capnp/serialize-text.c++
 
 bin_PROGRAMS = capnp capnpc-capnp capnpc-c++
 
@@ -382,6 +384,7 @@ heavy_tests =                                                  \
   src/capnp/dynamic-test.c++                                   \
   src/capnp/stringify-test.c++                                 \
   src/capnp/serialize-async-test.c++                           \
+  src/capnp/serialize-text-test.c++                            \
   src/capnp/rpc-test.c++                                       \
   src/capnp/rpc-twoparty-test.c++                              \
   src/capnp/ez-rpc-test.c++                                    \

c++/src/capnp/CMakeLists.txt

@@ -47,6 +47,7 @@ set(capnp_headers
   serialize.h
   serialize-async.h
   serialize-packed.h
+  serialize-text.h
   pointer-helpers.h
   generated-header-support.h
 )
@@ -105,6 +106,7 @@ set(capnpc_sources
   compiler/node-translator.c++
   compiler/compiler.c++
   schema-parser.c++
+  serialize-text.c++
 )
 if(NOT CAPNP_LITE)
   add_library(capnpc ${capnpc_sources})
@@ -218,6 +220,7 @@ if(BUILD_TESTING)
       dynamic-test.c++
       stringify-test.c++
       serialize-async-test.c++
+      serialize-text-test.c++
       rpc-test.c++
       rpc-twoparty-test.c++
       ez-rpc-test.c++

c++/src/capnp/compiler/node-translator.h

@@ -287,6 +287,10 @@ public:
 
   kj::Maybe<Orphan<DynamicValue>> compileValue(Expression::Reader src, Type type);
 
+  void fillStructValue(DynamicStruct::Builder builder,
+                       List<Expression::Param>::Reader assignments);
+  // Interprets the given assignments and uses them to fill in the given struct builder.
+
 private:
   Resolver& resolver;
   ErrorReporter& errorReporter;
@@ -295,10 +299,6 @@ private:
   Orphan<DynamicValue> compileValueInner(Expression::Reader src, Type type);
   // Helper for compileValue().
 
-  void fillStructValue(DynamicStruct::Builder builder,
-                       List<Expression::Param>::Reader assignments);
-  // Interprets the given assignments and uses them to fill in the given struct builder.
-
   kj::String makeNodeName(Schema node);
   kj::String makeTypeName(Type type);
 

c++/src/capnp/serialize-text-test.c++

+// Copyright (c) 2015 Philip Quinn.
+// Licensed under the MIT License:
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include "serialize-text.h"
+#include <kj/compat/gtest.h>
+#include <kj/string.h>
+#include <capnp/pretty-print.h>
+#include <capnp/message.h>
+#include "test-util.h"
+
+#include <capnp/test.capnp.h>
+
+namespace capnp {
+namespace _ {  // private
+namespace {
+
+KJ_TEST("TestAllTypes") {
+  MallocMessageBuilder builder;
+  initTestMessage(builder.initRoot<TestAllTypes>());
+
+  {
+    // Plain output
+    TextCodec codec;
+    codec.setPrettyPrint(false);
+    auto text = codec.encode(builder.getRoot<TestAllTypes>());
+
+    auto stringify = kj::str(builder.getRoot<TestAllTypes>());
+    KJ_EXPECT(text == stringify);
+
+    MallocMessageBuilder reader;
+    auto orphan = codec.decode<TestAllTypes>(text, reader.getOrphanage());
+    auto structReader = orphan.getReader();
+    checkTestMessage(structReader);
+  }
+  {
+    // Pretty output
+    TextCodec codec;
+    codec.setPrettyPrint(true);
+    auto text = codec.encode(builder.getRoot<TestAllTypes>());
+
+    auto stringify = prettyPrint(builder.getRoot<TestAllTypes>()).flatten();
+    KJ_EXPECT(text == stringify);
+
+    MallocMessageBuilder reader;
+    auto orphan = codec.decode<TestAllTypes>(text, reader.getOrphanage());
+    auto structReader = orphan.getReader();
+    checkTestMessage(structReader);
+  }
+}
+
+KJ_TEST("TestDefaults") {
+  MallocMessageBuilder builder;
+  initTestMessage(builder.initRoot<TestDefaults>());
+
+  TextCodec codec;
+  auto text = codec.encode(builder.getRoot<TestDefaults>());
+
+  MallocMessageBuilder reader;
+  auto orphan = codec.decode<TestDefaults>(text, reader.getOrphanage());
+  auto structReader = orphan.getReader();
+  checkTestMessage(structReader);
+}
+
+KJ_TEST("TestListDefaults") {
+  MallocMessageBuilder builder;
+  initTestMessage(builder.initRoot<TestListDefaults>());
+
+  TextCodec codec;
+  auto text = codec.encode(builder.getRoot<TestListDefaults>());
+
+  MallocMessageBuilder reader;
+  auto orphan = codec.decode<TestListDefaults>(text, reader.getOrphanage());
+  auto structReader = orphan.getReader();
+  checkTestMessage(structReader);
+}
+
+KJ_TEST("raw text") {
+  using TestType = capnproto_test::capnp::test::TestLateUnion;
+
+  kj::String message =
+      kj::str(R"((
+        foo = -123, bar = "bar", baz = 456,
+        # Test Comment
+        theUnion = ( qux = "qux" ),
+        anotherUnion = ( corge = [ 7, 8, 9 ] ),
+      ))");
+
+  MallocMessageBuilder builder;
+  auto testType = builder.initRoot<TestType>();
+
+  TextCodec codec;
+  codec.decode(message, testType);
+
+  auto reader = testType.asReader();
+  KJ_EXPECT(reader.getFoo() == -123);
+  KJ_EXPECT(reader.getBar() == "bar");
+  KJ_EXPECT(reader.getBaz() == 456);
+
+  KJ_EXPECT(reader.getTheUnion().isQux());
+  KJ_EXPECT(reader.getTheUnion().hasQux());
+  KJ_EXPECT(reader.getTheUnion().getQux() == "qux");
+
+  KJ_EXPECT(reader.getAnotherUnion().isCorge());
+  KJ_EXPECT(reader.getAnotherUnion().hasCorge());
+  KJ_EXPECT(reader.getAnotherUnion().getCorge().size() == 3);
+  KJ_EXPECT(reader.getAnotherUnion().getCorge()[0] == 7);
+  KJ_EXPECT(reader.getAnotherUnion().getCorge()[1] == 8);
+  KJ_EXPECT(reader.getAnotherUnion().getCorge()[2] == 9);
+}
+
+}  // namespace
+}  // namespace _ (private)
+}  // namespace capnp

c++/src/capnp/serialize-text.c++

+// Copyright (c) 2015 Philip Quinn.
+// Licensed under the MIT License:
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include "serialize-text.h"
+
+#include <kj/debug.h>
+
+#include "pretty-print.h"
+#include "compiler/lexer.capnp.h"
+#include "compiler/lexer.h"
+#include "compiler/node-translator.h"
+#include "compiler/parser.h"
+
+namespace {
+
+class ThrowingErrorReporter final: public capnp::compiler::ErrorReporter {
+  // Throws all errors as assertion failures.
+public:
+  void addError(uint32_t startByte, uint32_t endByte, kj::StringPtr message) override {
+    KJ_FAIL_REQUIRE(kj::str(message, " (", startByte, ":", endByte, ")."));
+  }
+
+  bool hadErrors() override { return false; }
+};
+
+class ExternalResolver final: public capnp::compiler::ValueTranslator::Resolver {
+  // Throws all external resolution requests as assertion failures.
+public:
+  kj::Maybe<capnp::DynamicValue::Reader>
+  resolveConstant(capnp::compiler::Expression::Reader name) override {
+    KJ_FAIL_REQUIRE("External constants not allowed.");
+  }
+
+  kj::Maybe<kj::Array<const capnp::byte>>
+  readEmbed(capnp::compiler::LocatedText::Reader filename) override {
+    KJ_FAIL_REQUIRE("External embeds not allowed.");
+  }
+};
+
+template <typename Function>
+void lexAndParseExpression(kj::StringPtr input, Function f) {
+  // Parses a single expression from the input and calls `f(expression)`.
+
+  ThrowingErrorReporter errorReporter;
+
+  capnp::MallocMessageBuilder tokenArena;
+  auto lexedTokens = tokenArena.initRoot<capnp::compiler::LexedTokens>();
+  capnp::compiler::lex(input, lexedTokens, errorReporter);
+
+  capnp::compiler::CapnpParser parser(tokenArena.getOrphanage(), errorReporter);
+  auto tokens = lexedTokens.asReader().getTokens();
+  capnp::compiler::CapnpParser::ParserInput parserInput(tokens.begin(), tokens.end());
+
+  if (parserInput.getPosition() != tokens.end()) {
+    KJ_IF_MAYBE(expression, parser.getParsers().expression(parserInput)) {
+      // The input is expected to contain a *single* message.
+      KJ_REQUIRE(parserInput.getPosition() == tokens.end(), "Extra tokens in input.");
+
+      f(expression->getReader());
+    } else {
+      auto best = parserInput.getBest();
+      if (best == tokens.end()) {
+        KJ_FAIL_REQUIRE("Premature end of input.");
+      } else {
+        errorReporter.addErrorOn(*best, "Parse error");
+      }
+    }
+  } else {
+    KJ_FAIL_REQUIRE("Failed to read input.");
+  }
+}
+
+}  // namespace
+
+namespace capnp {
+
+TextCodec::TextCodec() : prettyPrint(false) {}
+TextCodec::~TextCodec() noexcept(true) {}
+
+void TextCodec::setPrettyPrint(bool enabled) { prettyPrint = enabled; }
+
+kj::String TextCodec::encode(DynamicValue::Reader value) const {
+  if (!prettyPrint) {
+    return kj::str(value);
+  } else {
+    if (value.getType() == DynamicValue::Type::STRUCT) {
+      return capnp::prettyPrint(value.as<DynamicStruct>()).flatten();
+    } else if (value.getType() == DynamicValue::Type::LIST) {
+      return capnp::prettyPrint(value.as<DynamicList>()).flatten();
+    } else {
+      return kj::str(value);
+    }
+  }
+}
+
+void TextCodec::decode(kj::StringPtr input, DynamicStruct::Builder output) const {
+  lexAndParseExpression(input, [&output](compiler::Expression::Reader expression) {
+    KJ_REQUIRE(expression.isTuple(), "Input does not contain a struct.");
+
+    ThrowingErrorReporter errorReporter;
+    ExternalResolver nullResolver;
+
+    Orphanage orphanage = Orphanage::getForMessageContaining(output);
+    compiler::ValueTranslator translator(nullResolver, errorReporter, orphanage);
+    translator.fillStructValue(output, expression.getTuple());
+  });
+}
+
+Orphan<DynamicValue> TextCodec::decode(kj::StringPtr input, Type type, Orphanage orphanage) const {
+  Orphan<DynamicValue> output;
+  
+  lexAndParseExpression(input, [&type, &orphanage, &output](compiler::Expression::Reader expression) {
+    ThrowingErrorReporter errorReporter;
+    ExternalResolver nullResolver;
+
+    compiler::ValueTranslator translator(nullResolver, errorReporter, orphanage);
+    KJ_IF_MAYBE(value, translator.compileValue(expression, type)) {
+      output = *kj::mv(value);
+    } else {
+      // An error should have already been given to the errorReporter.
+    }
+  });
+  
+  return output;
+}
+
+}  // namespace capnp

c++/src/capnp/serialize-text.h

+// Copyright (c) 2015 Philip Quinn.
+// Licensed under the MIT License:
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#ifndef CAPNP_SERIALIZE_TEXT_H_
+#define CAPNP_SERIALIZE_TEXT_H_
+
+#if defined(__GNUC__) && !CAPNP_HEADER_WARNINGS
+#pragma GCC system_header
+#endif
+
+#include <kj/string.h>
+#include "dynamic.h"
+#include "orphan.h"
+#include "schema.h"
+
+namespace capnp {
+
+class TextCodec {
+  // Reads and writes Cap'n Proto objects in a plain text format (as used in the schema
+  // language for constants, and read/written by the 'decode' and 'encode' commands of
+  // the capnp tool).
+  //
+  // This format is useful for debugging or human input, but it is not a robust alternative
+  // to the binary format. Changes to a schema's types or names that are permitted in a
+  // schema's binary evolution will likely break messages stored in this format.
+  //
+  // Note that definitions or references (to constants, other fields, or files) are not
+  // permitted in this format. To evaluate declarations with the full expressiveness of the
+  // schema language, see `capnp::SchemaParser`.
+  //
+  // Requires linking with the capnpc library.
+
+public:
+  TextCodec();
+  ~TextCodec() noexcept(true);
+
+  void setPrettyPrint(bool enabled);
+  // If enabled, pads the output of `encode()` with spaces and newlines to make it more
+  // human-readable.
+
+  template <typename T>
+  kj::String encode(T&& value) const;
+  kj::String encode(DynamicValue::Reader value) const;
+  // Encode any Cap'n Proto value.
+
+  template <typename T>
+  Orphan<T> decode(kj::StringPtr input, Orphanage orphanage) const;
+  // Decode a text message into a Cap'n Proto object of type T, allocated in the given
+  // orphanage. Any errors parsing the input or assigning the fields of T are thrown as
+  // exceptions.
+
+  void decode(kj::StringPtr input, DynamicStruct::Builder output) const;
+  // Decode a text message for a struct into the given builder. Any errors parsing the
+  // input or assigning the fields of the output are thrown as exceptions.
+
+  // TODO(someday): expose some control over the error handling?
+private:
+  Orphan<DynamicValue> decode(kj::StringPtr input, Type type, Orphanage orphanage) const;
+
+  bool prettyPrint;
+};
+
+// =======================================================================================
+// inline stuff
+
+template <typename T>
+inline kj::String TextCodec::encode(T&& value) const {
+  return encode(DynamicValue::Reader(ReaderFor<FromAny<T>>(kj::fwd<T>(value))));
+}
+
+template <typename T>
+inline Orphan<T> TextCodec::decode(kj::StringPtr input, Orphanage orphanage) const {
+  return decode(input, Type::from<T>(), orphanage).template releaseAs<T>();
+}
+
+}  // namespace capnp
+
+#endif  // CAPNP_SERIALIZE_TEXT_H_