Add new `capnp convert` command with JSON support.

This supersedes the `capnp encode` and `capnp decode` commands. It didn't make sense to add JSON to those commands since it was unclear if JSON should be thought of as the "encoded" or "decoded" format. `convert` allows mapping anything to anything.

This command is also useful for, say, converting unpacked format to packed format or vice versa, which can now be done without a schema.

c++/Makefile.am

@@ -324,7 +324,7 @@ libcapnpc_la_SOURCES=                                          \
 
 bin_PROGRAMS = capnp capnpc-capnp capnpc-c++
 
-capnp_LDADD = libcapnpc.la libcapnp.la libkj.la $(PTHREAD_LIBS)
+capnp_LDADD = libcapnpc.la libcapnp-json.la libcapnp.la libkj.la $(PTHREAD_LIBS)
 capnp_SOURCES =                                                \
   src/capnp/compiler/module-loader.h                           \
   src/capnp/compiler/module-loader.c++                         \

c++/src/capnp/CMakeLists.txt

@@ -156,7 +156,7 @@ if(NOT CAPNP_LITE)
     compiler/module-loader.c++
     compiler/capnp.c++
   )
-  target_link_libraries(capnp_tool capnpc capnp kj)
+  target_link_libraries(capnp_tool capnpc capnp-json capnp kj)
   set_target_properties(capnp_tool PROPERTIES OUTPUT_NAME capnp)
   set_target_properties(capnp_tool PROPERTIES CAPNP_INCLUDE_DIRECTORY
     $<JOIN:$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>,$<INSTALL_INTERFACE:${CMAKE_INSTALL_BINDIR}/..>>

c++/src/capnp/any.h

@@ -461,6 +461,8 @@ public:
   inline Reader(T&& value)
       : _reader(_::PointerHelpers<FromReader<T>>::getInternalReader(kj::fwd<T>(value))) {}
 
+  inline MessageSize totalSize() const { return _reader.totalSize().asPublic(); }
+
   kj::ArrayPtr<const byte> getDataSection() {
     return _reader.getDataSectionAsBlob();
   }
@@ -483,6 +485,11 @@ public:
     // T must be a struct type.
     return typename T::Reader(_reader);
   }
+
+  template <typename T>
+  ReaderFor<T> as(StructSchema schema) const;
+  // T must be DynamicStruct. Defined in dynamic.h.
+
 private:
   _::StructReader _reader;
 
@@ -529,6 +536,11 @@ public:
     // T must be a struct type.
     return typename T::Builder(_builder);
   }
+
+  template <typename T>
+  BuilderFor<T> as(StructSchema schema);
+  // T must be DynamicStruct. Defined in dynamic.h.
+
 private:
   _::StructBuilder _builder;
   friend class Orphanage;

c++/src/capnp/compat/json.h

@@ -224,10 +224,14 @@ private:
 // =======================================================================================
 // inline implementation details
 
+template <bool isDynamic>
+struct EncodeImpl;
+
 template <typename T>
 kj::String JsonCodec::encode(T&& value) {
+  Type type = Type::from(value);
   typedef FromAny<kj::Decay<T>> Base;
-  return encode(DynamicValue::Reader(ReaderFor<Base>(kj::fwd<T>(value))), Type::from<Base>());
+  return encode(DynamicValue::Reader(ReaderFor<Base>(kj::fwd<T>(value))), type);
 }
 
 template <typename T>

c++/src/capnp/compiler/capnp-test.sh

@@ -40,6 +40,36 @@ fi
 SCHEMA=`dirname "$0"`/../test.capnp
 TESTDATA=`dirname "$0"`/../testdata
 
+# ========================================================================================
+# convert
+
+$CAPNP convert text:binary $SCHEMA TestAllTypes < $TESTDATA/short.txt | cmp $TESTDATA/binary - || fail encode
+$CAPNP convert text:flat $SCHEMA TestAllTypes < $TESTDATA/short.txt | cmp $TESTDATA/flat - || fail encode flat
+$CAPNP convert text:packed $SCHEMA TestAllTypes < $TESTDATA/short.txt | cmp $TESTDATA/packed - || fail encode packed
+$CAPNP convert text:flat-packed $SCHEMA TestAllTypes < $TESTDATA/short.txt | cmp $TESTDATA/packedflat - || fail encode packedflat
+$CAPNP convert text:binary $SCHEMA TestAllTypes < $TESTDATA/pretty.txt | cmp $TESTDATA/binary - || fail parse pretty
+
+$CAPNP convert binary:text $SCHEMA TestAllTypes < $TESTDATA/binary | cmp $TESTDATA/pretty.txt - || fail decode
+$CAPNP convert flat:text $SCHEMA TestAllTypes < $TESTDATA/flat | cmp $TESTDATA/pretty.txt - || fail decode flat
+$CAPNP convert packed:text $SCHEMA TestAllTypes < $TESTDATA/packed | cmp $TESTDATA/pretty.txt - || fail decode packed
+$CAPNP convert flat-packed:text $SCHEMA TestAllTypes < $TESTDATA/packedflat | cmp $TESTDATA/pretty.txt - || fail decode packedflat
+$CAPNP convert binary:text --short $SCHEMA TestAllTypes < $TESTDATA/binary | cmp $TESTDATA/short.txt - || fail decode short
+
+$CAPNP convert binary:text $SCHEMA TestAllTypes < $TESTDATA/segmented | cmp $TESTDATA/pretty.txt - || fail decode segmented
+$CAPNP convert packed:text $SCHEMA TestAllTypes < $TESTDATA/segmented-packed | cmp $TESTDATA/pretty.txt - || fail decode segmented-packed
+
+$CAPNP convert binary:packed < $TESTDATA/binary | cmp $TESTDATA/packed - || fail binary to packed
+$CAPNP convert packed:binary < $TESTDATA/packed | cmp $TESTDATA/binary - || fail packed to binary
+
+$CAPNP convert binary:json $SCHEMA TestAllTypes < $TESTDATA/binary | cmp $TESTDATA/pretty.json - || fail binary to json
+$CAPNP convert binary:json --short $SCHEMA TestAllTypes < $TESTDATA/binary | cmp $TESTDATA/short.json - || fail binary to short json
+
+$CAPNP convert json:binary $SCHEMA TestAllTypes < $TESTDATA/pretty.json | cmp $TESTDATA/binary - || fail json to binary
+$CAPNP convert json:binary $SCHEMA TestAllTypes < $TESTDATA/short.json | cmp $TESTDATA/binary - || fail short json to binary
+
+# ========================================================================================
+# DEPRECATED encode/decode
+
 $CAPNP encode $SCHEMA TestAllTypes < $TESTDATA/short.txt | cmp $TESTDATA/binary - || fail encode
 $CAPNP encode --flat $SCHEMA TestAllTypes < $TESTDATA/short.txt | cmp $TESTDATA/flat - || fail encode flat
 $CAPNP encode --packed $SCHEMA TestAllTypes < $TESTDATA/short.txt | cmp $TESTDATA/packed - || fail encode packed
@@ -55,6 +85,9 @@ $CAPNP decode --short $SCHEMA TestAllTypes < $TESTDATA/binary | cmp $TESTDATA/sh
 $CAPNP decode $SCHEMA TestAllTypes < $TESTDATA/segmented | cmp $TESTDATA/pretty.txt - || fail decode segmented
 $CAPNP decode --packed $SCHEMA TestAllTypes < $TESTDATA/segmented-packed | cmp $TESTDATA/pretty.txt - || fail decode segmented-packed
 
+# ========================================================================================
+# eval
+
 test_eval() {
   test "x`$CAPNP eval $SCHEMA $1 | tr -d '\r'`" = "x$2" || fail eval "$1 == $2"
 }
@@ -67,5 +100,7 @@ test_eval globalPrintableStruct '(someText = "foo")'
 test_eval TestConstants.enumConst corge
 test_eval 'TestListDefaults.lists.int32ListList[2][0]' 12341234
 
+test "x`$CAPNP eval $SCHEMA -ojson globalPrintableStruct | tr -d '\r'`" = "x{\"someText\": \"foo\"}" || fail eval json "globalPrintableStruct == {someText = \"foo\"}"
+
 $CAPNP compile -ofoo $TESTDATA/errors.capnp.nobuild 2>&1 | sed -e "s,^.*/errors[.]capnp[.]nobuild,file,g" | tr -d '\r' |
     cmp $TESTDATA/errors.txt - || fail error output

c++/src/capnp/dynamic.h

@@ -179,6 +179,8 @@ public:
   template <typename T, typename = kj::EnableIf<kind<FromReader<T>>() == Kind::STRUCT>>
   inline Reader(T&& value): Reader(toDynamic(value)) {}
 
+  inline operator AnyStruct::Reader() const { return AnyStruct::Reader(reader); }
+
   inline MessageSize totalSize() const { return reader.totalSize().asPublic(); }
 
   template <typename T>
@@ -234,6 +236,7 @@ private:
   friend class Orphan<DynamicStruct>;
   friend class Orphan<DynamicValue>;
   friend class Orphan<AnyPointer>;
+  friend class AnyStruct::Reader;
 };
 
 class DynamicStruct::Builder {
@@ -246,6 +249,8 @@ public:
   template <typename T, typename = kj::EnableIf<kind<FromBuilder<T>>() == Kind::STRUCT>>
   inline Builder(T&& value): Builder(toDynamic(value)) {}
 
+  inline operator AnyStruct::Reader() { return AnyStruct::Builder(builder); }
+
   inline MessageSize totalSize() const { return asReader().totalSize(); }
 
   template <typename T>
@@ -323,6 +328,7 @@ private:
   friend class Orphan<DynamicStruct>;
   friend class Orphan<DynamicValue>;
   friend class Orphan<AnyPointer>;
+  friend class AnyStruct::Builder;
 };
 
 class DynamicStruct::Pipeline {
@@ -364,6 +370,8 @@ public:
   template <typename T, typename = kj::EnableIf<kind<FromReader<T>>() == Kind::LIST>>
   inline Reader(T&& value): Reader(toDynamic(value)) {}
 
+  inline operator AnyList::Reader() const { return AnyList::Reader(reader); }
+
   template <typename T>
   typename T::Reader as() const;
   // Try to convert to any List<T>, Data, or Text.  Throws an exception if the underlying data
@@ -407,6 +415,8 @@ public:
   template <typename T, typename = kj::EnableIf<kind<FromBuilder<T>>() == Kind::LIST>>
   inline Builder(T&& value): Builder(toDynamic(value)) {}
 
+  inline operator AnyList::Builder() { return AnyList::Builder(builder); }
+
   template <typename T>
   typename T::Builder as();
   // Try to convert to any List<T>, Data, or Text.  Throws an exception if the underlying data
@@ -1516,6 +1526,16 @@ inline AnyStruct::Builder DynamicStruct::Builder::as<AnyStruct>() {
   return AnyStruct::Builder(builder);
 }
 
+template <>
+inline DynamicStruct::Reader AnyStruct::Reader::as<DynamicStruct>(StructSchema schema) const {
+  return DynamicStruct::Reader(schema, _reader);
+}
+
+template <>
+inline DynamicStruct::Builder AnyStruct::Builder::as<DynamicStruct>(StructSchema schema) {
+  return DynamicStruct::Builder(schema, _builder);
+}
+
 template <typename T>
 typename T::Pipeline DynamicStruct::Pipeline::releaseAs() {
   static_assert(kind<T>() == Kind::STRUCT,

c++/src/capnp/schema.h

@@ -599,6 +599,8 @@ public:
 
   template <typename T>
   inline static Type from();
+  template <typename T>
+  inline static Type from(T&& value);
 
   inline schema::Type::Which which() const;
 
@@ -680,6 +682,9 @@ private:
 
   void requireUsableAs(Type expected) const;
 
+  template <typename T, Kind k>
+  struct FromValueImpl;
+
   friend class ListSchema;  // only for requireUsableAs()
 };
 
@@ -899,6 +904,29 @@ inline schema::Type::AnyPointer::Unconstrained::Which Type::whichAnyPointerKind(
 template <typename T>
 inline Type Type::from() { return Type(Schema::from<T>()); }
 
+template <typename T, Kind k>
+struct Type::FromValueImpl {
+  template <typename U>
+  static inline Type type(U&& value) {
+    return Type::from<T>();
+  }
+};
+
+template <typename T>
+struct Type::FromValueImpl<T, Kind::OTHER> {
+  template <typename U>
+  static inline Type type(U&& value) {
+    // All dynamic types have getSchema().
+    return value.getSchema();
+  }
+};
+
+template <typename T>
+inline Type Type::from(T&& value) {
+  typedef FromAny<kj::Decay<T>> Base;
+  return Type::FromValueImpl<Base, kind<Base>()>::type(kj::fwd<T>(value));
+}
+
 inline bool Type::isVoid   () const { return baseType == schema::Type::VOID     && listDepth == 0; }
 inline bool Type::isBool   () const { return baseType == schema::Type::BOOL     && listDepth == 0; }
 inline bool Type::isInt8   () const { return baseType == schema::Type::INT8     && listDepth == 0; }

c++/src/capnp/testdata/pretty.json

+{ "voidField": null,
+  "boolField": true,
+  "int8Field": -123,
+  "int16Field": -12345,
+  "int32Field": -12345678,
+  "int64Field": "-123456789012345",
+  "uInt8Field": 234,
+  "uInt16Field": 45678,
+  "uInt32Field": 3456789012,
+  "uInt64Field": "12345678901234567890",
+  "float32Field": 1234.5,
+  "float64Field": -1.23e47,
+  "textField": "foo",
+  "dataField": [98, 97, 114],
+  "structField": {
+    "voidField": null,
+    "boolField": true,
+    "int8Field": -12,
+    "int16Field": 3456,
+    "int32Field": -78901234,
+    "int64Field": "56789012345678",
+    "uInt8Field": 90,
+    "uInt16Field": 1234,
+    "uInt32Field": 56789012,
+    "uInt64Field": "345678901234567890",
+    "float32Field": -1.2499999646475857e-10,
+    "float64Field": 345,
+    "textField": "baz",
+    "dataField": [113, 117, 120],
+    "structField": {
+      "voidField": null,
+      "boolField": false,
+      "int8Field": 0,
+      "int16Field": 0,
+      "int32Field": 0,
+      "int64Field": "0",
+      "uInt8Field": 0,
+      "uInt16Field": 0,
+      "uInt32Field": 0,
+      "uInt64Field": "0",
+      "float32Field": 0,
+      "float64Field": 0,
+      "textField": "nested",
+      "structField": {"voidField": null, "boolField": false, "int8Field": 0, "int16Field": 0, "int32Field": 0, "int64Field": "0", "uInt8Field": 0, "uInt16Field": 0, "uInt32Field": 0, "uInt64Field": "0", "float32Field": 0, "float64Field": 0, "textField": "really nested", "enumField": "foo", "interfaceField": null},
+      "enumField": "foo",
+      "interfaceField": null },
+    "enumField": "baz",
+    "interfaceField": null,
+    "voidList": [null, null, null],
+    "boolList": [false, true, false, true, true],
+    "int8List": [12, -34, -128, 127],
+    "int16List": [1234, -5678, -32768, 32767],
+    "int32List": [12345678, -90123456, -2147483648, 2147483647],
+    "int64List": ["123456789012345", "-678901234567890", "-9223372036854775808", "9223372036854775807"],
+    "uInt8List": [12, 34, 0, 255],
+    "uInt16List": [1234, 5678, 0, 65535],
+    "uInt32List": [12345678, 90123456, 0, 4294967295],
+    "uInt64List": ["123456789012345", "678901234567890", "0", "18446744073709551615"],
+    "float32List": [0, 1234567, 9.9999999338158125e36, -9.9999999338158125e36, 9.99999991097579e-38, -9.99999991097579e-38],
+    "float64List": [0, 123456789012345, 1e306, -1e306, 1e-306, -1e-306],
+    "textList": ["quux", "corge", "grault"],
+    "dataList": [[103, 97, 114, 112, 108, 121], [119, 97, 108, 100, 111], [102, 114, 101, 100]],
+    "structList": [
+      {"voidField": null, "boolField": false, "int8Field": 0, "int16Field": 0, "int32Field": 0, "int64Field": "0", "uInt8Field": 0, "uInt16Field": 0, "uInt32Field": 0, "uInt64Field": "0", "float32Field": 0, "float64Field": 0, "textField": "x structlist 1", "enumField": "foo", "interfaceField": null},
+      {"voidField": null, "boolField": false, "int8Field": 0, "int16Field": 0, "int32Field": 0, "int64Field": "0", "uInt8Field": 0, "uInt16Field": 0, "uInt32Field": 0, "uInt64Field": "0", "float32Field": 0, "float64Field": 0, "textField": "x structlist 2", "enumField": "foo", "interfaceField": null},
+      {"voidField": null, "boolField": false, "int8Field": 0, "int16Field": 0, "int32Field": 0, "int64Field": "0", "uInt8Field": 0, "uInt16Field": 0, "uInt32Field": 0, "uInt64Field": "0", "float32Field": 0, "float64Field": 0, "textField": "x structlist 3", "enumField": "foo", "interfaceField": null} ],
+    "enumList": ["qux", "bar", "grault"] },
+  "enumField": "corge",
+  "interfaceField": null,
+  "voidList": [null, null, null, null, null, null],
+  "boolList": [true, false, false, true],
+  "int8List": [111, -111],
+  "int16List": [11111, -11111],
+  "int32List": [111111111, -111111111],
+  "int64List": ["1111111111111111111", "-1111111111111111111"],
+  "uInt8List": [111, 222],
+  "uInt16List": [33333, 44444],
+  "uInt32List": [3333333333],
+  "uInt64List": ["11111111111111111111"],
+  "float32List": [5555.5, "Infinity", "-Infinity", "NaN"],
+  "float64List": [7777.75, "Infinity", "-Infinity", "NaN"],
+  "textList": ["plugh", "xyzzy", "thud"],
+  "dataList": [[111, 111, 112, 115], [101, 120, 104, 97, 117, 115, 116, 101, 100], [114, 102, 99, 51, 48, 57, 50]],
+  "structList": [
+    {"voidField": null, "boolField": false, "int8Field": 0, "int16Field": 0, "int32Field": 0, "int64Field": "0", "uInt8Field": 0, "uInt16Field": 0, "uInt32Field": 0, "uInt64Field": "0", "float32Field": 0, "float64Field": 0, "textField": "structlist 1", "enumField": "foo", "interfaceField": null},
+    {"voidField": null, "boolField": false, "int8Field": 0, "int16Field": 0, "int32Field": 0, "int64Field": "0", "uInt8Field": 0, "uInt16Field": 0, "uInt32Field": 0, "uInt64Field": "0", "float32Field": 0, "float64Field": 0, "textField": "structlist 2", "enumField": "foo", "interfaceField": null},
+    {"voidField": null, "boolField": false, "int8Field": 0, "int16Field": 0, "int32Field": 0, "int64Field": "0", "uInt8Field": 0, "uInt16Field": 0, "uInt32Field": 0, "uInt64Field": "0", "float32Field": 0, "float64Field": 0, "textField": "structlist 3", "enumField": "foo", "interfaceField": null} ],
+  "enumList": ["foo", "garply"] }

c++/src/capnp/testdata/short.json

+{"voidField":null,"boolField":true,"int8Field":-123,"int16Field":-12345,"int32Field":-12345678,"int64Field":"-123456789012345","uInt8Field":234,"uInt16Field":45678,"uInt32Field":3456789012,"uInt64Field":"12345678901234567890","float32Field":1234.5,"float64Field":-1.23e47,"textField":"foo","dataField":[98,97,114],"structField":{"voidField":null,"boolField":true,"int8Field":-12,"int16Field":3456,"int32Field":-78901234,"int64Field":"56789012345678","uInt8Field":90,"uInt16Field":1234,"uInt32Field":56789012,"uInt64Field":"345678901234567890","float32Field":-1.2499999646475857e-10,"float64Field":345,"textField":"baz","dataField":[113,117,120],"structField":{"voidField":null,"boolField":false,"int8Field":0,"int16Field":0,"int32Field":0,"int64Field":"0","uInt8Field":0,"uInt16Field":0,"uInt32Field":0,"uInt64Field":"0","float32Field":0,"float64Field":0,"textField":"nested","structField":{"voidField":null,"boolField":false,"int8Field":0,"int16Field":0,"int32Field":0,"int64Field":"0","uInt8Field":0,"uInt16Field":0,"uInt32Field":0,"uInt64Field":"0","float32Field":0,"float64Field":0,"textField":"really nested","enumField":"foo","interfaceField":null},"enumField":"foo","interfaceField":null},"enumField":"baz","interfaceField":null,"voidList":[null,null,null],"boolList":[false,true,false,true,true],"int8List":[12,-34,-128,127],"int16List":[1234,-5678,-32768,32767],"int32List":[12345678,-90123456,-2147483648,2147483647],"int64List":["123456789012345","-678901234567890","-9223372036854775808","9223372036854775807"],"uInt8List":[12,34,0,255],"uInt16List":[1234,5678,0,65535],"uInt32List":[12345678,90123456,0,4294967295],"uInt64List":["123456789012345","678901234567890","0","18446744073709551615"],"float32List":[0,1234567,9.9999999338158125e36,-9.9999999338158125e36,9.99999991097579e-38,-9.99999991097579e-38],"float64List":[0,123456789012345,1e306,-1e306,1e-306,-1e-306],"textList":["quux","corge","grault"],"dataList":[[103,97,114,112,108,121],[119,97,108,100,111],[102,114,101,100]],"structList":[{"voidField":null,"boolField":false,"int8Field":0,"int16Field":0,"int32Field":0,"int64Field":"0","uInt8Field":0,"uInt16Field":0,"uInt32Field":0,"uInt64Field":"0","float32Field":0,"float64Field":0,"textField":"x structlist 1","enumField":"foo","interfaceField":null},{"voidField":null,"boolField":false,"int8Field":0,"int16Field":0,"int32Field":0,"int64Field":"0","uInt8Field":0,"uInt16Field":0,"uInt32Field":0,"uInt64Field":"0","float32Field":0,"float64Field":0,"textField":"x structlist 2","enumField":"foo","interfaceField":null},{"voidField":null,"boolField":false,"int8Field":0,"int16Field":0,"int32Field":0,"int64Field":"0","uInt8Field":0,"uInt16Field":0,"uInt32Field":0,"uInt64Field":"0","float32Field":0,"float64Field":0,"textField":"x structlist 3","enumField":"foo","interfaceField":null}],"enumList":["qux","bar","grault"]},"enumField":"corge","interfaceField":null,"voidList":[null,null,null,null,null,null],"boolList":[true,false,false,true],"int8List":[111,-111],"int16List":[11111,-11111],"int32List":[111111111,-111111111],"int64List":["1111111111111111111","-1111111111111111111"],"uInt8List":[111,222],"uInt16List":[33333,44444],"uInt32List":[3333333333],"uInt64List":["11111111111111111111"],"float32List":[5555.5,"Infinity","-Infinity","NaN"],"float64List":[7777.75,"Infinity","-Infinity","NaN"],"textList":["plugh","xyzzy","thud"],"dataList":[[111,111,112,115],[101,120,104,97,117,115,116,101,100],[114,102,99,51,48,57,50]],"structList":[{"voidField":null,"boolField":false,"int8Field":0,"int16Field":0,"int32Field":0,"int64Field":"0","uInt8Field":0,"uInt16Field":0,"uInt32Field":0,"uInt64Field":"0","float32Field":0,"float64Field":0,"textField":"structlist 1","enumField":"foo","interfaceField":null},{"voidField":null,"boolField":false,"int8Field":0,"int16Field":0,"int32Field":0,"int64Field":"0","uInt8Field":0,"uInt16Field":0,"uInt32Field":0,"uInt64Field":"0","float32Field":0,"float64Field":0,"textField":"structlist 2","enumField":"foo","interfaceField":null},{"voidField":null,"boolField":false,"int8Field":0,"int16Field":0,"int32Field":0,"int64Field":"0","uInt8Field":0,"uInt16Field":0,"uInt32Field":0,"uInt64Field":"0","float32Field":0,"float64Field":0,"textField":"structlist 3","enumField":"foo","interfaceField":null}],"enumList":["foo","garply"]}