Merge pull request #677 from RReverser/json-decode

Rewrite JSON decoding

CONTRIBUTORS

@@ -17,6 +17,7 @@ Harris Hancock <vortrab@gmail.com>: MSVC support
 Branislav Katreniak <branislav.katreniak@digitalstrom.com>: JSON decode
 Matthew Maurer <matthew.r.maurer@gmail.com>: Canonicalization Support
 David Renshaw <david@sandstorm.io>: bugfixes and miscellaneous maintenance
+Ingvar Stepanyan <me@rreverser.com> <ingvar@cloudflare.com>: Custom handlers for JSON decode
 
 This file does not list people who maintain their own Cap'n Proto
 implementations as separate projects.  Those people are awesome too!  :)

c++/src/capnp/compat/json-test.c++

@@ -185,13 +185,20 @@ KJ_TEST("encode union") {
 
 KJ_TEST("decode all types") {
   JsonCodec json;
-#define CASE(s, f) \
+  json.setHasMode(HasMode::NON_DEFAULT);
+
+#define CASE_MAYBE_ROUNDTRIP(s, f, roundtrip) \
   { \
     MallocMessageBuilder message; \
     auto root = message.initRoot<TestAllTypes>(); \
-    json.decode(s, root); \
-    KJ_EXPECT((f)) \
-  }
+    kj::StringPtr input = s; \
+    json.decode(input, root); \
+    KJ_EXPECT((f), input, root); \
+    auto reencoded = json.encode(root); \
+    KJ_EXPECT(roundtrip == (input == reencoded), roundtrip, input, reencoded); \
+  }
+#define CASE_NO_ROUNDTRIP(s, f) CASE_MAYBE_ROUNDTRIP(s, f, false)
+#define CASE(s, f) CASE_MAYBE_ROUNDTRIP(s, f, true)
 #define CASE_THROW(s, errorMessage) \
   { \
     MallocMessageBuilder message; \
@@ -206,113 +213,126 @@ KJ_TEST("decode all types") {
   }
 
   CASE(R"({})", root.getBoolField() == false);
-  CASE(R"({"unknownField":7})", root.getBoolField() == false);
+  CASE_NO_ROUNDTRIP(R"({"unknownField":7})", root.getBoolField() == false);
   CASE(R"({"boolField":true})", root.getBoolField() == true);
   CASE(R"({"int8Field":-128})", root.getInt8Field() == -128);
-  CASE(R"({"int8Field":"127"})", root.getInt8Field() == 127);
+  CASE_NO_ROUNDTRIP(R"({"int8Field":"127"})", root.getInt8Field() == 127);
   CASE_THROW_RECOVERABLE(R"({"int8Field":"-129"})", "Value out-of-range");
   CASE_THROW_RECOVERABLE(R"({"int8Field":128})", "Value out-of-range");
   CASE(R"({"int16Field":-32768})", root.getInt16Field() == -32768);
-  CASE(R"({"int16Field":"32767"})", root.getInt16Field() == 32767);
+  CASE_NO_ROUNDTRIP(R"({"int16Field":"32767"})", root.getInt16Field() == 32767);
   CASE_THROW_RECOVERABLE(R"({"int16Field":"-32769"})", "Value out-of-range");
   CASE_THROW_RECOVERABLE(R"({"int16Field":32768})", "Value out-of-range");
   CASE(R"({"int32Field":-2147483648})", root.getInt32Field() == -2147483648);
-  CASE(R"({"int32Field":"2147483647"})", root.getInt32Field() == 2147483647);
-  CASE(R"({"int64Field":-9007199254740992})", root.getInt64Field() == -9007199254740992LL);
-  CASE(R"({"int64Field":9007199254740991})", root.getInt64Field() == 9007199254740991LL);
+  CASE_NO_ROUNDTRIP(R"({"int32Field":"2147483647"})", root.getInt32Field() == 2147483647);
+  CASE_NO_ROUNDTRIP(R"({"int64Field":-9007199254740992})", root.getInt64Field() == -9007199254740992LL);
+  CASE_NO_ROUNDTRIP(R"({"int64Field":9007199254740991})", root.getInt64Field() == 9007199254740991LL);
   CASE(R"({"int64Field":"-9223372036854775808"})", root.getInt64Field() == -9223372036854775808ULL);
   CASE(R"({"int64Field":"9223372036854775807"})", root.getInt64Field() == 9223372036854775807LL);
   CASE_THROW_RECOVERABLE(R"({"int64Field":"-9223372036854775809"})", "Value out-of-range");
   CASE_THROW_RECOVERABLE(R"({"int64Field":"9223372036854775808"})", "Value out-of-range");
   CASE(R"({"uInt8Field":255})", root.getUInt8Field() == 255);
-  CASE(R"({"uInt8Field":"0"})", root.getUInt8Field() == 0);
+  CASE_NO_ROUNDTRIP(R"({"uInt8Field":"0"})", root.getUInt8Field() == 0);
   CASE_THROW_RECOVERABLE(R"({"uInt8Field":"256"})", "Value out-of-range");
   CASE_THROW_RECOVERABLE(R"({"uInt8Field":-1})", "Value out-of-range");
   CASE(R"({"uInt16Field":65535})", root.getUInt16Field() == 65535);
-  CASE(R"({"uInt16Field":"0"})", root.getUInt16Field() == 0);
+  CASE_NO_ROUNDTRIP(R"({"uInt16Field":"0"})", root.getUInt16Field() == 0);
   CASE_THROW_RECOVERABLE(R"({"uInt16Field":"655356"})", "Value out-of-range");
   CASE_THROW_RECOVERABLE(R"({"uInt16Field":-1})", "Value out-of-range");
   CASE(R"({"uInt32Field":4294967295})", root.getUInt32Field() == 4294967295);
-  CASE(R"({"uInt32Field":"0"})", root.getUInt32Field() == 0);
+  CASE_NO_ROUNDTRIP(R"({"uInt32Field":"0"})", root.getUInt32Field() == 0);
   CASE_THROW_RECOVERABLE(R"({"uInt32Field":"42949672956"})", "Value out-of-range");
   CASE_THROW_RECOVERABLE(R"({"uInt32Field":-1})", "Value out-of-range");
-  CASE(R"({"uInt64Field":9007199254740991})", root.getUInt64Field() == 9007199254740991ULL);
+  CASE_NO_ROUNDTRIP(R"({"uInt64Field":9007199254740991})", root.getUInt64Field() == 9007199254740991ULL);
   CASE(R"({"uInt64Field":"18446744073709551615"})", root.getUInt64Field() == 18446744073709551615ULL);
-  CASE(R"({"uInt64Field":"0"})", root.getUInt64Field() == 0);
+  CASE_NO_ROUNDTRIP(R"({"uInt64Field":"0"})", root.getUInt64Field() == 0);
   CASE_THROW_RECOVERABLE(R"({"uInt64Field":"18446744073709551616"})", "Value out-of-range");
-  CASE(R"({"float32Field":0})", root.getFloat32Field() == 0);
+  CASE_NO_ROUNDTRIP(R"({"float32Field":0})", root.getFloat32Field() == 0);
   CASE(R"({"float32Field":4.5})", root.getFloat32Field() == 4.5);
-  CASE(R"({"float32Field":null})", kj::isNaN(root.getFloat32Field()));
-  CASE(R"({"float32Field":"nan"})", kj::isNaN(root.getFloat32Field()));
-  CASE(R"({"float32Field":"nan"})", kj::isNaN(root.getFloat32Field()));
+  CASE_NO_ROUNDTRIP(R"({"float32Field":null})", kj::isNaN(root.getFloat32Field()));
+  CASE(R"({"float32Field":"NaN"})", kj::isNaN(root.getFloat32Field()));
+  CASE_NO_ROUNDTRIP(R"({"float32Field":"nan"})", kj::isNaN(root.getFloat32Field()));
   CASE(R"({"float32Field":"Infinity"})", root.getFloat32Field() == kj::inf());
   CASE(R"({"float32Field":"-Infinity"})", root.getFloat32Field() == -kj::inf());
-  CASE(R"({"float64Field":0})", root.getFloat64Field() == 0);
+  CASE_NO_ROUNDTRIP(R"({"float32Field":"infinity"})", root.getFloat32Field() == kj::inf());
+  CASE_NO_ROUNDTRIP(R"({"float32Field":"-infinity"})", root.getFloat32Field() == -kj::inf());
+  CASE_NO_ROUNDTRIP(R"({"float32Field":"INF"})", root.getFloat32Field() == kj::inf());
+  CASE_NO_ROUNDTRIP(R"({"float32Field":"-INF"})", root.getFloat32Field() == -kj::inf());
+  CASE_NO_ROUNDTRIP(R"({"float32Field":1e39})", root.getFloat32Field() == kj::inf());
+  CASE_NO_ROUNDTRIP(R"({"float32Field":-1e39})", root.getFloat32Field() == -kj::inf());
+  CASE_NO_ROUNDTRIP(R"({"float64Field":0})", root.getFloat64Field() == 0);
   CASE(R"({"float64Field":4.5})", root.getFloat64Field() == 4.5);
-  CASE(R"({"float64Field":null})", kj::isNaN(root.getFloat64Field()));
-  CASE(R"({"float64Field":"nan"})", kj::isNaN(root.getFloat64Field()));
-  CASE(R"({"float64Field":"nan"})", kj::isNaN(root.getFloat64Field()));
+  CASE_NO_ROUNDTRIP(R"({"float64Field":null})", kj::isNaN(root.getFloat64Field()));
+  CASE(R"({"float64Field":"NaN"})", kj::isNaN(root.getFloat64Field()));
+  CASE_NO_ROUNDTRIP(R"({"float64Field":"nan"})", kj::isNaN(root.getFloat64Field()));
   CASE(R"({"float64Field":"Infinity"})", root.getFloat64Field() == kj::inf());
-  CASE(R"({"float64Field":"-Infinity"})", root.getFloat64Field() == -kj::inf());
+  CASE_NO_ROUNDTRIP(R"({"float64Field":"infinity"})", root.getFloat64Field() == kj::inf());
+  CASE_NO_ROUNDTRIP(R"({"float64Field":"-infinity"})", root.getFloat64Field() == -kj::inf());
+  CASE_NO_ROUNDTRIP(R"({"float64Field":"INF"})", root.getFloat64Field() == kj::inf());
+  CASE_NO_ROUNDTRIP(R"({"float64Field":"-INF"})", root.getFloat64Field() == -kj::inf());
+  CASE_NO_ROUNDTRIP(R"({"float64Field":1e309})", root.getFloat64Field() == kj::inf());
+  CASE_NO_ROUNDTRIP(R"({"float64Field":-1e309})", root.getFloat64Field() == -kj::inf());
   CASE(R"({"textField":"hello"})", kj::str("hello") == root.getTextField());
   CASE(R"({"dataField":[7,0,122]})",
       kj::heapArray<byte>({7,0,122}).asPtr() == root.getDataField());
-  CASE(R"({"structField":null})", root.hasStructField() == false);
   CASE(R"({"structField":{}})", root.hasStructField() == true);
   CASE(R"({"structField":{}})", root.getStructField().getBoolField() == false);
-  CASE(R"({"structField":{"boolField":false}})", root.getStructField().getBoolField() == false);
+  CASE_NO_ROUNDTRIP(R"({"structField":{"boolField":false}})", root.getStructField().getBoolField() == false);
   CASE(R"({"structField":{"boolField":true}})", root.getStructField().getBoolField() == true);
   CASE(R"({"enumField":"bar"})", root.getEnumField() == TestEnum::BAR);
 
+  CASE_THROW_RECOVERABLE(R"({"structField":null})", "Expected object value");
+  CASE_THROW_RECOVERABLE(R"({"structList":null})", "Expected list value");
+  CASE_THROW_RECOVERABLE(R"({"boolList":null})", "Expected list value");
+  CASE_THROW_RECOVERABLE(R"({"structList":[null]})", "Expected object value");
   CASE_THROW_RECOVERABLE(R"({"int64Field":"177a"})", "String does not contain valid");
   CASE_THROW_RECOVERABLE(R"({"uInt64Field":"177a"})", "String does not contain valid");
   CASE_THROW_RECOVERABLE(R"({"float64Field":"177a"})", "String does not contain valid");
 
   CASE(R"({})", root.hasBoolList() == false);
-  CASE(R"({"boolList":null})", root.hasBoolList() == false);
   CASE(R"({"boolList":[]})", root.hasBoolList() == true);
   CASE(R"({"boolList":[]})", root.getBoolList().size() == 0);
   CASE(R"({"boolList":[false]})", root.getBoolList().size() == 1);
   CASE(R"({"boolList":[false]})", root.getBoolList()[0] == false);
   CASE(R"({"boolList":[true]})", root.getBoolList()[0] == true);
   CASE(R"({"int8List":[7]})", root.getInt8List()[0] == 7);
-  CASE(R"({"int8List":["7"]})", root.getInt8List()[0] == 7);
+  CASE_NO_ROUNDTRIP(R"({"int8List":["7"]})", root.getInt8List()[0] == 7);
   CASE(R"({"int16List":[7]})", root.getInt16List()[0] == 7);
-  CASE(R"({"int16List":["7"]})", root.getInt16List()[0] == 7);
+  CASE_NO_ROUNDTRIP(R"({"int16List":["7"]})", root.getInt16List()[0] == 7);
   CASE(R"({"int32List":[7]})", root.getInt32List()[0] == 7);
-  CASE(R"({"int32List":["7"]})", root.getInt32List()[0] == 7);
-  CASE(R"({"int64List":[7]})", root.getInt64List()[0] == 7);
+  CASE_NO_ROUNDTRIP(R"({"int32List":["7"]})", root.getInt32List()[0] == 7);
+  CASE_NO_ROUNDTRIP(R"({"int64List":[7]})", root.getInt64List()[0] == 7);
   CASE(R"({"int64List":["7"]})", root.getInt64List()[0] == 7);
   CASE(R"({"uInt8List":[7]})", root.getUInt8List()[0] == 7);
-  CASE(R"({"uInt8List":["7"]})", root.getUInt8List()[0] == 7);
+  CASE_NO_ROUNDTRIP(R"({"uInt8List":["7"]})", root.getUInt8List()[0] == 7);
   CASE(R"({"uInt16List":[7]})", root.getUInt16List()[0] == 7);
-  CASE(R"({"uInt16List":["7"]})", root.getUInt16List()[0] == 7);
+  CASE_NO_ROUNDTRIP(R"({"uInt16List":["7"]})", root.getUInt16List()[0] == 7);
   CASE(R"({"uInt32List":[7]})", root.getUInt32List()[0] == 7);
-  CASE(R"({"uInt32List":["7"]})", root.getUInt32List()[0] == 7);
-  CASE(R"({"uInt64List":[7]})", root.getUInt64List()[0] == 7);
+  CASE_NO_ROUNDTRIP(R"({"uInt32List":["7"]})", root.getUInt32List()[0] == 7);
+  CASE_NO_ROUNDTRIP(R"({"uInt64List":[7]})", root.getUInt64List()[0] == 7);
   CASE(R"({"uInt64List":["7"]})", root.getUInt64List()[0] == 7);
   CASE(R"({"float32List":[4.5]})", root.getFloat32List()[0] == 4.5);
-  CASE(R"({"float32List":["4.5"]})", root.getFloat32List()[0] == 4.5);
-  CASE(R"({"float32List":[null]})", kj::isNaN(root.getFloat32List()[0]));
-  CASE(R"({"float32List":["nan"]})", kj::isNaN(root.getFloat32List()[0]));
-  CASE(R"({"float32List":["infinity"]})", root.getFloat32List()[0] == kj::inf());
-  CASE(R"({"float32List":["-infinity"]})", root.getFloat32List()[0] == -kj::inf());
+  CASE_NO_ROUNDTRIP(R"({"float32List":["4.5"]})", root.getFloat32List()[0] == 4.5);
+  CASE_NO_ROUNDTRIP(R"({"float32List":[null]})", kj::isNaN(root.getFloat32List()[0]));
+  CASE(R"({"float32List":["NaN"]})", kj::isNaN(root.getFloat32List()[0]));
+  CASE(R"({"float32List":["Infinity"]})", root.getFloat32List()[0] == kj::inf());
+  CASE(R"({"float32List":["-Infinity"]})", root.getFloat32List()[0] == -kj::inf());
   CASE(R"({"float64List":[4.5]})", root.getFloat64List()[0] == 4.5);
-  CASE(R"({"float64List":["4.5"]})", root.getFloat64List()[0] == 4.5);
-  CASE(R"({"float64List":[null]})", kj::isNaN(root.getFloat64List()[0]));
-  CASE(R"({"float64List":["nan"]})", kj::isNaN(root.getFloat64List()[0]));
-  CASE(R"({"float64List":["infinity"]})", root.getFloat64List()[0] == kj::inf());
-  CASE(R"({"float64List":["-infinity"]})", root.getFloat64List()[0] == -kj::inf());
+  CASE_NO_ROUNDTRIP(R"({"float64List":["4.5"]})", root.getFloat64List()[0] == 4.5);
+  CASE_NO_ROUNDTRIP(R"({"float64List":[null]})", kj::isNaN(root.getFloat64List()[0]));
+  CASE(R"({"float64List":["NaN"]})", kj::isNaN(root.getFloat64List()[0]));
+  CASE(R"({"float64List":["Infinity"]})", root.getFloat64List()[0] == kj::inf());
+  CASE(R"({"float64List":["-Infinity"]})", root.getFloat64List()[0] == -kj::inf());
   CASE(R"({"textList":["hello"]})", kj::str("hello") == root.getTextList()[0]);
   CASE(R"({"dataList":[[7,0,122]]})",
       kj::heapArray<byte>({7,0,122}).asPtr() == root.getDataList()[0]);
-  CASE(R"({"structList":null})", root.hasStructList() == false);
-  CASE(R"({"structList":[null]})", root.hasStructList() == true);
-  CASE(R"({"structList":[null]})", root.getStructList()[0].getBoolField() == false);
+  CASE(R"({"structList":[{}]})", root.hasStructList() == true);
   CASE(R"({"structList":[{}]})", root.getStructList()[0].getBoolField() == false);
-  CASE(R"({"structList":[{"boolField":false}]})", root.getStructList()[0].getBoolField() == false);
+  CASE_NO_ROUNDTRIP(R"({"structList":[{"boolField":false}]})", root.getStructList()[0].getBoolField() == false);
   CASE(R"({"structList":[{"boolField":true}]})", root.getStructList()[0].getBoolField() == true);
   CASE(R"({"enumList":["bar"]})", root.getEnumList()[0] == TestEnum::BAR);
+#undef CASE_MAYBE_ROUNDTRIP
+#undef CASE_NO_ROUNDTRIP
 #undef CASE
 #undef CASE_THROW
 #undef CASE_THROW_RECOVERABLE
@@ -515,20 +535,6 @@ KJ_TEST("basic json decoding") {
     KJ_EXPECT_THROW_MESSAGE("Unexpected", json.decodeRaw("+123", root));
   }
 
-  {
-    MallocMessageBuilder message;
-    auto root = message.initRoot<JsonValue>();
-
-    KJ_EXPECT_THROW_MESSAGE("Overflow", json.decodeRaw("1e1024", root));
-  }
-
-  {
-    MallocMessageBuilder message;
-    auto root = message.initRoot<JsonValue>();
-
-    KJ_EXPECT_THROW_MESSAGE("Underflow", json.decodeRaw("1e-1023", root));
-  }
-
   {
     MallocMessageBuilder message;
     auto root = message.initRoot<JsonValue>();
@@ -637,7 +643,7 @@ KJ_TEST("maximum nesting depth") {
   }
 }
 
-class TestHandler: public JsonCodec::Handler<Text> {
+class TestCallHandler: public JsonCodec::Handler<Text> {
 public:
   void encode(const JsonCodec& codec, Text::Reader input,
               JsonValue::Builder output) const override {
@@ -654,31 +660,144 @@ public:
   }
 };
 
-KJ_TEST("register handler") {
-  MallocMessageBuilder message;
-  auto root = message.getRoot<test::TestOldVersion>();
+class TestDynamicStructHandler: public JsonCodec::Handler<DynamicStruct> {
+public:
+  void encode(const JsonCodec& codec, DynamicStruct::Reader input,
+              JsonValue::Builder output) const override {
+    auto fields = input.getSchema().getFields();
+    auto items = output.initArray(fields.size());
+    for (auto field: fields) {
+      KJ_REQUIRE(field.getIndex() < items.size());
+      auto item = items[field.getIndex()];
+      if (input.has(field)) {
+        codec.encode(input.get(field), field.getType(), item);
+      } else {
+        item.setNull();
+      }
+    }
+  }
 
-  TestHandler handler;
+  void decode(const JsonCodec& codec, JsonValue::Reader input,
+              DynamicStruct::Builder output) const override {
+    auto orphanage = Orphanage::getForMessageContaining(output);
+    auto fields = output.getSchema().getFields();
+    auto items = input.getArray();
+    for (auto field: fields) {
+      KJ_REQUIRE(field.getIndex() < items.size());
+      auto item = items[field.getIndex()];
+      if (!item.isNull()) {
+        output.adopt(field, codec.decode(item, field.getType(), orphanage));
+      }
+    }
+  }
+};
+
+class TestStructHandler: public JsonCodec::Handler<test::TestOldVersion> {
+public:
+  void encode(const JsonCodec& codec, test::TestOldVersion::Reader input, JsonValue::Builder output) const override {
+    dynamicHandler.encode(codec, input, output);
+  }
+
+  void decode(const JsonCodec& codec, JsonValue::Reader input, test::TestOldVersion::Builder output) const override {
+    dynamicHandler.decode(codec, input, output);
+  }
+
+private:
+  TestDynamicStructHandler dynamicHandler;
+};
+
+KJ_TEST("register custom encoding handlers") {
   JsonCodec json;
-  json.addTypeHandler(handler);
 
+  TestStructHandler structHandler;
+  json.addTypeHandler(structHandler);
+
+  // JSON decoder can't parse calls back, so test only encoder here
+  TestCallHandler callHandler;
+  json.addTypeHandler(callHandler);
+
+  MallocMessageBuilder message;
+  auto root = message.getRoot<test::TestOldVersion>();
   root.setOld1(123);
   root.setOld2("foo");
-  KJ_EXPECT(json.encode(root) == "{\"old1\":\"123\",\"old2\":Frob(123,\"foo\")}");
+
+  KJ_EXPECT(json.encode(root) == "[\"123\",Frob(123,\"foo\"),null]");
 }
 
-KJ_TEST("register field handler") {
+KJ_TEST("register custom roundtrip handler") {
+  for (auto i = 1; i <= 2; i++) {
+    JsonCodec json;
+    TestStructHandler staticHandler;
+    TestDynamicStructHandler dynamicHandler;
+    kj::String encoded;
+
+    if (i == 1) {
+      // first iteration: test with explicit struct handler
+      json.addTypeHandler(staticHandler);
+    } else {
+      // second iteration: same checks, but with DynamicStruct handler
+      json.addTypeHandler(StructSchema::from<test::TestOldVersion>(), dynamicHandler);
+    }
+
+    {
       MallocMessageBuilder message;
-  auto root = message.getRoot<test::TestOutOfOrder>();
+      auto root = message.getRoot<test::TestOldVersion>();
+      root.setOld1(123);
+      root.initOld3().setOld2("foo");
+
+      encoded = json.encode(root);
+
+      KJ_EXPECT(encoded == "[\"123\",null,[\"0\",\"foo\",null]]");
+    }
 
-  TestHandler handler;
+    {
+      MallocMessageBuilder message;
+      auto root = message.getRoot<test::TestOldVersion>();
+      json.decode(encoded, root);
+
+      KJ_EXPECT(root.getOld1() == 123);
+      KJ_EXPECT(!root.hasOld2());
+      auto nested = root.getOld3();
+      KJ_EXPECT(nested.getOld1() == 0);
+      KJ_EXPECT("foo" == nested.getOld2());
+      KJ_EXPECT(!nested.hasOld3());
+    }
+  }
+}
+
+KJ_TEST("register field handler") {
+  TestStructHandler handler;
   JsonCodec json;
-  json.addFieldHandler(StructSchema::from<test::TestOutOfOrder>().getFieldByName("corge"),
+  json.addFieldHandler(StructSchema::from<test::TestOldVersion>().getFieldByName("old3"),
                        handler);
 
-  root.setBaz("abcd");
-  root.setCorge("efg");
-  KJ_EXPECT(json.encode(root) == "{\"corge\":Frob(123,\"efg\"),\"baz\":\"abcd\"}");
+  kj::String encoded;
+
+  {
+    MallocMessageBuilder message;
+    auto root = message.getRoot<test::TestOldVersion>();
+    root.setOld1(123);
+    root.setOld2("foo");
+    auto nested = root.initOld3();
+    nested.setOld2("bar");
+
+    encoded = json.encode(root);
+
+    KJ_EXPECT(encoded == "{\"old1\":\"123\",\"old2\":\"foo\",\"old3\":[\"0\",\"bar\",null]}")
+  }
+
+  {
+    MallocMessageBuilder message;
+    auto root = message.getRoot<test::TestOldVersion>();
+    json.decode(encoded, root);
+
+    KJ_EXPECT(root.getOld1() == 123);
+    KJ_EXPECT("foo" == root.getOld2());
+    auto nested = root.getOld3();
+    KJ_EXPECT(nested.getOld1() == 0);
+    KJ_EXPECT("bar" == nested.getOld2());
+    KJ_EXPECT(!nested.hasOld3());
+  }
 }
 
 class TestCapabilityHandler: public JsonCodec::Handler<test::TestInterface> {
@@ -703,29 +822,6 @@ KJ_TEST("register capability handler") {
   json.addTypeHandler(handler);
 }
 
-class TestDynamicStructHandler: public JsonCodec::Handler<DynamicStruct> {
-public:
-  void encode(const JsonCodec& codec, DynamicStruct::Reader input,
-              JsonValue::Builder output) const override {
-    KJ_UNIMPLEMENTED("TestDynamicStructHandler::encode");
-  }
-
-  void decode(const JsonCodec& codec, JsonValue::Reader input,
-              DynamicStruct::Builder output) const override {
-    KJ_UNIMPLEMENTED("TestDynamicStructHandler::decode");
-  }
-};
-
-
-KJ_TEST("register DynamicStruct handler") {
-  // This test currently only checks that this compiles, which at one point wasn't the caes.
-  // TODO(test): Actually run some code here.
-
-  TestDynamicStructHandler handler;
-  JsonCodec json;
-  json.addTypeHandler(Schema::from<TestAllTypes>(), handler);
-}
-
 }  // namespace
 }  // namespace _ (private)
 }  // namespace capnp

c++/src/capnp/compat/json.c++

@@ -386,189 +386,152 @@ void JsonCodec::encodeField(StructSchema::Field field, DynamicValue::Reader inpu
   encode(input, field.getType(), output);
 }
 
-namespace {
+Orphan<DynamicList> JsonCodec::decodeArray(List<JsonValue>::Reader input, ListSchema type, Orphanage orphanage) const {
+  auto orphan = orphanage.newOrphan(type, input.size());
+  auto output = orphan.get();
+  for (auto i: kj::indices(input)) {
+    output.adopt(i, decode(input[i], type.getElementType(), orphanage));
+  }
+  return orphan;
+}
+
+void JsonCodec::decodeObject(JsonValue::Reader input, StructSchema type, Orphanage orphanage, DynamicStruct::Builder output) const {
+  KJ_REQUIRE(input.isObject(), "Expected object value");
+  for (auto field: input.getObject()) {
+    KJ_IF_MAYBE(fieldSchema, type.findFieldByName(field.getName())) {
+      auto fieldValue = field.getValue();
+      auto fieldType = (*fieldSchema).getType();
+
+      auto iter = impl->fieldHandlers.find(*fieldSchema);
+      if (iter != impl->fieldHandlers.end()) {
+        output.adopt(*fieldSchema, iter->second->decodeBase(*this, fieldValue, fieldType, orphanage));
+      } else {
+        output.adopt(*fieldSchema, decode(fieldValue, fieldType, orphanage));
+      }
+    } else {
+      // Unknown json fields are ignored to allow schema evolution
+    }
+  }
+}
+
+void JsonCodec::decode(JsonValue::Reader input, DynamicStruct::Builder output) const {
+  auto type = output.getSchema();
+
+  auto iter = impl->typeHandlers.find(type);
+  if (iter != impl->typeHandlers.end()) {
+    return iter->second->decodeStructBase(*this, input, output);
+  }
+
+  decodeObject(input, type, Orphanage::getForMessageContaining(output), output);
+}
+
+Orphan<DynamicValue> JsonCodec::decode(
+    JsonValue::Reader input, Type type, Orphanage orphanage) const {
+  auto iter = impl->typeHandlers.find(type);
+  if (iter != impl->typeHandlers.end()) {
+    return iter->second->decodeBase(*this, input, type, orphanage);
+  }
 
-template <typename SetFn, typename DecodeArrayFn, typename DecodeObjectFn>
-void decodeField(Type type, JsonValue::Reader value, SetFn setFn, DecodeArrayFn decodeArrayFn,
-    DecodeObjectFn decodeObjectFn) {
-  // This code relies on conversions in DynamicValue::Reader::as<T>.
   switch(type.which()) {
     case schema::Type::VOID:
-      break;
+      return capnp::VOID;
     case schema::Type::BOOL:
-      switch (value.which()) {
+      switch (input.which()) {
         case JsonValue::BOOLEAN:
-          setFn(value.getBoolean());
-          break;
+          return input.getBoolean();
         default:
           KJ_FAIL_REQUIRE("Expected boolean value");
       }
-      break;
     case schema::Type::INT8:
     case schema::Type::INT16:
     case schema::Type::INT32:
     case schema::Type::INT64:
       // Relies on range check in DynamicValue::Reader::as<IntType>
-      switch (value.which()) {
+      switch (input.which()) {
         case JsonValue::NUMBER:
-          setFn(value.getNumber());
-          break;
+          return input.getNumber();
         case JsonValue::STRING:
-          setFn(value.getString().parseAs<int64_t>());
-          break;
+          return input.getString().parseAs<int64_t>();
         default:
           KJ_FAIL_REQUIRE("Expected integer value");
       }
-      break;
     case schema::Type::UINT8:
     case schema::Type::UINT16:
     case schema::Type::UINT32:
     case schema::Type::UINT64:
       // Relies on range check in DynamicValue::Reader::as<IntType>
-      switch (value.which()) {
+      switch (input.which()) {
         case JsonValue::NUMBER:
-          setFn(value.getNumber());
-          break;
+          return input.getNumber();
         case JsonValue::STRING:
-          setFn(value.getString().parseAs<uint64_t>());
-          break;
+          return input.getString().parseAs<uint64_t>();
         default:
           KJ_FAIL_REQUIRE("Expected integer value");
       }
-      break;
     case schema::Type::FLOAT32:
     case schema::Type::FLOAT64:
-      switch (value.which()) {
+      switch (input.which()) {
         case JsonValue::NULL_:
-          setFn(kj::nan());
-          break;
+          return kj::nan();
         case JsonValue::NUMBER:
-          setFn(value.getNumber());
-          break;
+          return input.getNumber();
         case JsonValue::STRING:
-          setFn(value.getString().parseAs<double>());
-          break;
+          return input.getString().parseAs<double>();
         default:
           KJ_FAIL_REQUIRE("Expected float value");
       }
-      break;
     case schema::Type::TEXT:
-      switch (value.which()) {
+      switch (input.which()) {
         case JsonValue::STRING:
-          setFn(value.getString());
-          break;
+          return orphanage.newOrphanCopy(input.getString());
         default:
           KJ_FAIL_REQUIRE("Expected text value");
       }
-      break;
     case schema::Type::DATA:
-      switch (value.which()) {
+      switch (input.which()) {
         case JsonValue::ARRAY: {
-          auto array = value.getArray();
-          kj::Vector<byte> data(array.size());
-          for (auto arrayObject : array) {
-            auto x = arrayObject.getNumber();
+          auto array = input.getArray();
+          auto orphan = orphanage.newOrphan<Data>(array.size());
+          auto data = orphan.get();
+          for (auto i: kj::indices(array)) {
+            auto x = array[i].getNumber();
             KJ_REQUIRE(byte(x) == x, "Number in byte array is not an integer in [0, 255]");
-            data.add(byte(x));
+            data[i] = x;
           }
-          setFn(Data::Reader(data.asPtr()));
-          break;
+          return kj::mv(orphan);
         }
         default:
           KJ_FAIL_REQUIRE("Expected data value");
       }
-      break;
     case schema::Type::LIST:
-      switch (value.which()) {
-        case JsonValue::NULL_:
-          // nothing to do
-          break;
+      switch (input.which()) {
         case JsonValue::ARRAY:
-          decodeArrayFn(value.getArray());
-          break;
+          return decodeArray(input.getArray(), type.asList(), orphanage);
         default:
           KJ_FAIL_REQUIRE("Expected list value");
       }
-      break;
     case schema::Type::ENUM:
-      switch (value.which()) {
+      switch (input.which()) {
         case JsonValue::STRING:
-          setFn(value.getString());
-          break;
+          return DynamicEnum(type.asEnum().getEnumerantByName(input.getString()));
         default:
           KJ_FAIL_REQUIRE("Expected enum value");
       }
-      break;
-    case schema::Type::STRUCT:
-      switch (value.which()) {
-        case JsonValue::NULL_:
-          // nothing to do
-          break;
-        case JsonValue::OBJECT:
-          decodeObjectFn(value.getObject());
-          break;
-        default:
-          KJ_FAIL_REQUIRE("Expected object value");
+    case schema::Type::STRUCT: {
+      auto structType = type.asStruct();
+      auto orphan = orphanage.newOrphan(structType);
+      decodeObject(input, structType, orphanage, orphan.get());
+      return kj::mv(orphan);
     }
-      break;
     case schema::Type::INTERFACE:
       KJ_FAIL_REQUIRE("don't know how to JSON-decode capabilities; "
-                      "JsonCodec::Handler not implemented yet :(");
+                      "please register a JsonCodec::Handler for this");
     case schema::Type::ANY_POINTER:
       KJ_FAIL_REQUIRE("don't know how to JSON-decode AnyPointer; "
-                      "JsonCodec::Handler not implemented yet :(");
-  }
-}
-} // namespace
-
-void JsonCodec::decodeArray(List<JsonValue>::Reader input, DynamicList::Builder output) const {
-  KJ_ASSERT(input.size() == output.size(), "Builder was not initialized to input size");
-  auto type = output.getSchema().getElementType();
-  for (auto i = 0; i < input.size(); i++) {
-    decodeField(type, input[i],
-        [&](DynamicValue::Reader value) { output.set(i, value); },
-        [&](List<JsonValue>::Reader array) {
-          decodeArray(array, output.init(i, array.size()).as<DynamicList>());
-        },
-        [&](List<JsonValue::Field>::Reader object) {
-          decodeObject(object, output[i].as<DynamicStruct>());
-        });
+                      "please register a JsonCodec::Handler for this");
   }
-}
 
-void JsonCodec::decodeObject(List<JsonValue::Field>::Reader input, DynamicStruct::Builder output)
-    const {
-  for (auto field : input) {
-    KJ_IF_MAYBE(fieldSchema, output.getSchema().findFieldByName(field.getName())) {
-      decodeField((*fieldSchema).getType(), field.getValue(),
-          [&](DynamicValue::Reader value) { output.set(*fieldSchema, value); },
-          [&](List<JsonValue>::Reader array) {
-            decodeArray(array, output.init(*fieldSchema, array.size()).as<DynamicList>());
-          },
-          [&](List<JsonValue::Field>::Reader object) {
-            decodeObject(object, output.init(*fieldSchema).as<DynamicStruct>());
-          });
-    } else {
-      // Unknown json fields are ignored to allow schema evolution
-    }
-  }
-}
-
-void JsonCodec::decode(JsonValue::Reader input, DynamicStruct::Builder output) const {
-  // TODO(soon): type and field handlers
-  switch (input.which()) {
-    case JsonValue::OBJECT:
-      decodeObject(input.getObject(), output);
-      break;
-    default:
-      KJ_FAIL_REQUIRE("Top level json value must be object");
-  };
-}
-
-Orphan<DynamicValue> JsonCodec::decode(
-    JsonValue::Reader input, Type type, Orphanage orphanage) const {
-  // TODO(soon)
-  KJ_FAIL_ASSERT("JSON decode into orphanage not implement yet. :(");
+  KJ_CLANG_KNOWS_THIS_IS_UNREACHABLE_BUT_GCC_DOESNT;
 }
 
 // -----------------------------------------------------------------------------
@@ -697,19 +660,7 @@ public:
   }
 
   void parseNumber(JsonValue::Builder& output) {
-    auto numberStr = consumeNumber();
-    char *endPtr;
-
-    errno = 0;
-    double value = strtod(numberStr.begin(), &endPtr);
-
-    KJ_ASSERT(endPtr != numberStr.begin(), "strtod should not fail! Is consumeNumber wrong?");
-    KJ_REQUIRE((value != HUGE_VAL && value != -HUGE_VAL) || errno != ERANGE,
-        "Overflow in JSON number.");
-    KJ_REQUIRE(value != 0.0 || errno != ERANGE,
-        "Underflow in JSON number.");
-
-    output.setNumber(value);
+    output.setNumber(consumeNumber().parseAs<double>());
   }
 
   void parseString(JsonValue::Builder& output) {

c++/src/capnp/compat/json.h

@@ -49,7 +49,7 @@ class JsonCodec {
   // - 64-bit integers are encoded as strings, since JSON "numbers" are double-precision floating
   //   points which cannot store a 64-bit integer without losing data.
   // - NaNs and infinite floating point numbers are not allowed by the JSON spec, and so are encoded
-  //   as null. This matches the behavior of `JSON.stringify` in at least Firefox and Chrome.
+  //   as strings.
   // - Data is encoded as an array of numbers in the range [0,255]. You probably want to register
   //   a handler that does something better, like maybe base64 encoding, but there are a zillion
   //   different ways people do this.
@@ -220,8 +220,8 @@ private:
 
   void encodeField(StructSchema::Field field, DynamicValue::Reader input,
                    JsonValue::Builder output) const;
-  void decodeArray(List<JsonValue>::Reader input, DynamicList::Builder output) const;
-  void decodeObject(List<JsonValue::Field>::Reader input, DynamicStruct::Builder output) const;
+  Orphan<DynamicList> decodeArray(List<JsonValue>::Reader input, ListSchema type, Orphanage orphanage) const;
+  void decodeObject(JsonValue::Reader input, StructSchema type, Orphanage orphanage, DynamicStruct::Builder output) const;
   void addTypeHandlerImpl(Type type, HandlerBase& handler);
   void addFieldHandlerImpl(StructSchema::Field field, Type type, HandlerBase& handler);
 };