Rewrite JSON decoding

 - Add support for decoding with Orphanage.
 - Allow decoding of non-struct root JSON objects.
 - Add support for custom decoding type handlers.
 - Add support for custom decoding field handlers.

c++/src/capnp/compat/json-test.c++

@@ -257,19 +257,21 @@ KJ_TEST("decode all types") {
   CASE(R"({"textField":"hello"})", kj::str("hello") == root.getTextField());
   CASE(R"({"dataField":[7,0,122]})",
       kj::heapArray<byte>({7,0,122}).asPtr() == root.getDataField());
-  CASE(R"({"structField":null})", root.hasStructField() == false);
   CASE(R"({"structField":{}})", root.hasStructField() == true);
   CASE(R"({"structField":{}})", root.getStructField().getBoolField() == false);
   CASE(R"({"structField":{"boolField":false}})", root.getStructField().getBoolField() == false);
   CASE(R"({"structField":{"boolField":true}})", root.getStructField().getBoolField() == true);
   CASE(R"({"enumField":"bar"})", root.getEnumField() == TestEnum::BAR);
 
+  CASE_THROW_RECOVERABLE(R"({"structField":null})", "Expected object value");
+  CASE_THROW_RECOVERABLE(R"({"structList":null})", "Expected list value");
+  CASE_THROW_RECOVERABLE(R"({"boolList":null})", "Expected list value");
+  CASE_THROW_RECOVERABLE(R"({"structList":[null]})", "Expected object value");
   CASE_THROW_RECOVERABLE(R"({"int64Field":"177a"})", "String does not contain valid");
   CASE_THROW_RECOVERABLE(R"({"uInt64Field":"177a"})", "String does not contain valid");
   CASE_THROW_RECOVERABLE(R"({"float64Field":"177a"})", "String does not contain valid");
 
   CASE(R"({})", root.hasBoolList() == false);
-  CASE(R"({"boolList":null})", root.hasBoolList() == false);
   CASE(R"({"boolList":[]})", root.hasBoolList() == true);
   CASE(R"({"boolList":[]})", root.getBoolList().size() == 0);
   CASE(R"({"boolList":[false]})", root.getBoolList().size() == 1);
@@ -306,9 +308,7 @@ KJ_TEST("decode all types") {
   CASE(R"({"textList":["hello"]})", kj::str("hello") == root.getTextList()[0]);
   CASE(R"({"dataList":[[7,0,122]]})",
       kj::heapArray<byte>({7,0,122}).asPtr() == root.getDataList()[0]);
-  CASE(R"({"structList":null})", root.hasStructList() == false);
-  CASE(R"({"structList":[null]})", root.hasStructList() == true);
-  CASE(R"({"structList":[null]})", root.getStructList()[0].getBoolField() == false);
+  CASE(R"({"structList":[{}]})", root.hasStructList() == true);
   CASE(R"({"structList":[{}]})", root.getStructList()[0].getBoolField() == false);
   CASE(R"({"structList":[{"boolField":false}]})", root.getStructList()[0].getBoolField() == false);
   CASE(R"({"structList":[{"boolField":true}]})", root.getStructList()[0].getBoolField() == true);
@@ -637,7 +637,7 @@ KJ_TEST("maximum nesting depth") {
   }
 }
 
-class TestHandler: public JsonCodec::Handler<Text> {
+class TestCallHandler: public JsonCodec::Handler<Text> {
 public:
   void encode(const JsonCodec& codec, Text::Reader input,
               JsonValue::Builder output) const override {
@@ -654,31 +654,144 @@ public:
   }
 };
 
-KJ_TEST("register handler") {
-  MallocMessageBuilder message;
-  auto root = message.getRoot<test::TestOldVersion>();
+class TestDynamicStructHandler: public JsonCodec::Handler<DynamicStruct> {
+public:
+  void encode(const JsonCodec& codec, DynamicStruct::Reader input,
+              JsonValue::Builder output) const override {
+    auto fields = input.getSchema().getFields();
+    auto items = output.initArray(fields.size());
+    for (auto field: fields) {
+      KJ_REQUIRE(field.getIndex() < items.size());
+      auto item = items[field.getIndex()];
+      if (input.has(field)) {
+        codec.encode(input.get(field), field.getType(), item);
+      } else {
+        item.setNull();
+      }
+    }
+  }
+
+  void decode(const JsonCodec& codec, JsonValue::Reader input,
+              DynamicStruct::Builder output) const override {
+    auto orphanage = Orphanage::getForMessageContaining(output);
+    auto fields = output.getSchema().getFields();
+    auto items = input.getArray();
+    for (auto field: fields) {
+      KJ_REQUIRE(field.getIndex() < items.size());
+      auto item = items[field.getIndex()];
+      if (!item.isNull()) {
+        output.adopt(field, codec.decode(item, field.getType(), orphanage));
+      }
+    }
+  }
+};
+
+class TestStructHandler: public JsonCodec::Handler<test::TestOldVersion> {
+public:
+  void encode(const JsonCodec& codec, test::TestOldVersion::Reader input, JsonValue::Builder output) const override {
+    dynamicHandler.encode(codec, input, output);
+  }
+
+  void decode(const JsonCodec& codec, JsonValue::Reader input, test::TestOldVersion::Builder output) const override {
+    dynamicHandler.decode(codec, input, output);
+  }
+
+private:
+  TestDynamicStructHandler dynamicHandler;
+};
 
-  TestHandler handler;
+KJ_TEST("register custom encoding handlers") {
   JsonCodec json;
-  json.addTypeHandler(handler);
 
+  TestStructHandler structHandler;
+  json.addTypeHandler(structHandler);
+
+  // JSON decoder can't parse calls back, so test only encoder here
+  TestCallHandler callHandler;
+  json.addTypeHandler(callHandler);
+
+  MallocMessageBuilder message;
+  auto root = message.getRoot<test::TestOldVersion>();
   root.setOld1(123);
   root.setOld2("foo");
-  KJ_EXPECT(json.encode(root) == "{\"old1\":\"123\",\"old2\":Frob(123,\"foo\")}");
+
+  KJ_EXPECT(json.encode(root) == "[\"123\",Frob(123,\"foo\"),null]");
 }
 
-KJ_TEST("register field handler") {
-  MallocMessageBuilder message;
-  auto root = message.getRoot<test::TestOutOfOrder>();
+KJ_TEST("register custom roundtrip handler") {
+  for (auto i = 1; i <= 2; i++) {
+    JsonCodec json;
+    TestStructHandler staticHandler;
+    TestDynamicStructHandler dynamicHandler;
+    kj::String encoded;
+
+    if (i == 1) {
+      // first iteration: test with explicit struct handler
+      json.addTypeHandler(staticHandler);
+    } else {
+      // second iteration: same checks, but with DynamicStruct handler
+      json.addTypeHandler(StructSchema::from<test::TestOldVersion>(), dynamicHandler);
+    }
+
+    {
+      MallocMessageBuilder message;
+      auto root = message.getRoot<test::TestOldVersion>();
+      root.setOld1(123);
+      root.initOld3().setOld2("foo");
+
+      encoded = json.encode(root);
+
+      KJ_EXPECT(encoded == "[\"123\",null,[\"0\",\"foo\",null]]");
+    }
 
-  TestHandler handler;
+    {
+      MallocMessageBuilder message;
+      auto root = message.getRoot<test::TestOldVersion>();
+      json.decode(encoded, root);
+
+      KJ_EXPECT(root.getOld1() == 123);
+      KJ_EXPECT(!root.hasOld2());
+      auto nested = root.getOld3();
+      KJ_EXPECT(nested.getOld1() == 0);
+      KJ_EXPECT("foo" == nested.getOld2());
+      KJ_EXPECT(!nested.hasOld3());
+    }
+  }
+}
+
+KJ_TEST("register field handler") {
+  TestStructHandler handler;
   JsonCodec json;
-  json.addFieldHandler(StructSchema::from<test::TestOutOfOrder>().getFieldByName("corge"),
+  json.addFieldHandler(StructSchema::from<test::TestOldVersion>().getFieldByName("old3"),
                        handler);
 
-  root.setBaz("abcd");
-  root.setCorge("efg");
-  KJ_EXPECT(json.encode(root) == "{\"corge\":Frob(123,\"efg\"),\"baz\":\"abcd\"}");
+  kj::String encoded;
+
+  {
+    MallocMessageBuilder message;
+    auto root = message.getRoot<test::TestOldVersion>();
+    root.setOld1(123);
+    root.setOld2("foo");
+    auto nested = root.initOld3();
+    nested.setOld2("bar");
+
+    encoded = json.encode(root);
+
+    KJ_EXPECT(encoded == "{\"old1\":\"123\",\"old2\":\"foo\",\"old3\":[\"0\",\"bar\",null]}")
+  }
+
+  {
+    MallocMessageBuilder message;
+    auto root = message.getRoot<test::TestOldVersion>();
+    json.decode(encoded, root);
+
+    KJ_EXPECT(root.getOld1() == 123);
+    KJ_EXPECT("foo" == root.getOld2());
+    auto nested = root.getOld3();
+    KJ_EXPECT(nested.getOld1() == 0);
+    KJ_EXPECT("bar" == nested.getOld2());
+    KJ_EXPECT(!nested.hasOld3());
+  }
 }
 
 class TestCapabilityHandler: public JsonCodec::Handler<test::TestInterface> {
@@ -703,29 +816,6 @@ KJ_TEST("register capability handler") {
   json.addTypeHandler(handler);
 }
 
-class TestDynamicStructHandler: public JsonCodec::Handler<DynamicStruct> {
-public:
-  void encode(const JsonCodec& codec, DynamicStruct::Reader input,
-              JsonValue::Builder output) const override {
-    KJ_UNIMPLEMENTED("TestDynamicStructHandler::encode");
-  }
-
-  void decode(const JsonCodec& codec, JsonValue::Reader input,
-              DynamicStruct::Builder output) const override {
-    KJ_UNIMPLEMENTED("TestDynamicStructHandler::decode");
-  }
-};
-
-
-KJ_TEST("register DynamicStruct handler") {
-  // This test currently only checks that this compiles, which at one point wasn't the caes.
-  // TODO(test): Actually run some code here.
-
-  TestDynamicStructHandler handler;
-  JsonCodec json;
-  json.addTypeHandler(Schema::from<TestAllTypes>(), handler);
-}
-
 }  // namespace
 }  // namespace _ (private)
 }  // namespace capnp

c++/src/capnp/compat/json.c++

@@ -386,191 +386,152 @@ void JsonCodec::encodeField(StructSchema::Field field, DynamicValue::Reader inpu
   encode(input, field.getType(), output);
 }
 
-namespace {
+Orphan<DynamicList> JsonCodec::decodeArray(List<JsonValue>::Reader input, ListSchema type, Orphanage orphanage) const {
+  auto orphan = orphanage.newOrphan(type, input.size());
+  auto output = orphan.get();
+  for (auto i: kj::indices(input)) {
+    output.adopt(i, decode(input[i], type.getElementType(), orphanage));
+  }
+  return orphan;
+}
+
+void JsonCodec::decodeObject(JsonValue::Reader input, StructSchema type, Orphanage orphanage, DynamicStruct::Builder output) const {
+  KJ_REQUIRE(input.isObject(), "Expected object value");
+  for (auto field: input.getObject()) {
+    KJ_IF_MAYBE(fieldSchema, type.findFieldByName(field.getName())) {
+      auto fieldValue = field.getValue();
+      auto fieldType = (*fieldSchema).getType();
+
+      auto iter = impl->fieldHandlers.find(*fieldSchema);
+      if (iter != impl->fieldHandlers.end()) {
+        output.adopt(*fieldSchema, iter->second->decodeBase(*this, fieldValue, fieldType, orphanage));
+      } else {
+        output.adopt(*fieldSchema, decode(fieldValue, fieldType, orphanage));
+      }
+    } else {
+      // Unknown json fields are ignored to allow schema evolution
+    }
+  }
+}
+
+void JsonCodec::decode(JsonValue::Reader input, DynamicStruct::Builder output) const {
+  auto type = output.getSchema();
+
+  auto iter = impl->typeHandlers.find(type);
+  if (iter != impl->typeHandlers.end()) {
+    return iter->second->decodeStructBase(*this, input, output);
+  }
+
+  decodeObject(input, type, Orphanage::getForMessageContaining(output), output);
+}
+
+Orphan<DynamicValue> JsonCodec::decode(
+    JsonValue::Reader input, Type type, Orphanage orphanage) const {
+  auto iter = impl->typeHandlers.find(type);
+  if (iter != impl->typeHandlers.end()) {
+    return iter->second->decodeBase(*this, input, type, orphanage);
+  }
 
-template <typename SetFn, typename DecodeArrayFn, typename DecodeObjectFn>
-void decodeField(Type type, JsonValue::Reader value, SetFn setFn, DecodeArrayFn decodeArrayFn,
-    DecodeObjectFn decodeObjectFn) {
-  // This code relies on conversions in DynamicValue::Reader::as<T>.
   switch(type.which()) {
     case schema::Type::VOID:
-      break;
+      return capnp::VOID;
     case schema::Type::BOOL:
-      switch (value.which()) {
+      switch (input.which()) {
         case JsonValue::BOOLEAN:
-          setFn(value.getBoolean());
-          break;
+          return input.getBoolean();
         default:
           KJ_FAIL_REQUIRE("Expected boolean value");
       }
-      break;
     case schema::Type::INT8:
     case schema::Type::INT16:
     case schema::Type::INT32:
     case schema::Type::INT64:
       // Relies on range check in DynamicValue::Reader::as<IntType>
-      switch (value.which()) {
+      switch (input.which()) {
         case JsonValue::NUMBER:
-          setFn(value.getNumber());
-          break;
+          return input.getNumber();
         case JsonValue::STRING:
-          setFn(value.getString().parseAs<int64_t>());
-          break;
+          return input.getString().parseAs<int64_t>();
         default:
           KJ_FAIL_REQUIRE("Expected integer value");
       }
-      break;
     case schema::Type::UINT8:
     case schema::Type::UINT16:
     case schema::Type::UINT32:
     case schema::Type::UINT64:
       // Relies on range check in DynamicValue::Reader::as<IntType>
-      switch (value.which()) {
+      switch (input.which()) {
         case JsonValue::NUMBER:
-          setFn(value.getNumber());
-          break;
+          return input.getNumber();
         case JsonValue::STRING:
-          setFn(value.getString().parseAs<uint64_t>());
-          break;
+          return input.getString().parseAs<uint64_t>();
         default:
           KJ_FAIL_REQUIRE("Expected integer value");
       }
-      break;
     case schema::Type::FLOAT32:
     case schema::Type::FLOAT64:
-      switch (value.which()) {
+      switch (input.which()) {
         case JsonValue::NULL_:
-          setFn(kj::nan());
-          break;
+          return kj::nan();
         case JsonValue::NUMBER:
-          setFn(value.getNumber());
-          break;
+          return input.getNumber();
         case JsonValue::STRING:
-          setFn(value.getString().parseAs<double>());
-          break;
+          return input.getString().parseAs<double>();
         default:
           KJ_FAIL_REQUIRE("Expected float value");
       }
-      break;
     case schema::Type::TEXT:
-      switch (value.which()) {
+      switch (input.which()) {
         case JsonValue::STRING:
-          setFn(value.getString());
-          break;
+          return orphanage.newOrphanCopy(input.getString());
         default:
           KJ_FAIL_REQUIRE("Expected text value");
       }
-      break;
     case schema::Type::DATA:
-      switch (value.which()) {
+      switch (input.which()) {
         case JsonValue::ARRAY: {
-          auto array = value.getArray();
-          kj::Vector<byte> data(array.size());
-          for (auto arrayObject : array) {
-            auto x = arrayObject.getNumber();
+          auto array = input.getArray();
+          auto orphan = orphanage.newOrphan<Data>(array.size());
+          auto data = orphan.get();
+          for (auto i: kj::indices(array)) {
+            auto x = array[i].getNumber();
             KJ_REQUIRE(byte(x) == x, "Number in byte array is not an integer in [0, 255]");
-            data.add(byte(x));
+            data[i] = x;
           }
-          setFn(Data::Reader(data.asPtr()));
-          break;
+          return orphan;
         }
         default:
           KJ_FAIL_REQUIRE("Expected data value");
       }
-      break;
     case schema::Type::LIST:
-      switch (value.which()) {
-        case JsonValue::NULL_:
-          // nothing to do
-          break;
+      switch (input.which()) {
         case JsonValue::ARRAY:
-          decodeArrayFn(value.getArray());
-          break;
+          return decodeArray(input.getArray(), type.asList(), orphanage);
         default:
           KJ_FAIL_REQUIRE("Expected list value");
       }
-      break;
     case schema::Type::ENUM:
-      switch (value.which()) {
+      switch (input.which()) {
         case JsonValue::STRING:
-          setFn(value.getString());
-          break;
+          return DynamicEnum(type.asEnum().getEnumerantByName(input.getString()));
         default:
           KJ_FAIL_REQUIRE("Expected enum value");
       }
-      break;
-    case schema::Type::STRUCT:
-      switch (value.which()) {
-        case JsonValue::NULL_:
-          // nothing to do
-          break;
-        case JsonValue::OBJECT:
-          decodeObjectFn(value.getObject());
-          break;
-        default:
-          KJ_FAIL_REQUIRE("Expected object value");
-      }
-      break;
+    case schema::Type::STRUCT: {
+      auto structType = type.asStruct();
+      auto orphan = orphanage.newOrphan(structType);
+      decodeObject(input, structType, orphanage, orphan.get());
+      return orphan;
+    }
     case schema::Type::INTERFACE:
       KJ_FAIL_REQUIRE("don't know how to JSON-decode capabilities; "
-                      "JsonCodec::Handler not implemented yet :(");
+                      "please register a JsonCodec::Handler for this");
     case schema::Type::ANY_POINTER:
       KJ_FAIL_REQUIRE("don't know how to JSON-decode AnyPointer; "
-                      "JsonCodec::Handler not implemented yet :(");
-  }
-}
-} // namespace
-
-void JsonCodec::decodeArray(List<JsonValue>::Reader input, DynamicList::Builder output) const {
-  KJ_ASSERT(input.size() == output.size(), "Builder was not initialized to input size");
-  auto type = output.getSchema().getElementType();
-  for (auto i = 0; i < input.size(); i++) {
-    decodeField(type, input[i],
-        [&](DynamicValue::Reader value) { output.set(i, value); },
-        [&](List<JsonValue>::Reader array) {
-          decodeArray(array, output.init(i, array.size()).as<DynamicList>());
-        },
-        [&](List<JsonValue::Field>::Reader object) {
-          decodeObject(object, output[i].as<DynamicStruct>());
-        });
-  }
-}
-
-void JsonCodec::decodeObject(List<JsonValue::Field>::Reader input, DynamicStruct::Builder output)
-    const {
-  for (auto field : input) {
-    KJ_IF_MAYBE(fieldSchema, output.getSchema().findFieldByName(field.getName())) {
-      decodeField((*fieldSchema).getType(), field.getValue(),
-          [&](DynamicValue::Reader value) { output.set(*fieldSchema, value); },
-          [&](List<JsonValue>::Reader array) {
-            decodeArray(array, output.init(*fieldSchema, array.size()).as<DynamicList>());
-          },
-          [&](List<JsonValue::Field>::Reader object) {
-            decodeObject(object, output.init(*fieldSchema).as<DynamicStruct>());
-          });
-    } else {
-      // Unknown json fields are ignored to allow schema evolution
-    }
+                      "please register a JsonCodec::Handler for this");
   }
 }
 
-void JsonCodec::decode(JsonValue::Reader input, DynamicStruct::Builder output) const {
-  // TODO(soon): type and field handlers
-  switch (input.which()) {
-    case JsonValue::OBJECT:
-      decodeObject(input.getObject(), output);
-      break;
-    default:
-      KJ_FAIL_REQUIRE("Top level json value must be object");
-  };
-}
-
-Orphan<DynamicValue> JsonCodec::decode(
-    JsonValue::Reader input, Type type, Orphanage orphanage) const {
-  // TODO(soon)
-  KJ_FAIL_ASSERT("JSON decode into orphanage not implement yet. :(");
-}
-
 // -----------------------------------------------------------------------------
 
 namespace {

c++/src/capnp/compat/json.h

@@ -220,8 +220,8 @@ private:
 
   void encodeField(StructSchema::Field field, DynamicValue::Reader input,
                    JsonValue::Builder output) const;
-  void decodeArray(List<JsonValue>::Reader input, DynamicList::Builder output) const;
-  void decodeObject(List<JsonValue::Field>::Reader input, DynamicStruct::Builder output) const;
+  Orphan<DynamicList> decodeArray(List<JsonValue>::Reader input, ListSchema type, Orphanage orphanage) const;
+  void decodeObject(JsonValue::Reader input, StructSchema type, Orphanage orphanage, DynamicStruct::Builder output) const;
   void addTypeHandlerImpl(Type type, HandlerBase& handler);
   void addFieldHandlerImpl(StructSchema::Field field, Type type, HandlerBase& handler);
 };