Unverified Commit d76ba885 authored by Kenton Varda's avatar Kenton Varda Committed by GitHub

Merge pull request #700 from capnproto/json-annotations

Implement annotations to control JSON parsing.
parents 0fdad20a 0591a020
......@@ -380,7 +380,8 @@ endif LITE_MODE
test_capnpc_inputs = \
src/capnp/test.capnp \
src/capnp/test-import.capnp \
src/capnp/test-import2.capnp
src/capnp/test-import2.capnp \
src/capnp/compat/json-test.capnp
test_capnpc_outputs = \
src/capnp/test.capnp.c++ \
......@@ -388,7 +389,9 @@ test_capnpc_outputs = \
src/capnp/test-import.capnp.c++ \
src/capnp/test-import.capnp.h \
src/capnp/test-import2.capnp.c++ \
src/capnp/test-import2.capnp.h
src/capnp/test-import2.capnp.h \
src/capnp/compat/json-test.capnp.c++ \
src/capnp/compat/json-test.capnp.h
if USE_EXTERNAL_CAPNP
......
......@@ -192,6 +192,7 @@ if(BUILD_TESTING)
test.capnp
test-import.capnp
test-import2.capnp
compat/json-test.capnp
)
set(CAPNPC_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/test_capnp")
......
......@@ -22,6 +22,7 @@
#include "json.h"
#include <capnp/test-util.h>
#include <capnp/compat/json.capnp.h>
#include <capnp/compat/json-test.capnp.h>
#include <kj/debug.h>
#include <kj/string.h>
#include <kj/test.h>
......@@ -822,6 +823,147 @@ KJ_TEST("register capability handler") {
json.addTypeHandler(handler);
}
static constexpr kj::StringPtr GOLDEN_ANNOTATED =
R"({ "names-can_contain!anything Really": "foo",
"flatFoo": 123,
"flatBar": "abc",
"renamed-flatBaz": {"hello": true},
"flatQux": "cba",
"pfx.foo": "this is a long string in order to force multi-line pretty printing",
"pfx.renamed-bar": 321,
"pfx.baz": {"hello": true},
"pfx.xfp.qux": "fed",
"union-type": "renamed-bar",
"barMember": 789,
"multiMember": "ghi",
"dependency": {"renamed-foo": "corge"},
"simpleGroup": {"renamed-grault": "garply"},
"enums": ["qux", "renamed-bar", "foo", "renamed-baz"],
"innerJson": [123, "hello", {"object": true}],
"customFieldHandler": "add-prefix-waldo",
"testBase64": "ZnJlZA==",
"testHex": "706c756768",
"bUnion": "renamed-bar",
"bValue": 678 })"_kj;
static constexpr kj::StringPtr GOLDEN_ANNOTATED_REVERSE =
R"({
"bValue": 678,
"bUnion": "renamed-bar",
"testHex": "706c756768",
"testBase64": "ZnJlZA==",
"customFieldHandler": "add-prefix-waldo",
"innerJson": [123, "hello", {"object": true}],
"enums": ["qux", "renamed-bar", "foo", "renamed-baz"],
"simpleGroup": { "renamed-grault": "garply" },
"dependency": { "renamed-foo": "corge" },
"multiMember": "ghi",
"barMember": 789,
"union-type": "renamed-bar",
"pfx.xfp.qux": "fed",
"pfx.baz": {"hello": true},
"pfx.renamed-bar": 321,
"pfx.foo": "this is a long string in order to force multi-line pretty printing",
"flatQux": "cba",
"renamed-flatBaz": {"hello": true},
"flatBar": "abc",
"flatFoo": 123,
"names-can_contain!anything Really": "foo"
})"_kj;
class PrefixAdder: public JsonCodec::Handler<capnp::Text> {
public:
void encode(const JsonCodec& codec, capnp::Text::Reader input, JsonValue::Builder output) const {
output.setString(kj::str("add-prefix-", input));
}
Orphan<capnp::Text> decode(const JsonCodec& codec, JsonValue::Reader input,
Orphanage orphanage) const {
return orphanage.newOrphanCopy(capnp::Text::Reader(input.getString().slice(11)));
}
};
KJ_TEST("rename fields") {
JsonCodec json;
json.handleByAnnotation<TestJsonAnnotations>();
json.setPrettyPrint(true);
PrefixAdder customHandler;
json.addFieldHandler(Schema::from<TestJsonAnnotations>().getFieldByName("customFieldHandler"),
customHandler);
kj::String goldenText;
{
MallocMessageBuilder message;
auto root = message.getRoot<TestJsonAnnotations>();
root.setSomeField("foo");
auto aGroup = root.getAGroup();
aGroup.setFlatFoo(123);
aGroup.setFlatBar("abc");
aGroup.getFlatBaz().setHello(true);
aGroup.getDoubleFlat().setFlatQux("cba");
auto prefixedGroup = root.getPrefixedGroup();
prefixedGroup.setFoo("this is a long string in order to force multi-line pretty printing");
prefixedGroup.setBar(321);
prefixedGroup.getBaz().setHello(true);
prefixedGroup.getMorePrefix().setQux("fed");
auto unionBar = root.getAUnion().initBar();
unionBar.setBarMember(789);
unionBar.setMultiMember("ghi");
root.initDependency().setFoo("corge");
root.initSimpleGroup().setGrault("garply");
root.setEnums({
TestJsonAnnotatedEnum::QUX,
TestJsonAnnotatedEnum::BAR,
TestJsonAnnotatedEnum::FOO,
TestJsonAnnotatedEnum::BAZ
});
auto val = root.initInnerJson();
auto arr = val.initArray(3);
arr[0].setNumber(123);
arr[1].setString("hello");
auto field = arr[2].initObject(1)[0];
field.setName("object");
field.initValue().setBoolean(true);
root.setCustomFieldHandler("waldo");
root.setTestBase64("fred"_kj.asBytes());
root.setTestHex("plugh"_kj.asBytes());
root.getBUnion().setBar(678);
auto encoded = json.encode(root.asReader());
KJ_EXPECT(encoded == GOLDEN_ANNOTATED, encoded);
goldenText = kj::str(root);
}
{
MallocMessageBuilder message;
auto root = message.getRoot<TestJsonAnnotations>();
json.decode(GOLDEN_ANNOTATED, root);
KJ_EXPECT(kj::str(root) == goldenText, root, goldenText);
}
{
// Try parsing in reverse, mostly to test that union tags can come after content.
MallocMessageBuilder message;
auto root = message.getRoot<TestJsonAnnotations>();
json.decode(GOLDEN_ANNOTATED_REVERSE, root);
KJ_EXPECT(kj::str(root) == goldenText, root, goldenText);
}
}
} // namespace
} // namespace _ (private)
} // namespace capnp
# Copyright (c) 2018 Cloudflare, Inc. and contributors
# Licensed under the MIT License:
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
@0xc9d405cf4333e4c9;
using Json = import "/capnp/compat/json.capnp";
$import "/capnp/c++.capnp".namespace("capnp");
struct TestJsonAnnotations {
someField @0 :Text $Json.name("names-can_contain!anything Really");
aGroup :group $Json.flatten() {
flatFoo @1 :UInt32;
flatBar @2 :Text;
flatBaz :group $Json.name("renamed-flatBaz") {
hello @3 :Bool;
}
doubleFlat :group $Json.flatten() {
flatQux @4 :Text;
}
}
prefixedGroup :group $Json.flatten(prefix = "pfx.") {
foo @5 :Text;
bar @6 :UInt32 $Json.name("renamed-bar");
baz :group {
hello @7 :Bool;
}
morePrefix :group $Json.flatten(prefix = "xfp.") {
qux @8 :Text;
}
}
aUnion :union $Json.flatten() $Json.discriminator(name = "union-type") {
foo :group $Json.flatten() {
fooMember @9 :Text;
multiMember @10 :UInt32;
}
bar :group $Json.flatten() $Json.name("renamed-bar") {
barMember @11 :UInt32;
multiMember @12 :Text;
}
}
dependency @13 :TestJsonAnnotations2;
# To test that dependencies are loaded even if not flattened.
simpleGroup :group {
# To test that group types are loaded even if not flattened.
grault @14 :Text $Json.name("renamed-grault");
}
enums @15 :List(TestJsonAnnotatedEnum);
innerJson @16 :Json.Value;
customFieldHandler @17 :Text;
testBase64 @18 :Data $Json.base64;
testHex @19 :Data $Json.hex;
bUnion :union $Json.flatten() $Json.discriminator(valueName = "bValue") {
foo @20 :Text;
bar @21 :UInt32 $Json.name("renamed-bar");
}
}
struct TestJsonAnnotations2 {
foo @0 :Text $Json.name("renamed-foo");
cycle @1 :TestJsonAnnotations;
}
enum TestJsonAnnotatedEnum {
foo @0;
bar @1 $Json.name("renamed-bar");
baz @2 $Json.name("renamed-baz");
qux @3;
}
# TODO(now): enums
This diff is collapsed.
......@@ -21,15 +21,15 @@
@0x8ef99297a43a5e34;
$import "/capnp/c++.capnp".namespace("capnp");
$import "/capnp/c++.capnp".namespace("capnp::json");
struct JsonValue {
struct Value {
union {
null @0 :Void;
boolean @1 :Bool;
number @2 :Float64;
string @3 :Text;
array @4 :List(JsonValue);
array @4 :List(Value);
object @5 :List(Field);
# Standard JSON values.
......@@ -48,11 +48,65 @@ struct JsonValue {
struct Field {
name @0 :Text;
value @1 :JsonValue;
value @1 :Value;
}
struct Call {
function @0 :Text;
params @1 :List(JsonValue);
params @1 :List(Value);
}
}
# ========================================================================================
# Annotations to control parsing. Typical usage:
#
# using Json = import "/capnp/compat/json.capnp";
#
# And then later on:
#
# myField @0 :Text $Json.name("my_field");
annotation name @0xfa5b1fd61c2e7c3d (field, enumerant, method, group, union): Text;
# Define an alternative name to use when encoding the given item in JSON. This can be used, for
# example, to use snake_case names where needed, even though Cap'n Proto uses strictly camelCase.
#
# (However, because JSON is derived from JavaScript, you *should* use camelCase names when
# defining JSON-based APIs. But, when supporting a pre-existing API you may not have a choice.)
annotation flatten @0x82d3e852af0336bf (field, group, union): FlattenOptions;
# Specifies that an aggregate field should be flattened into its parent.
#
# In order to flatten a member of a union, the union (or, for an anonymous union, the parent
# struct type) must have the $jsonDiscriminator annotation.
#
# TODO(someday): Maybe support "flattening" a List(Value.Field) as a way to support unknown JSON
# fields?
struct FlattenOptions {
prefix @0 :Text = "";
# Optional: Adds the given prefix to flattened field names.
}
annotation discriminator @0xcfa794e8d19a0162 (struct, union): DiscriminatorOptions;
# Specifies that a union's variant will be decided not by which fields are present, but instead
# by a special discriminator field. The value of the discriminator field is a string naming which
# variant is active. This allows the members of the union to have the $jsonFlatten annotation, or
# to all have the same name.
struct DiscriminatorOptions {
name @0 :Text;
# The name of the discriminator field. Defaults to matching the name of the union.
valueName @1 :Text;
# If non-null, specifies that the union's value shall have the given field name, rather than the
# value's name. In this case the union's variant can only be determined by looking at the
# discriminant field, not by inspecting which value field is present.
#
# It is an error to use `valueName` while also declaring some variants as $flatten.
}
annotation base64 @0xd7d879450a253e4b (field): Void;
# Place on a field of type `Data` to indicate that its JSON representation is a Base64 string.
annotation hex @0xf061e22f0ae5c7b5 (field): Void;
# Place on a field of type `Data` to indicate that its JSON representation is a hex string.
This diff is collapsed.
This diff is collapsed.
......@@ -27,6 +27,11 @@
namespace capnp {
typedef json::Value JsonValue;
// For backwards-compatibility.
//
// TODO(cleanup): Consider replacing all uses of JsonValue with json::Value?
class JsonCodec {
// Flexible class for encoding Cap'n Proto types as JSON, and decoding JSON back to Cap'n Proto.
//
......@@ -198,6 +203,16 @@ public:
void addFieldHandler(StructSchema::Field field, Handler<T>& handler);
// Matches only the specific field. T can be a dynamic type. T must match the field's type.
void handleByAnnotation(Schema schema);
template <typename T> void handleByAnnotation();
// Inspects the given type (as specified by type parameter or dynamic schema) and all its
// dependencies looking for JSON annotations (see json.capnp), building and registering Handlers
// based on these annotations.
//
// If you'd like to use annotations to control JSON, you must call these functions before you
// start using the codec. They are not loaded "on demand" because that would require mutex
// locking.
// ---------------------------------------------------------------------------
// Hack to support string literal parameters
......@@ -214,6 +229,11 @@ public:
private:
class HandlerBase;
class AnnotatedHandler;
class AnnotatedEnumHandler;
class Base64Handler;
class HexHandler;
class JsonValueHandler;
struct Impl;
kj::Own<Impl> impl;
......@@ -222,8 +242,16 @@ private:
JsonValue::Builder output) const;
Orphan<DynamicList> decodeArray(List<JsonValue>::Reader input, ListSchema type, Orphanage orphanage) const;
void decodeObject(JsonValue::Reader input, StructSchema type, Orphanage orphanage, DynamicStruct::Builder output) const;
void decodeField(StructSchema::Field fieldSchema, JsonValue::Reader fieldValue,
Orphanage orphanage, DynamicStruct::Builder output) const;
void addTypeHandlerImpl(Type type, HandlerBase& handler);
void addFieldHandlerImpl(StructSchema::Field field, Type type, HandlerBase& handler);
AnnotatedHandler& loadAnnotatedHandler(
StructSchema schema,
kj::Maybe<json::DiscriminatorOptions::Reader> discriminator,
kj::Maybe<kj::StringPtr> unionDeclName,
kj::Vector<Schema>& dependencies);
};
// =======================================================================================
......@@ -480,4 +508,9 @@ template <> void JsonCodec::addTypeHandler(Handler<DynamicCapability>& handler)
// TODO(someday): Implement support for registering handlers that cover thinsg like "all structs"
// or "all lists". Currently you can only target a specific struct or list type.
template <typename T>
void JsonCodec::handleByAnnotation() {
return handleByAnnotation(Schema::from<T>());
}
} // namespace capnp
......@@ -38,7 +38,9 @@ else
fi
SCHEMA=`dirname "$0"`/../test.capnp
JSON_SCHEMA=`dirname "$0"`/../compat/json-test.capnp
TESTDATA=`dirname "$0"`/../testdata
SRCDIR=`dirname "$0"`/../..
SUFFIX=${TESTDATA#*/src/}
PREFIX=${TESTDATA%${SUFFIX}}
......@@ -74,6 +76,9 @@ $CAPNP convert binary:json --short $SCHEMA TestAllTypes < $TESTDATA/binary | cmp
$CAPNP convert json:binary $SCHEMA TestAllTypes < $TESTDATA/pretty.json | cmp $TESTDATA/binary - || fail json to binary
$CAPNP convert json:binary $SCHEMA TestAllTypes < $TESTDATA/short.json | cmp $TESTDATA/binary - || fail short json to binary
$CAPNP convert json:binary $JSON_SCHEMA TestJsonAnnotations -I"$SRCDIR" < $TESTDATA/annotated.json | cmp $TESTDATA/annotated-json.binary || fail annotated json to binary
$CAPNP convert binary:json $JSON_SCHEMA TestJsonAnnotations -I"$SRCDIR" < $TESTDATA/annotated-json.binary | cmp $TESTDATA/annotated.json || fail annotated json to binary
# ========================================================================================
# DEPRECATED encode/decode
......
......@@ -714,6 +714,13 @@ public:
return kj::str("unknown format: ", to);
}
if (convertFrom == Format::JSON || convertTo == Format::JSON) {
// We need annotations to process JSON.
// TODO(someday): Find a way that we can process annotations from json.capnp without
// requiring other annotation-only imports like c++.capnp
annotationFlag = Compiler::COMPILE_ANNOTATIONS;
}
return true;
} else {
return "invalid conversion, format is: <from>:<to>";
......@@ -1038,6 +1045,7 @@ private:
MallocMessageBuilder message;
JsonCodec codec;
codec.setPrettyPrint(pretty);
codec.handleByAnnotation(rootType);
auto root = message.initRoot<DynamicStruct>(rootType);
codec.decode(text, root);
return writeConversion(root.asReader(), output);
......@@ -1096,6 +1104,7 @@ private:
case Format::JSON: {
JsonCodec codec;
codec.setPrettyPrint(pretty);
codec.handleByAnnotation(rootType);
auto text = codec.encode(reader.as<DynamicStruct>(rootType));
output.write({text.asBytes(), kj::StringPtr("\n").asBytes()});
return;
......
......@@ -262,7 +262,7 @@ public:
template <typename T, typename = kj::EnableIf<kind<FromBuilder<T>>() == Kind::STRUCT>>
inline Builder(T&& value): Builder(toDynamic(value)) {}
inline operator AnyStruct::Reader() { return AnyStruct::Builder(builder); }
inline operator AnyStruct::Builder() { return AnyStruct::Builder(builder); }
inline MessageSize totalSize() const { return asReader().totalSize(); }
......
......@@ -35,6 +35,7 @@
#include "any.h"
#include <kj/string.h>
#include <kj/string-tree.h>
#include <kj/hash.h>
namespace capnp {
......
......@@ -25,6 +25,12 @@
namespace capnp {
namespace schema {
uint KJ_HASHCODE(Type::Which w) { return kj::hashCode(static_cast<uint16_t>(w)); }
// TODO(cleanup): Cap'n Proto does not declare stringifiers nor hashers for `Which` enums, unlike
// all other enums. Fix that and remove this.
}
namespace _ { // private
// Null schemas generated using the below schema file with:
......@@ -868,7 +874,7 @@ bool Type::operator==(const Type& other) const {
KJ_UNREACHABLE;
}
size_t Type::hashCode() const {
uint Type::hashCode() const {
switch (baseType) {
case schema::Type::VOID:
case schema::Type::BOOL:
......@@ -884,12 +890,12 @@ size_t Type::hashCode() const {
case schema::Type::FLOAT64:
case schema::Type::TEXT:
case schema::Type::DATA:
return (static_cast<size_t>(baseType) << 3) + listDepth;
return kj::hashCode(baseType, listDepth);
case schema::Type::STRUCT:
case schema::Type::ENUM:
case schema::Type::INTERFACE:
return reinterpret_cast<size_t>(schema) + listDepth;
return kj::hashCode(schema, listDepth);
case schema::Type::LIST:
KJ_UNREACHABLE;
......@@ -897,9 +903,9 @@ size_t Type::hashCode() const {
case schema::Type::ANY_POINTER: {
// Trying to comply with strict aliasing rules. Hopefully the compiler realizes that
// both branches compile to the same instructions and can optimize it away.
size_t val = scopeId != 0 || isImplicitParam ?
uint16_t val = scopeId != 0 || isImplicitParam ?
paramIndex : static_cast<uint16_t>(anyPointerKind);
return (val << 1 | isImplicitParam) ^ scopeId;
return kj::hashCode(val, isImplicitParam, scopeId);
}
}
......
......@@ -30,6 +30,7 @@
#endif
#include <capnp/schema.capnp.h>
#include <kj/hash.h>
#include <kj/windows-sanity.h> // work-around macro conflict with `VOID`
namespace capnp {
......@@ -129,6 +130,8 @@ public:
// you want to check if two Schemas represent the same type (but possibly different versions of
// it), compare their IDs instead.
inline uint hashCode() const { return kj::hashCode(raw); }
template <typename T>
void requireUsableAs() const;
// Throws an exception if a value with this Schema cannot safely be cast to a native value of
......@@ -302,6 +305,7 @@ public:
inline bool operator==(const Field& other) const;
inline bool operator!=(const Field& other) const { return !(*this == other); }
inline uint hashCode() const;
private:
StructSchema parent;
......@@ -400,6 +404,7 @@ public:
inline bool operator==(const Enumerant& other) const;
inline bool operator!=(const Enumerant& other) const { return !(*this == other); }
inline uint hashCode() const;
private:
EnumSchema parent;
......@@ -492,6 +497,7 @@ public:
inline bool operator==(const Method& other) const;
inline bool operator!=(const Method& other) const { return !(*this == other); }
inline uint hashCode() const;
private:
InterfaceSchema parent;
......@@ -644,7 +650,7 @@ public:
bool operator==(const Type& other) const;
inline bool operator!=(const Type& other) const { return !(*this == other); }
size_t hashCode() const;
uint hashCode() const;
inline Type wrapInList(uint depth = 1) const;
// Return the Type formed by wrapping this type in List() `depth` times.
......@@ -796,6 +802,16 @@ inline bool InterfaceSchema::Method::operator==(const Method& other) const {
return parent == other.parent && ordinal == other.ordinal;
}
inline uint StructSchema::Field::hashCode() const {
return kj::hashCode(parent, index);
}
inline uint EnumSchema::Enumerant::hashCode() const {
return kj::hashCode(parent, ordinal);
}
inline uint InterfaceSchema::Method::hashCode() const {
return kj::hashCode(parent, ordinal);
}
inline ListSchema ListSchema::of(StructSchema elementType) {
return ListSchema(Type(elementType));
}
......
{ "names-can_contain!anything Really": "foo",
"flatFoo": 123,
"flatBar": "abc",
"renamed-flatBaz": {"hello": true},
"flatQux": "cba",
"pfx.foo": "this is a long string in order to force multi-line pretty printing",
"pfx.renamed-bar": 321,
"pfx.baz": {"hello": true},
"pfx.xfp.qux": "fed",
"union-type": "renamed-bar",
"barMember": 789,
"multiMember": "ghi",
"dependency": {"renamed-foo": "corge"},
"simpleGroup": {"renamed-grault": "garply"},
"enums": ["qux", "renamed-bar", "foo", "renamed-baz"],
"innerJson": [123, "hello", {"object": true}],
"testBase64": "ZnJlZA==",
"testHex": "706c756768",
"bUnion": "renamed-bar",
"bValue": 678 }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment