Update json schema suite and add perf test

bin/jsonschema/README.md

@@ -60,21 +60,80 @@ Who Uses the Test Suite
 
 This suite is being used by:
 
-  * [json-schema-validator (Java)](https://github.com/fge/json-schema-validator)
-  * [jsonschema (python)](https://github.com/Julian/jsonschema)
-  * [aeson-schema (haskell)](https://github.com/timjb/aeson-schema)
-  * [direct-schema (javascript)](https://github.com/IreneKnapp/direct-schema)
-  * [jsonschema (javascript)](https://github.com/tdegrunt/jsonschema)
-  * [JaySchema (javascript)](https://github.com/natesilva/jayschema)
-  * [z-schema (javascript)](https://github.com/zaggino/z-schema)
-  * [jassi (javascript)](https://github.com/iclanzan/jassi)
-  * [json-schema-valid (javascript)](https://github.com/ericgj/json-schema-valid)
-  * [jesse (Erlang)](https://github.com/klarna/jesse)
-  * [json-schema (PHP)](https://github.com/justinrainbow/json-schema)
-  * [gojsonschema (Go)](https://github.com/sigu-399/gojsonschema) 
-  * [json_schema (Dart)](https://github.com/patefacio/json_schema) 
-  * [tv4 (JavaScript)](https://github.com/geraintluff/tv4)
-  * [Jsonary (JavaScript)](https://github.com/jsonary-js/jsonary)
+### Coffeescript ###
+
+* [jsck](https://github.com/pandastrike/jsck)
+
+### Dart ###
+
+* [json_schema](https://github.com/patefacio/json_schema) 
+
+### Erlang ###
+
+* [jesse](https://github.com/klarna/jesse)
+
+### Go ###
+
+* [gojsonschema](https://github.com/sigu-399/gojsonschema) 
+* [validate-json](https://github.com/cesanta/validate-json)
+
+### Haskell ###
+
+* [aeson-schema](https://github.com/timjb/aeson-schema)
+* [hjsonschema](https://github.com/seagreen/hjsonschema)
+
+### Java ###
+
+* [json-schema-validator](https://github.com/fge/json-schema-validator)
+
+### JavaScript ###
+
+* [json-schema-benchmark](https://github.com/Muscula/json-schema-benchmark)
+* [direct-schema](https://github.com/IreneKnapp/direct-schema)
+* [is-my-json-valid](https://github.com/mafintosh/is-my-json-valid)
+* [jassi](https://github.com/iclanzan/jassi)
+* [JaySchema](https://github.com/natesilva/jayschema)
+* [json-schema-valid](https://github.com/ericgj/json-schema-valid)
+* [Jsonary](https://github.com/jsonary-js/jsonary)
+* [jsonschema](https://github.com/tdegrunt/jsonschema)
+* [request-validator](https://github.com/bugventure/request-validator)
+* [skeemas](https://github.com/Prestaul/skeemas)
+* [tv4](https://github.com/geraintluff/tv4)
+* [z-schema](https://github.com/zaggino/z-schema)
+* [jsen](https://github.com/bugventure/jsen)
+* [ajv](https://github.com/epoberezkin/ajv)
+
+### Node.js ###
+
+The JSON Schema Test Suite is also available as an
+[npm](https://www.npmjs.com/package/json-schema-test-suite) package.
+Node-specific support is maintained on the [node branch](https://github.com/json-schema/JSON-Schema-Test-Suite/tree/node).
+See [NODE-README.md](https://github.com/json-schema/JSON-Schema-Test-Suite/blob/node/NODE-README.md)
+for more information.
+
+### .NET ###
+
+* [Newtonsoft.Json.Schema](https://github.com/JamesNK/Newtonsoft.Json.Schema)
+
+### PHP ###
+
+* [json-schema](https://github.com/justinrainbow/json-schema)
+
+### Python ###
+
+* [jsonschema](https://github.com/Julian/jsonschema)
+
+### Ruby ###
+
+* [json-schema](https://github.com/hoxworth/json-schema)
+
+### Rust ###
+
+* [valico](https://github.com/rustless/valico)
+
+### Swift ###
+
+* [JSONSchema](https://github.com/kylef/JSONSchema.swift)
 
 If you use it as well, please fork and send a pull request adding yourself to
 the list :).
@@ -85,5 +144,5 @@ Contributing
 If you see something missing or incorrect, a pull request is most welcome!
 
 There are some sanity checks in place for testing the test suite. You can run
-them with `bin/jsonschema_suite check`. They will be run automatically by
+them with `bin/jsonschema_suite check` or `tox`. They will be run automatically by
 [Travis CI](https://travis-ci.org/) as well.

bin/jsonschema/tests/draft3/additionalProperties.json

@@ -55,6 +55,25 @@
             }
         ]
     },
+    {
+        "description":
+            "additionalProperties can exist by itself",
+        "schema": {
+            "additionalProperties": {"type": "boolean"}
+        },
+        "tests": [
+            {
+                "description": "an additional valid property is valid",
+                "data": {"foo" : true},
+                "valid": true
+            },
+            {
+                "description": "an additional invalid property is invalid",
+                "data": {"foo" : 1},
+                "valid": false
+            }
+        ]
+    },
     {
         "description": "additionalProperties are allowed by default",
         "schema": {"properties": {"foo": {}, "bar": {}}},

bin/jsonschema/tests/draft3/default.json

+[
+    {
+        "description": "invalid type for default",
+        "schema": {
+            "properties": {
+                "foo": {
+                    "type": "integer",
+                    "default": []
+                }
+            }
+        },
+        "tests": [
+            {
+                "description": "valid when property is specified",
+                "data": {"foo": 13},
+                "valid": true
+            },
+            {
+                "description": "still valid when the invalid default is used",
+                "data": {},
+                "valid": true
+            }
+        ]
+    },
+    {
+        "description": "invalid string value for default",
+        "schema": {
+            "properties": {
+                "bar": {
+                    "type": "string",
+                    "minLength": 4,
+                    "default": "bad"
+                }
+            }
+        },
+        "tests": [
+            {
+                "description": "valid when property is specified",
+                "data": {"bar": "good"},
+                "valid": true
+            },
+            {
+                "description": "still valid when the invalid default is used",
+                "data": {},
+                "valid": true
+            }
+        ]
+    }
+]

bin/jsonschema/tests/draft3/optional/bignum.json

@@ -21,6 +21,28 @@
             }
         ]
     },
+    {
+        "description": "integer",
+        "schema": {"type": "integer"},
+        "tests": [
+            {
+                "description": "a negative bignum is an integer",
+                "data": -12345678910111213141516171819202122232425262728293031,
+                "valid": true
+            }
+        ]
+    },
+    {
+        "description": "number",
+        "schema": {"type": "number"},
+        "tests": [
+            {
+                "description": "a negative bignum is a number",
+                "data": -98249283749234923498293171823948729348710298301928331,
+                "valid": true
+            }
+        ]
+    },
     {
         "description": "string",
         "schema": {"type": "string"},
@@ -56,5 +78,30 @@
                 "valid": false
             }
         ]
+    },
+    {
+        "description": "integer comparison",
+        "schema": {"minimum": -18446744073709551615},
+        "tests": [
+            {
+                "description": "comparison works for very negative numbers",
+                "data": -18446744073709551600,
+                "valid": true
+            }
+        ]
+    },
+    {
+        "description": "float comparison with high precision on negative numbers",
+        "schema": {
+            "minimum": -972783798187987123879878123.18878137,
+            "exclusiveMinimum": true
+        },
+        "tests": [
+            {
+                "description": "comparison works for very negative numbers",
+                "data": -972783798187987123879878123.188781371,
+                "valid": false
+            }
+        ]
     }
 ]

bin/jsonschema/tests/draft3/optional/format.json

@@ -77,6 +77,11 @@
                 "data": "http://foo.bar/?baz=qux#quux",
                 "valid": true
             },
+            {
+                "description": "a valid protocol-relative URI",
+                "data": "//foo.bar/?baz=qux#quux",
+                "valid": true
+            },
             {
                 "description": "an invalid URI",
                 "data": "\\\\WINDOWS\\fileshare",

bin/jsonschema/tests/draft3/pattern.json

@@ -19,5 +19,16 @@
                 "valid": true
             }
         ]
+    },
+    {
+        "description": "pattern is not anchored",
+        "schema": {"pattern": "a+"},
+        "tests": [
+            {
+                "description": "matches a substring",
+                "data": "xxaayy",
+                "valid": true
+            }
+        ]
     }
 ]

bin/jsonschema/tests/draft3/ref.json

@@ -86,19 +86,34 @@
         },
         "tests": [
             {
-                "description": "slash",
+                "description": "slash invalid",
                 "data": {"slash": "aoeu"},
                 "valid": false
             },
             {
-                "description": "tilda",
+                "description": "tilda invalid",
                 "data": {"tilda": "aoeu"},
                 "valid": false
             },
             {
-                "description": "percent",
+                "description": "percent invalid",
                 "data": {"percent": "aoeu"},
                 "valid": false
+            },
+            {
+                "description": "slash valid",
+                "data": {"slash": 123},
+                "valid": true
+            },
+            {
+                "description": "tilda valid",
+                "data": {"tilda": 123},
+                "valid": true
+            },
+            {
+                "description": "percent valid",
+                "data": {"percent": 123},
+                "valid": true
             }
         ]
     },

bin/jsonschema/tests/draft3/type.json

@@ -188,7 +188,7 @@
                 "valid": false
             },
             {
-                "description": "an array is not an array",
+                "description": "an array is an array",
                 "data": [],
                 "valid": true
             },
@@ -234,7 +234,7 @@
                 "valid": false
             },
             {
-                "description": "a boolean is not a boolean",
+                "description": "a boolean is a boolean",
                 "data": true,
                 "valid": true
             },

bin/jsonschema/tests/draft4/additionalProperties.json

@@ -55,6 +55,25 @@
             }
         ]
     },
+    {
+        "description":
+            "additionalProperties can exist by itself",
+        "schema": {
+            "additionalProperties": {"type": "boolean"}
+        },
+        "tests": [
+            {
+                "description": "an additional valid property is valid",
+                "data": {"foo" : true},
+                "valid": true
+            },
+            {
+                "description": "an additional invalid property is invalid",
+                "data": {"foo" : 1},
+                "valid": false
+            }
+        ]
+    },
     {
         "description": "additionalProperties are allowed by default",
         "schema": {"properties": {"foo": {}, "bar": {}}},

bin/jsonschema/tests/draft4/default.json

+[
+    {
+        "description": "invalid type for default",
+        "schema": {
+            "properties": {
+                "foo": {
+                    "type": "integer",
+                    "default": []
+                }
+            }
+        },
+        "tests": [
+            {
+                "description": "valid when property is specified",
+                "data": {"foo": 13},
+                "valid": true
+            },
+            {
+                "description": "still valid when the invalid default is used",
+                "data": {},
+                "valid": true
+            }
+        ]
+    },
+    {
+        "description": "invalid string value for default",
+        "schema": {
+            "properties": {
+                "bar": {
+                    "type": "string",
+                    "minLength": 4,
+                    "default": "bad"
+                }
+            }
+        },
+        "tests": [
+            {
+                "description": "valid when property is specified",
+                "data": {"bar": "good"},
+                "valid": true
+            },
+            {
+                "description": "still valid when the invalid default is used",
+                "data": {},
+                "valid": true
+            }
+        ]
+    }
+]

bin/jsonschema/tests/draft4/maxLength.json

@@ -20,7 +20,7 @@
             },
             {
                 "description": "ignores non-strings",
-                "data": 10,
+                "data": 100,
                 "valid": true
             },
             {

bin/jsonschema/tests/draft4/optional/bignum.json

@@ -21,6 +21,28 @@
             }
         ]
     },
+    {
+        "description": "integer",
+        "schema": {"type": "integer"},
+        "tests": [
+            {
+                "description": "a negative bignum is an integer",
+                "data": -12345678910111213141516171819202122232425262728293031,
+                "valid": true
+            }
+        ]
+    },
+    {
+        "description": "number",
+        "schema": {"type": "number"},
+        "tests": [
+            {
+                "description": "a negative bignum is a number",
+                "data": -98249283749234923498293171823948729348710298301928331,
+                "valid": true
+            }
+        ]
+    },
     {
         "description": "string",
         "schema": {"type": "string"},
@@ -56,5 +78,30 @@
                 "valid": false
             }
         ]
+    },
+    {
+        "description": "integer comparison",
+        "schema": {"minimum": -18446744073709551615},
+        "tests": [
+            {
+                "description": "comparison works for very negative numbers",
+                "data": -18446744073709551600,
+                "valid": true
+            }
+        ]
+    },
+    {
+        "description": "float comparison with high precision on negative numbers",
+        "schema": {
+            "minimum": -972783798187987123879878123.18878137,
+            "exclusiveMinimum": true
+        },
+        "tests": [
+            {
+                "description": "comparison works for very negative numbers",
+                "data": -972783798187987123879878123.188781371,
+                "valid": false
+            }
+        ]
     }
 ]

bin/jsonschema/tests/draft4/optional/format.json

@@ -29,6 +29,11 @@
                 "data": "http://foo.bar/?baz=qux#quux",
                 "valid": true
             },
+            {
+                "description": "a valid protocol-relative URI",
+                "data": "//foo.bar/?baz=qux#quux",
+                "valid": true
+            },
             {
                 "description": "an invalid URI",
                 "data": "\\\\WINDOWS\\fileshare",

bin/jsonschema/tests/draft4/pattern.json

@@ -19,5 +19,16 @@
                 "valid": true
             }
         ]
+    },
+    {
+        "description": "pattern is not anchored",
+        "schema": {"pattern": "a+"},
+        "tests": [
+            {
+                "description": "matches a substring",
+                "data": "xxaayy",
+                "valid": true
+            }
+        ]
     }
 ]

bin/jsonschema/tests/draft4/ref.json

@@ -86,19 +86,34 @@
         },
         "tests": [
             {
-                "description": "slash",
+                "description": "slash invalid",
                 "data": {"slash": "aoeu"},
                 "valid": false
             },
             {
-                "description": "tilda",
+                "description": "tilda invalid",
                 "data": {"tilda": "aoeu"},
                 "valid": false
             },
             {
-                "description": "percent",
+                "description": "percent invalid",
                 "data": {"percent": "aoeu"},
                 "valid": false
+            },
+            {
+                "description": "slash valid",
+                "data": {"slash": 123},
+                "valid": true
+            },
+            {
+                "description": "tilda valid",
+                "data": {"tilda": 123},
+                "valid": true
+            },
+            {
+                "description": "percent valid",
+                "data": {"percent": 123},
+                "valid": true
             }
         ]
     },

bin/jsonschema/tests/draft4/type.json

@@ -188,7 +188,7 @@
                 "valid": false
             },
             {
-                "description": "an array is not an array",
+                "description": "an array is an array",
                 "data": [],
                 "valid": true
             },
@@ -234,7 +234,7 @@
                 "valid": false
             },
             {
-                "description": "a boolean is not a boolean",
+                "description": "a boolean is a boolean",
                 "data": true,
                 "valid": true
             },

bin/jsonschema/tox.ini

+[tox]
+minversion = 1.6
+envlist = py27
+skipsdist = True
+
+[testenv]
+deps = jsonschema
+commands = {envpython} bin/jsonschema_suite check

test/perftest/CMakeLists.txt

@@ -2,7 +2,8 @@ set(PERFTEST_SOURCES
     misctest.cpp
     perftest.cpp
     platformtest.cpp
-    rapidjsontest.cpp)
+    rapidjsontest.cpp
+    schematest.cpp)
 
 add_executable(perftest ${PERFTEST_SOURCES})
 target_link_libraries(perftest ${TEST_LIBRARIES})

test/perftest/schematest.cpp

+#include "perftest.h"
+
+#if TEST_RAPIDJSON
+
+#include "rapidjson/schema.h"
+#include <ctime>
+#include <string>
+#include <vector>
+
+#define ARRAY_SIZE(a) sizeof(a) / sizeof(a[0])
+
+using namespace rapidjson;
+
+template <typename Allocator>
+static char* ReadFile(const char* filename, Allocator& allocator) {
+    const char *paths[] = {
+        "",
+        "bin/",
+        "../bin/",
+        "../../bin/",
+        "../../../bin/"
+    };
+    char buffer[1024];
+    FILE *fp = 0;
+    for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) {
+        sprintf(buffer, "%s%s", paths[i], filename);
+        fp = fopen(buffer, "rb");
+        if (fp)
+            break;
+    }
+
+    if (!fp)
+        return 0;
+
+    fseek(fp, 0, SEEK_END);
+    size_t length = static_cast<size_t>(ftell(fp));
+    fseek(fp, 0, SEEK_SET);
+    char* json = reinterpret_cast<char*>(allocator.Malloc(length + 1));
+    size_t readLength = fread(json, 1, length, fp);
+    json[readLength] = '\0';
+    fclose(fp);
+    return json;
+}
+
+class Schema : public PerfTest {
+public:
+    Schema() {}
+
+    virtual void SetUp() {
+        PerfTest::SetUp();
+
+        const char* filenames[] = {
+            "additionalItems.json",
+            "additionalProperties.json",
+            "allOf.json",
+            "anyOf.json",
+            "default.json",
+            "definitions.json",
+            "dependencies.json",
+            "enum.json",
+            "items.json",
+            "maximum.json",
+            "maxItems.json",
+            "maxLength.json",
+            "maxProperties.json",
+            "minimum.json",
+            "minItems.json",
+            "minLength.json",
+            "minProperties.json",
+            "multipleOf.json",
+            "not.json",
+            "oneOf.json",
+            "pattern.json",
+            "patternProperties.json",
+            "properties.json",
+            "ref.json",
+            "refRemote.json",
+            "required.json",
+            "type.json",
+            "uniqueItems.json"
+        };
+
+        char jsonBuffer[65536];
+        MemoryPoolAllocator<> jsonAllocator(jsonBuffer, sizeof(jsonBuffer));
+
+        for (size_t i = 0; i < ARRAY_SIZE(filenames); i++) {
+            char filename[FILENAME_MAX];
+            sprintf(filename, "jsonschema/tests/draft4/%s", filenames[i]);
+            char* json = ReadFile(filename, jsonAllocator);
+            if (!json) {
+                printf("json test suite file %s not found", filename);
+                return;
+            }
+
+            Document d;
+            d.Parse(json);
+            if (d.HasParseError()) {
+                printf("json test suite file %s has parse error", filename);
+                return;
+            }
+
+            for (Value::ConstValueIterator schemaItr = d.Begin(); schemaItr != d.End(); ++schemaItr) {
+                if (IsExcludeTestSuite((*schemaItr)["description"].GetString()))
+                    continue;
+
+                TestSuite* ts = new TestSuite;
+                ts->schema = new SchemaDocument((*schemaItr)["schema"]);
+
+                const Value& tests = (*schemaItr)["tests"];
+                for (Value::ConstValueIterator testItr = tests.Begin(); testItr != tests.End(); ++testItr) {
+                    if (IsExcludeTest((*testItr)["description"].GetString()))
+                        continue;
+
+                    Document* d2 = new Document;
+                    d2->CopyFrom((*testItr)["data"], d2->GetAllocator());
+                    ts->tests.push_back(d2);
+                }
+                testSuites.push_back(ts);
+            }
+        }
+    }
+
+    virtual void TearDown() {
+        PerfTest::TearDown();
+        for (TestSuiteList::const_iterator itr = testSuites.begin(); itr != testSuites.end(); ++itr)
+            delete *itr;
+        testSuites.clear();
+    }
+
+private:
+    // Using the same exclusion in https://github.com/json-schema/JSON-Schema-Test-Suite
+    static bool IsExcludeTestSuite(const std::string& description) {
+        const char* excludeTestSuites[] = {
+            //lost failing these tests
+            "remote ref",
+            "remote ref, containing refs itself",
+            "fragment within remote ref",
+            "ref within remote ref",
+            "change resolution scope",
+            // these below were added to get jsck in the benchmarks)
+            "uniqueItems validation",
+            "valid definition",
+            "invalid definition"
+        };
+
+        for (size_t i = 0; i < ARRAY_SIZE(excludeTestSuites); i++)
+            if (excludeTestSuites[i] == description)
+                return true;
+        return false;
+    }
+
+    // Using the same exclusion in https://github.com/json-schema/JSON-Schema-Test-Suite
+    static bool IsExcludeTest(const std::string& description) {
+        const char* excludeTests[] = {
+            //lots of validators fail these
+            "invalid definition, invalid definition schema",
+            "maxLength validation, two supplementary Unicode code points is long enough",
+            "minLength validation, one supplementary Unicode code point is not long enough",
+            //this is to get tv4 in the benchmarks
+            "heterogeneous enum validation, something else is invalid"
+        };
+
+        for (size_t i = 0; i < ARRAY_SIZE(excludeTests); i++)
+            if (excludeTests[i] == description)
+                return true;
+        return false;
+    }
+
+    Schema(const Schema&);
+    Schema& operator=(const Schema&);
+
+protected:
+    typedef std::vector<Document*> DocumentList;
+
+    struct TestSuite {
+        TestSuite() : schema() {}
+        ~TestSuite() {
+            delete schema;
+            for (DocumentList::iterator itr = tests.begin(); itr != tests.end(); ++itr)
+                delete *itr;
+        }
+        SchemaDocument* schema;
+        DocumentList tests;
+    };
+
+    typedef std::vector<TestSuite* > TestSuiteList;
+    TestSuiteList testSuites;
+};
+
+TEST_F(Schema, TestSuite) {
+    char validatorBuffer[65536];
+    MemoryPoolAllocator<> validatorAllocator(validatorBuffer, sizeof(validatorBuffer));
+
+    int testCount = 0;
+    clock_t start = clock();
+    for (int i = 0; i < 10000; i++) {
+        for (TestSuiteList::const_iterator itr = testSuites.begin(); itr != testSuites.end(); ++itr) {
+            const TestSuite& ts = **itr;
+            GenericSchemaValidator<SchemaDocument, BaseReaderHandler<UTF8<> >, MemoryPoolAllocator<> >  validator(*ts.schema, &validatorAllocator);
+            for (DocumentList::const_iterator testItr = ts.tests.begin(); testItr != ts.tests.end(); ++testItr) {
+                validator.Reset();
+                (*testItr)->Accept(validator);
+                testCount++;
+            }
+            validatorAllocator.Clear();
+        }
+    }
+    clock_t end = clock();
+    double duration = double(end - start) / CLOCKS_PER_SEC;
+    printf("%d tests in %f s -> %f tests per sec\n", testCount, duration, testCount / duration);
+}
+
+#endif

test/unittest/schematest.cpp

@@ -992,11 +992,11 @@ private:
 
 TEST(SchemaValidator, TestSuite) {
     const char* filenames[] = {
-        "properties.json",
         "additionalItems.json",
         "additionalProperties.json",
         "allOf.json",
         "anyOf.json",
+        "default.json",
         "definitions.json",
         "dependencies.json",
         "enum.json",