Merge pull request #842 from StilesCrisis/token-by-token-parsing

Token-by-token pull parsing

example/CMakeLists.txt

@@ -18,6 +18,7 @@ set(EXAMPLES
     serialize
     simpledom
     simplereader
+    simplepullreader
     simplewriter
     tutorial)
     

example/simplepullreader/simplepullreader.cpp

+#include "rapidjson/reader.h"
+#include <iostream>
+#include <sstream>
+
+using namespace rapidjson;
+using namespace std;
+
+// If you can require C++11, you could use std::to_string here
+template <typename T> std::string stringify(T x) {
+    std::stringstream ss;
+    ss << x;
+    return ss.str();
+}
+
+struct MyHandler {
+    const char* type;
+    std::string data;
+    
+    bool Null() { type = "Null"; data.clear(); return true; }
+    bool Bool(bool b) { type = "Bool:"; data = b? "true": "false"; return true; }
+    bool Int(int i) { type = "Int:"; data = stringify(i); return true; }
+    bool Uint(unsigned u) { type = "Uint:"; data = stringify(u); return true; }
+    bool Int64(int64_t i) { type = "Int64:"; data = stringify(i); return true; }
+    bool Uint64(uint64_t u) { type = "Uint64:"; data = stringify(u); return true; }
+    bool Double(double d) { type = "Double:"; data = stringify(d); return true; }
+    bool RawNumber(const char* str, SizeType length, bool) { type = "Number:"; data = std::string(str, length); return true; }
+    bool String(const char* str, SizeType length, bool) { type = "String:"; data = std::string(str, length); return true; }
+    bool StartObject() { type = "StartObject"; data.clear(); return true; }
+    bool Key(const char* str, SizeType length, bool) { type = "Key:"; data = std::string(str, length); return true; }
+    bool EndObject(SizeType memberCount) { type = "EndObject:"; data = stringify(memberCount); return true; }
+    bool StartArray() { type = "StartArray"; data.clear(); return true; }
+    bool EndArray(SizeType elementCount) { type = "EndArray:"; data = stringify(elementCount); return true; }
+};
+
+int main() {
+    const char json[] = " { \"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3, 4] } ";
+
+    MyHandler handler;
+    Reader reader;
+    StringStream ss(json);
+    reader.IterativeParseInit();
+    while (!reader.IterativeParseComplete()) {
+        reader.IterativeParseNext<kParseDefaultFlags>(ss, handler);
+        cout << handler.type << handler.data << endl;
+    }
+
+    return 0;
+}

test/perftest/rapidjsontest.cpp

@@ -152,6 +152,35 @@ TEST_F(RapidJson, SIMD_SUFFIX(ReaderParseIterativeInsitu_DummyHandler)) {
     }
 }
 
+TEST_F(RapidJson, SIMD_SUFFIX(ReaderParseIterativePull_DummyHandler)) {
+    for (size_t i = 0; i < kTrialCount; i++) {
+        StringStream s(json_);
+        BaseReaderHandler<> h;
+        Reader reader;
+        reader.IterativeParseInit();
+        while (!reader.IterativeParseComplete()) {
+            if (!reader.IterativeParseNext<kParseDefaultFlags>(s, h))
+                break;
+        }
+        EXPECT_FALSE(reader.HasParseError());
+    }
+}
+
+TEST_F(RapidJson, SIMD_SUFFIX(ReaderParseIterativePullInsitu_DummyHandler)) {
+    for (size_t i = 0; i < kTrialCount; i++) {
+        memcpy(temp_, json_, length_ + 1);
+        InsituStringStream s(temp_);
+        BaseReaderHandler<> h;
+        Reader reader;
+        reader.IterativeParseInit();
+        while (!reader.IterativeParseComplete()) {
+            if (!reader.IterativeParseNext<kParseDefaultFlags|kParseInsituFlag>(s, h))
+                break;
+        }
+        EXPECT_FALSE(reader.HasParseError());
+    }
+}
+
 TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler_ValidateEncoding)) {
     for (size_t i = 0; i < kTrialCount; i++) {
         StringStream s(json_);

test/unittest/jsoncheckertest.cpp

@@ -48,6 +48,24 @@ static char* ReadFile(const char* filename, size_t& length) {
     return json;
 }
 
+struct NoOpHandler {
+    bool Null() { return true; }
+    bool Bool(bool) { return true; }
+    bool Int(int) { return true; }
+    bool Uint(unsigned) { return true; }
+    bool Int64(int64_t) { return true; }
+    bool Uint64(uint64_t) { return true; }
+    bool Double(double) { return true; }
+    bool RawNumber(const char*, SizeType, bool) { return true; }
+    bool String(const char*, SizeType, bool) { return true; }
+    bool StartObject() { return true; }
+    bool Key(const char*, SizeType, bool) { return true; }
+    bool EndObject(SizeType) { return true; }
+    bool StartArray() { return true; }
+    bool EndArray(SizeType) { return true; }
+};
+
+
 TEST(JsonChecker, Reader) {
     char filename[256];
 
@@ -67,13 +85,26 @@ TEST(JsonChecker, Reader) {
             continue;
         }
 
+        // Test stack-based parsing.
         GenericDocument<UTF8<>, CrtAllocator> document; // Use Crt allocator to check exception-safety (no memory leak)
         document.Parse(json);
         EXPECT_TRUE(document.HasParseError()) << filename;
 
+        // Test iterative parsing.
         document.Parse<kParseIterativeFlag>(json);
         EXPECT_TRUE(document.HasParseError()) << filename;
 
+        // Test iterative pull-parsing.
+        Reader reader;
+        StringStream ss(json);
+        NoOpHandler h;
+        reader.IterativeParseInit();
+        while (!reader.IterativeParseComplete()) {
+            if (!reader.IterativeParseNext<kParseDefaultFlags>(ss, h))
+                break;
+        }
+        EXPECT_TRUE(reader.HasParseError()) << filename;
+        
         free(json);
     }
 
@@ -87,12 +118,25 @@ TEST(JsonChecker, Reader) {
             continue;
         }
 
+        // Test stack-based parsing.
         GenericDocument<UTF8<>, CrtAllocator> document; // Use Crt allocator to check exception-safety (no memory leak)
         document.Parse(json);
         EXPECT_FALSE(document.HasParseError()) << filename;
 
+        // Test iterative parsing.
         document.Parse<kParseIterativeFlag>(json);
         EXPECT_FALSE(document.HasParseError()) << filename;
+        
+        // Test iterative pull-parsing.
+        Reader reader;
+        StringStream ss(json);
+        NoOpHandler h;
+        reader.IterativeParseInit();
+        while (!reader.IterativeParseComplete()) {
+            if (!reader.IterativeParseNext<kParseDefaultFlags>(ss, h))
+                break;
+        }
+        EXPECT_FALSE(reader.HasParseError()) << filename;
 
         free(json);
     }

test/unittest/readertest.cpp

@@ -1157,22 +1157,22 @@ template<typename Encoding = UTF8<> >
 struct IterativeParsingReaderHandler {
     typedef typename Encoding::Ch Ch;
 
-    const static int LOG_NULL = -1;
-    const static int LOG_BOOL = -2;
-    const static int LOG_INT = -3;
-    const static int LOG_UINT = -4;
-    const static int LOG_INT64 = -5;
-    const static int LOG_UINT64 = -6;
-    const static int LOG_DOUBLE = -7;
-    const static int LOG_STRING = -8;
-    const static int LOG_STARTOBJECT = -9;
-    const static int LOG_KEY = -10;
-    const static int LOG_ENDOBJECT = -11;
-    const static int LOG_STARTARRAY = -12;
-    const static int LOG_ENDARRAY = -13;
+    const static uint32_t LOG_NULL        = 0x10000000;
+    const static uint32_t LOG_BOOL        = 0x20000000;
+    const static uint32_t LOG_INT         = 0x30000000;
+    const static uint32_t LOG_UINT        = 0x40000000;
+    const static uint32_t LOG_INT64       = 0x50000000;
+    const static uint32_t LOG_UINT64      = 0x60000000;
+    const static uint32_t LOG_DOUBLE      = 0x70000000;
+    const static uint32_t LOG_STRING      = 0x80000000;
+    const static uint32_t LOG_STARTOBJECT = 0x90000000;
+    const static uint32_t LOG_KEY         = 0xA0000000;
+    const static uint32_t LOG_ENDOBJECT   = 0xB0000000;
+    const static uint32_t LOG_STARTARRAY  = 0xC0000000;
+    const static uint32_t LOG_ENDARRAY    = 0xD0000000;
 
     const static size_t LogCapacity = 256;
-    int Logs[LogCapacity];
+    uint32_t Logs[LogCapacity];
     size_t LogCount;
 
     IterativeParsingReaderHandler() : LogCount(0) {
@@ -1202,8 +1202,8 @@ struct IterativeParsingReaderHandler {
 
     bool EndObject(SizeType c) {
         RAPIDJSON_ASSERT(LogCount < LogCapacity);
-        Logs[LogCount++] = LOG_ENDOBJECT;
-        Logs[LogCount++] = static_cast<int>(c);
+        RAPIDJSON_ASSERT((static_cast<uint32_t>(c) & 0xF0000000) == 0);
+        Logs[LogCount++] = LOG_ENDOBJECT | static_cast<uint32_t>(c);
         return true;
     }
 
@@ -1211,8 +1211,8 @@ struct IterativeParsingReaderHandler {
 
     bool EndArray(SizeType c) {
         RAPIDJSON_ASSERT(LogCount < LogCapacity);
-        Logs[LogCount++] = LOG_ENDARRAY;
-        Logs[LogCount++] = static_cast<int>(c);
+        RAPIDJSON_ASSERT((static_cast<uint32_t>(c) & 0xF0000000) == 0);
+        Logs[LogCount++] = LOG_ENDARRAY | static_cast<uint32_t>(c);
         return true;
     }
 };
@@ -1228,7 +1228,7 @@ TEST(Reader, IterativeParsing_General) {
         EXPECT_FALSE(r.IsError());
         EXPECT_FALSE(reader.HasParseError());
 
-        int e[] = {
+        uint32_t e[] = {
             handler.LOG_STARTARRAY,
             handler.LOG_INT,
             handler.LOG_STARTOBJECT,
@@ -1236,14 +1236,14 @@ TEST(Reader, IterativeParsing_General) {
             handler.LOG_STARTARRAY,
             handler.LOG_INT,
             handler.LOG_INT,
-            handler.LOG_ENDARRAY, 2,
-            handler.LOG_ENDOBJECT, 1,
+            handler.LOG_ENDARRAY | 2,
+            handler.LOG_ENDOBJECT | 1,
             handler.LOG_NULL,
             handler.LOG_BOOL,
             handler.LOG_BOOL,
             handler.LOG_STRING,
             handler.LOG_DOUBLE,
-            handler.LOG_ENDARRAY, 7
+            handler.LOG_ENDARRAY | 7
         };
 
         EXPECT_EQ(sizeof(e) / sizeof(int), handler.LogCount);
@@ -1265,20 +1265,20 @@ TEST(Reader, IterativeParsing_Count) {
         EXPECT_FALSE(r.IsError());
         EXPECT_FALSE(reader.HasParseError());
 
-        int e[] = {
+        uint32_t e[] = {
             handler.LOG_STARTARRAY,
             handler.LOG_STARTOBJECT,
-            handler.LOG_ENDOBJECT, 0,
+            handler.LOG_ENDOBJECT | 0,
             handler.LOG_STARTOBJECT,
             handler.LOG_KEY,
             handler.LOG_INT,
-            handler.LOG_ENDOBJECT, 1,
+            handler.LOG_ENDOBJECT | 1,
             handler.LOG_STARTARRAY,
             handler.LOG_INT,
-            handler.LOG_ENDARRAY, 1,
+            handler.LOG_ENDARRAY | 1,
             handler.LOG_STARTARRAY,
-            handler.LOG_ENDARRAY, 0,
-            handler.LOG_ENDARRAY, 4
+            handler.LOG_ENDARRAY | 0,
+            handler.LOG_ENDARRAY | 4
         };
 
         EXPECT_EQ(sizeof(e) / sizeof(int), handler.LogCount);
@@ -1289,6 +1289,51 @@ TEST(Reader, IterativeParsing_Count) {
     }
 }
 
+TEST(Reader, IterativePullParsing_General) {
+    {
+        IterativeParsingReaderHandler<> handler;
+        uint32_t e[] = {
+            handler.LOG_STARTARRAY,
+            handler.LOG_INT,
+            handler.LOG_STARTOBJECT,
+            handler.LOG_KEY,
+            handler.LOG_STARTARRAY,
+            handler.LOG_INT,
+            handler.LOG_INT,
+            handler.LOG_ENDARRAY | 2,
+            handler.LOG_ENDOBJECT | 1,
+            handler.LOG_NULL,
+            handler.LOG_BOOL,
+            handler.LOG_BOOL,
+            handler.LOG_STRING,
+            handler.LOG_DOUBLE,
+            handler.LOG_ENDARRAY | 7
+        };
+        
+        StringStream is("[1, {\"k\": [1, 2]}, null, false, true, \"string\", 1.2]");
+        Reader reader;
+        
+        reader.IterativeParseInit();
+        while (!reader.IterativeParseComplete()) {
+            size_t oldLogCount = handler.LogCount;
+            EXPECT_TRUE(oldLogCount < sizeof(e) / sizeof(int)) << "overrun";
+            
+            EXPECT_TRUE(reader.IterativeParseNext<kParseDefaultFlags>(is, handler)) << "parse fail";
+            EXPECT_EQ(handler.LogCount, oldLogCount + 1) << "handler should be invoked exactly once each time";
+            EXPECT_EQ(e[oldLogCount], handler.Logs[oldLogCount]) << "wrong event returned";
+        }
+        
+        EXPECT_FALSE(reader.HasParseError());
+        EXPECT_EQ(sizeof(e) / sizeof(int), handler.LogCount) << "handler invoked wrong number of times";
+
+        // The handler should not be invoked when the JSON has been fully read, but it should not fail
+        size_t oldLogCount = handler.LogCount;
+        EXPECT_TRUE(reader.IterativeParseNext<kParseDefaultFlags>(is, handler)) << "parse-next past complete is allowed";
+        EXPECT_EQ(handler.LogCount, oldLogCount) << "parse-next past complete should not invoke handler";
+        EXPECT_FALSE(reader.HasParseError()) << "parse-next past complete should not generate parse error";
+    }
+}
+
 // Test iterative parsing on kParseErrorTermination.
 struct HandlerTerminateAtStartObject : public IterativeParsingReaderHandler<> {
     bool StartObject() { return false; }