Added trailing comma support to iterative parser

This also fixes cases where the iterative parser should have produced
kParseErrorValueInvalid rather than kParseErrorUnspecifiedSyntaxError
when expecting a value (after a colon in an object, after a comma in an
array, and at the start of an array.)

include/rapidjson/reader.h

@@ -640,9 +640,9 @@ private:
 
             if (parseFlags & kParseTrailingCommasFlag) {
                 if (is.Peek() == '}') {
-                    is.Take();
                     if (RAPIDJSON_UNLIKELY(!handler.EndObject(memberCount)))
                         RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell());
+                    is.Take();
                     return;
                 }
             }
@@ -689,9 +689,9 @@ private:
 
             if (parseFlags & kParseTrailingCommasFlag) {
                 if (is.Peek() == ']') {
-                    is.Take();
                     if (RAPIDJSON_UNLIKELY(!handler.EndArray(elementCount)))
                         RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell());
+                    is.Take();
                     return;
                 }
             }
@@ -1541,7 +1541,7 @@ private:
                 IterativeParsingErrorState,         // Left bracket
                 IterativeParsingErrorState,         // Right bracket
                 IterativeParsingErrorState,         // Left curly bracket
-                IterativeParsingErrorState,         // Right curly bracket
+                IterativeParsingObjectFinishState,  // Right curly bracket
                 IterativeParsingErrorState,         // Comma
                 IterativeParsingErrorState,         // Colon
                 IterativeParsingMemberKeyState,     // String
@@ -1587,7 +1587,7 @@ private:
             // ElementDelimiter
             {
                 IterativeParsingArrayInitialState,      // Left bracket(push Element state)
-                IterativeParsingErrorState,             // Right bracket
+                IterativeParsingArrayFinishState,       // Right bracket
                 IterativeParsingObjectInitialState,     // Left curly bracket(push Element state)
                 IterativeParsingErrorState,             // Right curly bracket
                 IterativeParsingErrorState,             // Comma
@@ -1689,6 +1689,11 @@ private:
 
         case IterativeParsingObjectFinishState:
         {
+            // Transit from delimiter is only allowed when trailing commas are enabled
+            if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingMemberDelimiterState) {
+                RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorObjectMissName, is.Tell());
+                return IterativeParsingErrorState;
+            }
             // Get member count.
             SizeType c = *stack_.template Pop<SizeType>(1);
             // If the object is not empty, count the last member.
@@ -1714,6 +1719,11 @@ private:
 
         case IterativeParsingArrayFinishState:
         {
+            // Transit from delimiter is only allowed when trailing commas are enabled
+            if (!(parseFlags & kParseTrailingCommasFlag) && src == IterativeParsingElementDelimiterState) {
+                RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorValueInvalid, is.Tell());
+                return IterativeParsingErrorState;
+            }
             // Get element count.
             SizeType c = *stack_.template Pop<SizeType>(1);
             // If the array is not empty, count the last element.
@@ -1773,6 +1783,9 @@ private:
         case IterativeParsingMemberDelimiterState:  RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell()); return;
         case IterativeParsingMemberKeyState:        RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell()); return;
         case IterativeParsingMemberValueState:      RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell()); return;
+        case IterativeParsingKeyValueDelimiterState:
+        case IterativeParsingArrayInitialState:
+        case IterativeParsingElementDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorValueInvalid, is.Tell()); return;
         case IterativeParsingElementState:          RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell()); return;
         default:                                    RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell()); return;
         }       

test/unittest/readertest.cpp

@@ -1127,6 +1127,16 @@ TEST(Reader, IterativeParsing_ErrorHandling) {
     TESTERRORHANDLING("{\"a\": 1", kParseErrorObjectMissCommaOrCurlyBracket, 7u);
     TESTERRORHANDLING("[1 2 3]", kParseErrorArrayMissCommaOrSquareBracket, 3u);
     TESTERRORHANDLING("{\"a: 1", kParseErrorStringMissQuotationMark, 6u);
+    TESTERRORHANDLING("{\"a\":}", kParseErrorValueInvalid, 5u);
+    TESTERRORHANDLING("{\"a\":]", kParseErrorValueInvalid, 5u);
+    TESTERRORHANDLING("[1,2,}", kParseErrorValueInvalid, 5u);
+    TESTERRORHANDLING("[}]", kParseErrorValueInvalid, 1u);
+    TESTERRORHANDLING("[,]", kParseErrorValueInvalid, 1u);
+    TESTERRORHANDLING("[1,,]", kParseErrorValueInvalid, 3u);
+
+    // Trailing commas are not allowed without kParseTrailingCommasFlag
+    TESTERRORHANDLING("{\"a\": 1,}", kParseErrorObjectMissName, 8u);
+    TESTERRORHANDLING("[1,2,3,]", kParseErrorValueInvalid, 7u);
 
     // Any JSON value can be a valid root element in RFC7159.
     TESTERRORHANDLING("\"ab", kParseErrorStringMissQuotationMark, 3u);
@@ -1560,12 +1570,13 @@ TEST(Reader, NumbersAsStrings) {
 	}
 }
 
-TEST(Reader, TrailingCommas) {
+template <unsigned extraFlags>
+void TestTrailingCommas() {
     {
         StringStream s("[1,2,3,]");
         ParseArrayHandler<3> h;
         Reader reader;
-        EXPECT_TRUE(reader.Parse<kParseTrailingCommasFlag>(s, h));
+        EXPECT_TRUE(reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h));
         EXPECT_EQ(5u, h.step_);
     }
     {
@@ -1574,7 +1585,7 @@ TEST(Reader, TrailingCommas) {
         StringStream s(json);
         ParseObjectHandler h;
         Reader reader;
-        EXPECT_TRUE(reader.Parse<kParseTrailingCommasFlag>(s, h));
+        EXPECT_TRUE(reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h));
         EXPECT_EQ(20u, h.step_);
     }
     {
@@ -1584,7 +1595,7 @@ TEST(Reader, TrailingCommas) {
         StringStream s(json);
         ParseObjectHandler h;
         Reader reader;
-        EXPECT_TRUE(reader.Parse<kParseTrailingCommasFlag>(s, h));
+        EXPECT_TRUE(reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h));
         EXPECT_EQ(20u, h.step_);
     }
     {
@@ -1594,18 +1605,27 @@ TEST(Reader, TrailingCommas) {
         StringStream s(json);
         ParseObjectHandler h;
         Reader reader;
-        EXPECT_TRUE(reader.Parse<kParseTrailingCommasFlag|kParseCommentsFlag>(s, h));
+        EXPECT_TRUE(reader.Parse<extraFlags|kParseTrailingCommasFlag|kParseCommentsFlag>(s, h));
         EXPECT_EQ(20u, h.step_);
     }
 }
 
-TEST(Reader, MultipleTrailingCommaErrors) {
+TEST(Reader, TrailingCommas) {
+    TestTrailingCommas<kParseNoFlags>();
+}
+
+TEST(Reader, TrailingCommasIterative) {
+    TestTrailingCommas<kParseIterativeFlag>();
+}
+
+template <unsigned extraFlags>
+void TestMultipleTrailingCommaErrors() {
     // only a single trailing comma is allowed.
     {
         StringStream s("[1,2,3,,]");
         ParseArrayHandler<3> h;
         Reader reader;
-        ParseResult r = reader.Parse<kParseTrailingCommasFlag>(s, h);
+        ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
         EXPECT_TRUE(reader.HasParseError());
         EXPECT_EQ(kParseErrorValueInvalid, r.Code());
         EXPECT_EQ(7u, r.Offset());
@@ -1616,21 +1636,30 @@ TEST(Reader, MultipleTrailingCommaErrors) {
         StringStream s(json);
         ParseObjectHandler h;
         Reader reader;
-        ParseResult r = reader.Parse<kParseTrailingCommasFlag>(s, h);
+        ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
         EXPECT_TRUE(reader.HasParseError());
         EXPECT_EQ(kParseErrorObjectMissName, r.Code());
         EXPECT_EQ(95u, r.Offset());
     }
 }
 
-TEST(Reader, EmptyExceptForCommaErrors) {
+TEST(Reader, MultipleTrailingCommaErrors) {
+    TestMultipleTrailingCommaErrors<kParseNoFlags>();
+}
+
+TEST(Reader, MultipleTrailingCommaErrorsIterative) {
+    TestMultipleTrailingCommaErrors<kParseIterativeFlag>();
+}
+
+template <unsigned extraFlags>
+void TestEmptyExceptForCommaErrors() {
     // not allowed even with trailing commas enabled; the
     // trailing comma must follow a value.
     {
         StringStream s("[,]");
         ParseArrayHandler<3> h;
         Reader reader;
-        ParseResult r = reader.Parse<kParseTrailingCommasFlag>(s, h);
+        ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
         EXPECT_TRUE(reader.HasParseError());
         EXPECT_EQ(kParseErrorValueInvalid, r.Code());
         EXPECT_EQ(1u, r.Offset());
@@ -1639,34 +1668,51 @@ TEST(Reader, EmptyExceptForCommaErrors) {
         StringStream s("{,}");
         ParseObjectHandler h;
         Reader reader;
-        ParseResult r = reader.Parse<kParseTrailingCommasFlag>(s, h);
+        ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
         EXPECT_TRUE(reader.HasParseError());
         EXPECT_EQ(kParseErrorObjectMissName, r.Code());
         EXPECT_EQ(1u, r.Offset());
     }
 }
 
-TEST(Reader, TrailingCommaHandlerTermination) {
+TEST(Reader, EmptyExceptForCommaErrors) {
+    TestEmptyExceptForCommaErrors<kParseNoFlags>();
+}
+
+TEST(Reader, EmptyExceptForCommaErrorsIterative) {
+    TestEmptyExceptForCommaErrors<kParseIterativeFlag>();
+}
+
+template <unsigned extraFlags>
+void TestTrailingCommaHandlerTermination() {
     {
         HandlerTerminateAtEndArray h;
         Reader reader;
         StringStream s("[1,2,3,]");
-        ParseResult r = reader.Parse<kParseTrailingCommasFlag>(s, h);
+        ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
         EXPECT_TRUE(reader.HasParseError());
         EXPECT_EQ(kParseErrorTermination, r.Code());
-        EXPECT_EQ(8u, r.Offset());
+        EXPECT_EQ(7u, r.Offset());
     }
     {
         HandlerTerminateAtEndObject h;
         Reader reader;
         StringStream s("{\"t\": true, \"f\": false,}");
-        ParseResult r = reader.Parse<kParseTrailingCommasFlag>(s, h);
+        ParseResult r = reader.Parse<extraFlags|kParseTrailingCommasFlag>(s, h);
         EXPECT_TRUE(reader.HasParseError());
         EXPECT_EQ(kParseErrorTermination, r.Code());
-        EXPECT_EQ(24u, r.Offset());
+        EXPECT_EQ(23u, r.Offset());
     }
 }
 
+TEST(Reader, TrailingCommaHandlerTermination) {
+    TestTrailingCommaHandlerTermination<kParseNoFlags>();
+}
+
+TEST(Reader, TrailingCommaHandlerTerminationIterative) {
+    TestTrailingCommaHandlerTermination<kParseIterativeFlag>();
+}
+
 #ifdef __GNUC__
 RAPIDJSON_DIAG_POP
 #endif