Merge pull request #443 from andrusha97/master

Introduce comments support

include/rapidjson/reader.h

@@ -140,6 +140,7 @@ enum ParseFlag {
     kParseIterativeFlag = 4,        //!< Iterative(constant complexity in terms of function call stack size) parsing.
     kParseStopWhenDoneFlag = 8,     //!< After parsing a complete JSON root from stream, stop further processing the rest of stream. When this flag is used, parser will not generate kParseErrorDocumentRootNotSingular error.
     kParseFullPrecisionFlag = 16,   //!< Parse number in full precision (but slower).
+    kParseCommentsFlag = 32,        //!< Allow one-line (//) and multi-line (/**/) comments.
     kParseDefaultFlags = RAPIDJSON_PARSE_DEFAULT_FLAGS  //!< Default parse flags. Can be customized by defining RAPIDJSON_PARSE_DEFAULT_FLAGS
 };
 
@@ -398,7 +399,8 @@ public:
 
         ClearStackOnExit scope(*this);
 
-        SkipWhitespace(is);
+        SkipWhitespaceAndComments<parseFlags>(is);
+        RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_);
 
         if (is.Peek() == '\0') {
             RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentEmpty, is.Tell());
@@ -409,7 +411,8 @@ public:
             RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_);
 
             if (!(parseFlags & kParseStopWhenDoneFlag)) {
-                SkipWhitespace(is);
+                SkipWhitespaceAndComments<parseFlags>(is);
+                RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_);
 
                 if (is.Peek() != '\0') {
                     RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorDocumentRootNotSingular, is.Tell());
@@ -462,6 +465,40 @@ private:
         ClearStackOnExit& operator=(const ClearStackOnExit&);
     };
 
+    template<unsigned parseFlags, typename InputStream>
+    void SkipWhitespaceAndComments(InputStream& is) {
+        SkipWhitespace(is);
+
+        if (parseFlags & kParseCommentsFlag) {
+            while (is.Peek() == '/') {
+                is.Take();
+
+                if (is.Peek() == '*') {
+                    is.Take();
+                    while (true) {
+                        if (is.Peek() == '\0')
+                            RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell());
+
+                        if (is.Take() == '*') {
+                            if (is.Peek() == '\0')
+                                RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell());
+
+                            if (is.Take() == '/')
+                                break;
+                        }
+                    }
+                } else if (is.Peek() == '/') {
+                    is.Take();
+                    while (is.Peek() != '\0' && is.Take() != '\n') { }
+                } else {
+                    RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell());
+                }
+
+                SkipWhitespace(is);
+            }
+        }
+    }
+
     // Parse object: { string : value, ... }
     template<unsigned parseFlags, typename InputStream, typename Handler>
     void ParseObject(InputStream& is, Handler& handler) {
@@ -471,7 +508,8 @@ private:
         if (!handler.StartObject())
             RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell());
 
-        SkipWhitespace(is);
+        SkipWhitespaceAndComments<parseFlags>(is);
+        RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID;
 
         if (is.Peek() == '}') {
             is.Take();
@@ -487,22 +525,28 @@ private:
             ParseString<parseFlags>(is, handler, true);
             RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID;
 
-            SkipWhitespace(is);
+            SkipWhitespaceAndComments<parseFlags>(is);
+            RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID;
 
             if (is.Take() != ':')
                 RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell());
 
-            SkipWhitespace(is);
+            SkipWhitespaceAndComments<parseFlags>(is);
+            RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID;
 
             ParseValue<parseFlags>(is, handler);
             RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID;
 
-            SkipWhitespace(is);
+            SkipWhitespaceAndComments<parseFlags>(is);
+            RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID;
 
             ++memberCount;
 
             switch (is.Take()) {
-                case ',': SkipWhitespace(is); break;
+                case ',':
+                    SkipWhitespaceAndComments<parseFlags>(is);
+                    RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID;
+                    break;
                 case '}': 
                     if (!handler.EndObject(memberCount))
                         RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell());
@@ -521,7 +565,8 @@ private:
         if (!handler.StartArray())
             RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell());
         
-        SkipWhitespace(is);
+        SkipWhitespaceAndComments<parseFlags>(is);
+        RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID;
 
         if (is.Peek() == ']') {
             is.Take();
@@ -535,10 +580,14 @@ private:
             RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID;
 
             ++elementCount;
-            SkipWhitespace(is);
+            SkipWhitespaceAndComments<parseFlags>(is);
+            RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID;
 
             switch (is.Take()) {
-                case ',': SkipWhitespace(is); break;
+                case ',':
+                    SkipWhitespaceAndComments<parseFlags>(is);
+                    RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID;
+                    break;
                 case ']': 
                     if (!handler.EndArray(elementCount))
                         RAPIDJSON_PARSE_ERROR(kParseErrorTermination, is.Tell());
@@ -1404,7 +1453,8 @@ private:
         ClearStackOnExit scope(*this);
         IterativeParsingState state = IterativeParsingStartState;
 
-        SkipWhitespace(is);
+        SkipWhitespaceAndComments<parseFlags>(is);
+        RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_);
         while (is.Peek() != '\0') {
             Token t = Tokenize(is.Peek());
             IterativeParsingState n = Predict(state, t);
@@ -1421,7 +1471,8 @@ private:
             if ((parseFlags & kParseStopWhenDoneFlag) && state == IterativeParsingFinishState)
                 break;
 
-            SkipWhitespace(is);
+            SkipWhitespaceAndComments<parseFlags>(is);
+            RAPIDJSON_PARSE_ERROR_EARLY_RETURN(parseResult_);
         }
 
         // Handle the end of file.

test/unittest/readertest.cpp

@@ -1349,6 +1349,109 @@ TEST(Reader, ParseTerminationByHandler) {
     TEST_TERMINATION(12, "{\"a\":[1]"); // non-empty array
 }
 
+TEST(Reader, ParseComments) {
+    const char* json =
+    "// Here is a one-line comment.\n"
+    "{// And here's another one\n"
+    "   /*And here's an in-line one.*/\"hello\" : \"world\","
+    "   \"t\" :/* And one with '*' symbol*/true ,"
+    "/* A multiline comment\n"
+    "   goes here*/"
+    "   \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3]"
+    "}/*And the last one to be sure */";
+
+    StringStream s(json);
+    ParseObjectHandler h;
+    Reader reader;
+    EXPECT_TRUE(reader.Parse<kParseCommentsFlag>(s, h));
+    EXPECT_EQ(20u, h.step_);
+}
+
+TEST(Reader, ParseEmptyInlineComment) {
+    const char* json = "{/**/\"hello\" : \"world\", \"t\" : true, \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }";
+
+    StringStream s(json);
+    ParseObjectHandler h;
+    Reader reader;
+    EXPECT_TRUE(reader.Parse<kParseCommentsFlag>(s, h));
+    EXPECT_EQ(20u, h.step_);
+}
+
+TEST(Reader, ParseEmptyOnelineComment) {
+    const char* json = "{//\n\"hello\" : \"world\", \"t\" : true, \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }";
+
+    StringStream s(json);
+    ParseObjectHandler h;
+    Reader reader;
+    EXPECT_TRUE(reader.Parse<kParseCommentsFlag>(s, h));
+    EXPECT_EQ(20u, h.step_);
+}
+
+TEST(Reader, ParseMultipleCommentsInARow) {
+    const char* json = 
+    "{/* first comment *//* second */\n"
+    "/* third */ /*fourth*/// last one\n"
+    "\"hello\" : \"world\", \"t\" : true, \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }";
+
+    StringStream s(json);
+    ParseObjectHandler h;
+    Reader reader;
+    EXPECT_TRUE(reader.Parse<kParseCommentsFlag>(s, h));
+    EXPECT_EQ(20u, h.step_);
+}
+
+TEST(Reader, InlineCommentsAreDisabledByDefault) {
+    {
+        const char* json = "{/* Inline comment. */\"hello\" : \"world\", \"t\" : true, \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }";
+
+        StringStream s(json);
+        ParseObjectHandler h;
+        Reader reader;
+        EXPECT_FALSE(reader.Parse<kParseDefaultFlags>(s, h));
+    }
+
+    {
+        const char* json =
+        "{\"hello\" : /* Multiline comment starts here\n"
+        " continues here\n"
+        " and ends here */\"world\", \"t\" :true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }";
+
+        StringStream s(json);
+        ParseObjectHandler h;
+        Reader reader;
+        EXPECT_FALSE(reader.Parse<kParseDefaultFlags>(s, h));
+    }
+}
+
+TEST(Reader, OnelineCommentsAreDisabledByDefault) {
+    const char* json = "{// One-line comment\n\"hello\" : \"world\", \"t\" : true , \"f\" : false, \"n\": null, \"i\":123, \"pi\": 3.1416, \"a\":[1, 2, 3] }";
+
+    StringStream s(json);
+    ParseObjectHandler h;
+    Reader reader;
+    EXPECT_FALSE(reader.Parse<kParseDefaultFlags>(s, h));
+}
+
+TEST(Reader, EofAfterOneLineComment) {
+    const char* json = "{\"hello\" : \"world\" // EOF is here -->\0 \n}";
+
+    StringStream s(json);
+    ParseObjectHandler h;
+    Reader reader;
+    EXPECT_FALSE(reader.Parse<kParseCommentsFlag>(s, h));
+    EXPECT_EQ(kParseErrorObjectMissCommaOrCurlyBracket, reader.GetParseErrorCode());
+}
+
+TEST(Reader, IncompleteMultilineComment) {
+    const char* json = "{\"hello\" : \"world\" /* EOF is here -->\0 */}";
+
+    StringStream s(json);
+    ParseObjectHandler h;
+    Reader reader;
+    EXPECT_FALSE(reader.Parse<kParseCommentsFlag>(s, h));
+    EXPECT_EQ(kParseErrorUnspecificSyntaxError, reader.GetParseErrorCode());
+}
+
 #ifdef __GNUC__
 RAPIDJSON_DIAG_POP
 #endif