Merge pull request #564 from corporateshark/stringnumbers

Implemented feature as in #560 (ints/doubles as strings)

example/capitalize/capitalize.cpp

@@ -24,6 +24,7 @@ struct CapitalizeFilter {
     bool Int64(int64_t i) { return out_.Int64(i); }
     bool Uint64(uint64_t u) { return out_.Uint64(u); }
     bool Double(double d) { return out_.Double(d); }
+    bool RawNumber(const char* str, SizeType length, bool copy) { return out_.RawNumber(str, length, copy); }
     bool String(const char* str, SizeType length, bool) {
         buffer_.clear();
         for (SizeType i = 0; i < length; i++)

example/jsonx/jsonx.cpp

@@ -57,6 +57,13 @@ public:
         return WriteNumberElement(buffer, sprintf(buffer, "%.17g", d));
     }
 
+    bool RawNumber(const char* str, SizeType length, bool) {
+        return
+            WriteStartElement("number") &&
+            WriteEscapedText(str, length) &&
+            WriteEndElement("number");
+    }
+
     bool String(const char* str, SizeType length, bool) {
         return
             WriteStartElement("string") &&

example/simplereader/simplereader.cpp

@@ -12,6 +12,10 @@ struct MyHandler {
     bool Int64(int64_t i) { cout << "Int64(" << i << ")" << endl; return true; }
     bool Uint64(uint64_t u) { cout << "Uint64(" << u << ")" << endl; return true; }
     bool Double(double d) { cout << "Double(" << d << ")" << endl; return true; }
+    bool RawNumber(const char* str, SizeType length, bool copy) { 
+        cout << "Number(" << str << ", " << length << ", " << boolalpha << copy << ")" << endl;
+        return true;
+    }
     bool String(const char* str, SizeType length, bool copy) { 
         cout << "String(" << str << ", " << length << ", " << boolalpha << copy << ")" << endl;
         return true;

include/rapidjson/document.h

@@ -2325,6 +2325,14 @@ public:
     bool Uint64(uint64_t i) { new (stack_.template Push<ValueType>()) ValueType(i); return true; }
     bool Double(double d) { new (stack_.template Push<ValueType>()) ValueType(d); return true; }
 
+    bool RawNumber(const Ch* str, SizeType length, bool copy) { 
+        if (copy) 
+            new (stack_.template Push<ValueType>()) ValueType(str, length, GetAllocator());
+        else
+            new (stack_.template Push<ValueType>()) ValueType(str, length);
+        return true;
+    }
+
     bool String(const Ch* str, SizeType length, bool copy) { 
         if (copy) 
             new (stack_.template Push<ValueType>()) ValueType(str, length, GetAllocator());

include/rapidjson/prettywriter.h

@@ -74,6 +74,12 @@ public:
     bool Uint64(uint64_t u64)   { PrettyPrefix(kNumberType); return Base::WriteUint64(u64);  }
     bool Double(double d)       { PrettyPrefix(kNumberType); return Base::WriteDouble(d); }
 
+    bool RawNumber(const Ch* str, SizeType length, bool copy = false) {
+        (void)copy;
+        PrettyPrefix(kNumberType);
+        return Base::WriteString(str, length);
+    }
+
     bool String(const Ch* str, SizeType length, bool copy = false) {
         (void)copy;
         PrettyPrefix(kStringType);

include/rapidjson/reader.h

@@ -148,6 +148,7 @@ enum ParseFlag {
     kParseStopWhenDoneFlag = 8,     //!< After parsing a complete JSON root from stream, stop further processing the rest of stream. When this flag is used, parser will not generate kParseErrorDocumentRootNotSingular error.
     kParseFullPrecisionFlag = 16,   //!< Parse number in full precision (but slower).
     kParseCommentsFlag = 32,        //!< Allow one-line (//) and multi-line (/**/) comments.
+    kParseNumbersAsStringsFlag = 64,    //!< Parse all numbers (ints/doubles) as strings.
     kParseDefaultFlags = RAPIDJSON_PARSE_DEFAULT_FLAGS  //!< Default parse flags. Can be customized by defining RAPIDJSON_PARSE_DEFAULT_FLAGS
 };
 
@@ -169,6 +170,8 @@ concept Handler {
     bool Int64(int64_t i);
     bool Uint64(uint64_t i);
     bool Double(double d);
+    /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length)
+    bool RawNumber(const Ch* str, SizeType length, bool copy);
     bool String(const Ch* str, SizeType length, bool copy);
     bool StartObject();
     bool Key(const Ch* str, SizeType length, bool copy);
@@ -199,6 +202,8 @@ struct BaseReaderHandler {
     bool Int64(int64_t) { return static_cast<Override&>(*this).Default(); }
     bool Uint64(uint64_t) { return static_cast<Override&>(*this).Default(); }
     bool Double(double) { return static_cast<Override&>(*this).Default(); }
+    /// enabled via kParseNumbersAsStringsFlag, string is not null-terminated (use length)
+    bool RawNumber(const Ch* str, SizeType len, bool copy) { return static_cast<Override&>(*this).String(str, len, copy); }
     bool String(const Ch*, SizeType, bool) { return static_cast<Override&>(*this).Default(); }
     bool StartObject() { return static_cast<Override&>(*this).Default(); }
     bool Key(const Ch* str, SizeType len, bool copy) { return static_cast<Override&>(*this).String(str, len, copy); }
@@ -1053,6 +1058,8 @@ private:
         RAPIDJSON_FORCEINLINE Ch Peek() const { return is.Peek(); }
         RAPIDJSON_FORCEINLINE Ch TakePush() { return is.Take(); }
         RAPIDJSON_FORCEINLINE Ch Take() { return is.Take(); }
+		  RAPIDJSON_FORCEINLINE void Push(char) {}
+
         size_t Tell() { return is.Tell(); }
         size_t Length() { return 0; }
         const char* Pop() { return 0; }
@@ -1075,6 +1082,10 @@ private:
             return Base::is.Take();
         }
 
+		  RAPIDJSON_FORCEINLINE void Push(char c) {
+			  stackStream.Put(c);
+		  }
+
         size_t Length() { return stackStream.Length(); }
 
         const char* Pop() {
@@ -1089,7 +1100,11 @@ private:
     template<unsigned parseFlags, typename InputStream, typename Handler>
     void ParseNumber(InputStream& is, Handler& handler) {
         internal::StreamLocalCopy<InputStream> copy(is);
-        NumberStream<InputStream, (parseFlags & kParseFullPrecisionFlag) != 0> s(*this, copy.s);
+        NumberStream<InputStream,
+            ((parseFlags & kParseNumbersAsStringsFlag) != 0) ?
+                ((parseFlags & kParseInsituFlag) == 0) :
+                ((parseFlags & kParseFullPrecisionFlag) != 0)> s(*this, copy.s);
+
         size_t startOffset = s.Tell();
 
         // Parse minus
@@ -1176,6 +1191,9 @@ private:
         int expFrac = 0;
         size_t decimalPosition;
         if (Consume(s, '.')) {
+            if (((parseFlags & kParseNumbersAsStringsFlag) != 0) && ((parseFlags & kParseInsituFlag) == 0)) {
+					s.Push('.');
+				}
             decimalPosition = s.Length();
 
             if (RAPIDJSON_UNLIKELY(!(s.Peek() >= '0' && s.Peek() <= '9')))
@@ -1223,6 +1241,10 @@ private:
         // Parse exp = e [ minus / plus ] 1*DIGIT
         int exp = 0;
         if (Consume(s, 'e') || Consume(s, 'E')) {
+            if ( ((parseFlags & kParseNumbersAsStringsFlag) != 0) && ((parseFlags & kParseInsituFlag) == 0) ) {
+                s.Push( 'e' );
+            }
+
 			   if (!useDouble) {
                 d = static_cast<double>(use64bit ? i64 : i);
                 useDouble = true;
@@ -1263,6 +1285,32 @@ private:
 
         // Finish parsing, call event according to the type of number.
         bool cont = true;
+
+        if (parseFlags & kParseNumbersAsStringsFlag) {
+
+            if (parseFlags & kParseInsituFlag) {
+                s.Pop();  // Pop stack no matter if it will be used or not.
+                typename InputStream::Ch* head = is.PutBegin();
+                const size_t length = s.Tell() - startOffset;
+                RAPIDJSON_ASSERT(length <= 0xFFFFFFFF);
+//                *(head + length) = '\0';
+                const typename TargetEncoding::Ch* const str = reinterpret_cast<typename TargetEncoding::Ch*>(head);
+                cont = handler.RawNumber(str, SizeType(length), false);
+            }
+            else {
+                StackStream<typename TargetEncoding::Ch> stackStream(stack_);
+                SizeType numCharsToCopy = static_cast<SizeType>(s.Length());
+                while (numCharsToCopy--) {
+                    Transcoder<SourceEncoding, TargetEncoding>::Transcode(is, stackStream);
+                }
+                stackStream.Put('\0');
+                const typename TargetEncoding::Ch* str = stackStream.Pop();
+                const SizeType length = static_cast<SizeType>(stackStream.Length()) - 1;
+                cont = handler.RawNumber(str, SizeType(length), true);
+            }
+
+        }
+        else {
            size_t length = s.Length();
            const char* decimal = s.Pop();  // Pop stack no matter if it will be used or not.
 
@@ -1289,6 +1337,7 @@ private:
                        cont = handler.Uint(i);
                }
            }
+        }
         if (RAPIDJSON_UNLIKELY(!cont))
             RAPIDJSON_PARSE_ERROR(kParseErrorTermination, startOffset);
     }

include/rapidjson/schema.h

@@ -183,6 +183,11 @@ public:
         return WriteNumber(n);
     }
 
+    bool RawNumber(const Ch* str, SizeType len, bool) {
+        WriteBuffer(kNumberType, str, len * sizeof(Ch));
+        return true;
+    }
+
     bool String(const Ch* str, SizeType len, bool) {
         WriteBuffer(kStringType, str, len * sizeof(Ch));
         return true;
@@ -1679,6 +1684,8 @@ RAPIDJSON_MULTILINEMACRO_END
     bool Int64(int64_t i)   { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int64,  (CurrentContext(), i), (i)); }
     bool Uint64(uint64_t u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint64, (CurrentContext(), u), (u)); }
     bool Double(double d)   { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Double, (CurrentContext(), d), (d)); }
+    bool RawNumber(const Ch* str, SizeType length, bool copy)
+                                    { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); }
     bool String(const Ch* str, SizeType length, bool copy)
                                     { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); }
 

include/rapidjson/writer.h

@@ -181,6 +181,12 @@ public:
     */
     bool Double(double d)       { Prefix(kNumberType); return WriteDouble(d); }
 
+    bool RawNumber(const Ch* str, SizeType length, bool copy = false) {
+        (void)copy;
+        Prefix(kNumberType);
+        return WriteString(str, length);
+    }
+
     bool String(const Ch* str, SizeType length, bool copy = false) {
         (void)copy;
         Prefix(kStringType);

test/unittest/readertest.cpp

@@ -1170,6 +1170,8 @@ struct IterativeParsingReaderHandler {
 
     bool Double(double) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_DOUBLE; return true; }
 
+    bool RawNumber(const Ch*, SizeType, bool) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_STRING; return true; }
+
     bool String(const Ch*, SizeType, bool) { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_STRING; return true; }
 
     bool StartObject() { RAPIDJSON_ASSERT(LogCount < LogCapacity); Logs[LogCount++] = LOG_STARTOBJECT; return true; }
@@ -1349,12 +1351,13 @@ struct TerminateHandler {
     bool Int64(int64_t) { return e != 4; }
     bool Uint64(uint64_t) { return e != 5;  }
     bool Double(double) { return e != 6; }
-    bool String(const char*, SizeType, bool) { return e != 7; }
-    bool StartObject() { return e != 8; }
-    bool Key(const char*, SizeType, bool)  { return e != 9; }
-    bool EndObject(SizeType) { return e != 10; }
-    bool StartArray() { return e != 11; }
-    bool EndArray(SizeType) { return e != 12; }
+    bool RawNumber(const char*, SizeType, bool) { return e != 7; }
+    bool String(const char*, SizeType, bool) { return e != 8; }
+    bool StartObject() { return e != 9; }
+    bool Key(const char*, SizeType, bool)  { return e != 10; }
+    bool EndObject(SizeType) { return e != 11; }
+    bool StartArray() { return e != 12; }
+    bool EndArray(SizeType) { return e != 13; }
 };
 
 #define TEST_TERMINATION(e, json)\
@@ -1375,14 +1378,15 @@ TEST(Reader, ParseTerminationByHandler) {
     TEST_TERMINATION(4, "[-1234567890123456789");
     TEST_TERMINATION(5, "[1234567890123456789");
     TEST_TERMINATION(6, "[0.5]");
-    TEST_TERMINATION(7, "[\"a\"");
-    TEST_TERMINATION(8, "[{");
-    TEST_TERMINATION(9, "[{\"a\"");
-    TEST_TERMINATION(10, "[{}");
-    TEST_TERMINATION(10, "[{\"a\":1}"); // non-empty object
-    TEST_TERMINATION(11, "{\"a\":[");
-    TEST_TERMINATION(12, "{\"a\":[]");
-    TEST_TERMINATION(12, "{\"a\":[1]"); // non-empty array
+    // RawNumber() is never called
+    TEST_TERMINATION(8, "[\"a\"");
+    TEST_TERMINATION(9, "[{");
+    TEST_TERMINATION(10, "[{\"a\"");
+    TEST_TERMINATION(11, "[{}");
+    TEST_TERMINATION(11, "[{\"a\":1}"); // non-empty object
+    TEST_TERMINATION(12, "{\"a\":[");
+    TEST_TERMINATION(13, "{\"a\":[]");
+    TEST_TERMINATION(13, "{\"a\":[1]"); // non-empty array
 }
 
 TEST(Reader, ParseComments) {
@@ -1508,6 +1512,46 @@ TEST(Reader, UnrecognizedComment) {
     EXPECT_EQ(kParseErrorUnspecificSyntaxError, reader.GetParseErrorCode());
 }
 
+struct NumbersAsStringsHandler {
+    bool Null() { return true; }
+    bool Bool(bool) { return true; }
+    bool Int(int) { return true; }
+    bool Uint(unsigned) { return true; }
+    bool Int64(int64_t) { return true; }
+    bool Uint64(uint64_t) { return true;  }
+    bool Double(double) { return true; }
+    // 'str' is not null-terminated
+    bool RawNumber(const char* str, SizeType length, bool) {
+        EXPECT_TRUE(str != 0);
+        EXPECT_TRUE(strncmp(str, "3.1416", length) == 0);
+        return true;
+    }
+    bool String(const char*, SizeType, bool) { return true; }
+    bool StartObject() { return true; }
+    bool Key(const char*, SizeType, bool) { return true; }
+    bool EndObject(SizeType) { return true; }
+    bool StartArray() { return true; }
+    bool EndArray(SizeType) { return true; }
+};
+
+TEST(Reader, NumbersAsStrings) {
+	{
+      const char* json = "{ \"pi\": 3.1416 } ";
+		StringStream s(json);
+		NumbersAsStringsHandler h;
+		Reader reader;
+		EXPECT_TRUE(reader.Parse<kParseNumbersAsStringsFlag>(s, h));
+	}
+	{
+      char* json = StrDup("{ \"pi\": 3.1416 } ");
+		InsituStringStream s(json);
+		NumbersAsStringsHandler h;
+		Reader reader;
+		EXPECT_TRUE(reader.Parse<kParseInsituFlag|kParseNumbersAsStringsFlag>(s, h));
+      free(json);
+	}
+}
+
 #ifdef __GNUC__
 RAPIDJSON_DIAG_POP
 #endif

test/unittest/valuetest.cpp

@@ -1644,12 +1644,13 @@ struct TerminateHandler {
     bool Int64(int64_t) { return e != 4; }
     bool Uint64(uint64_t) { return e != 5; }
     bool Double(double) { return e != 6; }
-    bool String(const char*, SizeType, bool) { return e != 7; }
-    bool StartObject() { return e != 8; }
-    bool Key(const char*, SizeType, bool)  { return e != 9; }
-    bool EndObject(SizeType) { return e != 10; }
-    bool StartArray() { return e != 11; }
-    bool EndArray(SizeType) { return e != 12; }
+    bool RawNumber(const char*, SizeType, bool) { return e != 7; }
+    bool String(const char*, SizeType, bool) { return e != 8; }
+    bool StartObject() { return e != 9; }
+    bool Key(const char*, SizeType, bool)  { return e != 10; }
+    bool EndObject(SizeType) { return e != 11; }
+    bool StartArray() { return e != 12; }
+    bool EndArray(SizeType) { return e != 13; }
 };
 
 #define TEST_TERMINATION(e, json)\
@@ -1670,12 +1671,13 @@ TEST(Value, AcceptTerminationByHandler) {
     TEST_TERMINATION(4, "[-1234567890123456789]");
     TEST_TERMINATION(5, "[9223372036854775808]");
     TEST_TERMINATION(6, "[0.5]");
-    TEST_TERMINATION(7, "[\"a\"]");
-    TEST_TERMINATION(8, "[{}]");
-    TEST_TERMINATION(9, "[{\"a\":1}]");
-    TEST_TERMINATION(10, "[{}]");
-    TEST_TERMINATION(11, "{\"a\":[]}");
+    // RawNumber() is never called
+    TEST_TERMINATION(8, "[\"a\"]");
+    TEST_TERMINATION(9, "[{}]");
+    TEST_TERMINATION(10, "[{\"a\":1}]");
+    TEST_TERMINATION(11, "[{}]");
     TEST_TERMINATION(12, "{\"a\":[]}");
+    TEST_TERMINATION(13, "{\"a\":[]}");
 }
 
 struct ValueIntComparer {