Commit 47849a28 authored by Milo Yip's avatar Milo Yip

Merge pull request #81 from miloyip/IterativeParseOptimization

Iterative parse optimization
parents c4ce48cd 2a389ad6
...@@ -1221,13 +1221,12 @@ public: ...@@ -1221,13 +1221,12 @@ public:
\tparam SourceEncoding Encoding of input stream \tparam SourceEncoding Encoding of input stream
\tparam InputStream Type of input stream, implementing Stream concept \tparam InputStream Type of input stream, implementing Stream concept
\param is Input stream to be parsed. \param is Input stream to be parsed.
\param limit Parsing stack size limit(in bytes). Pass 0 means no limit.
\return The document itself for fluent API. \return The document itself for fluent API.
*/ */
template <unsigned parseFlags, typename SourceEncoding, typename InputStream> template <unsigned parseFlags, typename SourceEncoding, typename InputStream>
GenericDocument& ParseStream(InputStream& is, size_t limit = 0) { GenericDocument& ParseStream(InputStream& is) {
ValueType::SetNull(); // Remove existing root if exist ValueType::SetNull(); // Remove existing root if exist
GenericReader<SourceEncoding, Encoding, Allocator> reader(limit, &GetAllocator()); GenericReader<SourceEncoding, Encoding, Allocator> reader(&GetAllocator());
ClearStackOnExit scope(*this); ClearStackOnExit scope(*this);
parseResult_ = reader.template Parse<parseFlags>(is, *this); parseResult_ = reader.template Parse<parseFlags>(is, *this);
if (parseResult_) { if (parseResult_) {
...@@ -1241,23 +1240,21 @@ public: ...@@ -1241,23 +1240,21 @@ public:
/*! \tparam parseFlags Combination of \ref ParseFlag. /*! \tparam parseFlags Combination of \ref ParseFlag.
\tparam InputStream Type of input stream, implementing Stream concept \tparam InputStream Type of input stream, implementing Stream concept
\param is Input stream to be parsed. \param is Input stream to be parsed.
\param limit Parsing stack size limit(in bytes). Pass 0 means no limit.
\return The document itself for fluent API. \return The document itself for fluent API.
*/ */
template <unsigned parseFlags, typename InputStream> template <unsigned parseFlags, typename InputStream>
GenericDocument& ParseStream(InputStream& is, size_t limit = 0) { GenericDocument& ParseStream(InputStream& is) {
return ParseStream<parseFlags,Encoding,InputStream>(is, limit); return ParseStream<parseFlags,Encoding,InputStream>(is);
} }
//! Parse JSON text from an input stream (with \ref kParseDefaultFlags) //! Parse JSON text from an input stream (with \ref kParseDefaultFlags)
/*! \tparam InputStream Type of input stream, implementing Stream concept /*! \tparam InputStream Type of input stream, implementing Stream concept
\param is Input stream to be parsed. \param is Input stream to be parsed.
\param limit Parsing stack size limit(in bytes). Pass 0 means no limit.
\return The document itself for fluent API. \return The document itself for fluent API.
*/ */
template <typename InputStream> template <typename InputStream>
GenericDocument& ParseStream(InputStream& is, size_t limit = 0) { GenericDocument& ParseStream(InputStream& is) {
return ParseStream<kParseDefaultFlags, Encoding, InputStream>(is, limit); return ParseStream<kParseDefaultFlags, Encoding, InputStream>(is);
} }
//!@} //!@}
...@@ -1268,33 +1265,30 @@ public: ...@@ -1268,33 +1265,30 @@ public:
/*! \tparam parseFlags Combination of \ref ParseFlag. /*! \tparam parseFlags Combination of \ref ParseFlag.
\tparam SourceEncoding Transcoding from input Encoding \tparam SourceEncoding Transcoding from input Encoding
\param str Mutable zero-terminated string to be parsed. \param str Mutable zero-terminated string to be parsed.
\param limit Parsing stack size limit(in bytes). Pass 0 means no limit.
\return The document itself for fluent API. \return The document itself for fluent API.
*/ */
template <unsigned parseFlags, typename SourceEncoding> template <unsigned parseFlags, typename SourceEncoding>
GenericDocument& ParseInsitu(Ch* str, size_t limit = 0) { GenericDocument& ParseInsitu(Ch* str) {
GenericInsituStringStream<Encoding> s(str); GenericInsituStringStream<Encoding> s(str);
return ParseStream<parseFlags | kParseInsituFlag, SourceEncoding>(s, limit); return ParseStream<parseFlags | kParseInsituFlag, SourceEncoding>(s);
} }
//! Parse JSON text from a mutable string //! Parse JSON text from a mutable string
/*! \tparam parseFlags Combination of \ref ParseFlag. /*! \tparam parseFlags Combination of \ref ParseFlag.
\param str Mutable zero-terminated string to be parsed. \param str Mutable zero-terminated string to be parsed.
\param limit Parsing stack size limit(in bytes). Pass 0 means no limit.
\return The document itself for fluent API. \return The document itself for fluent API.
*/ */
template <unsigned parseFlags> template <unsigned parseFlags>
GenericDocument& ParseInsitu(Ch* str, size_t limit = 0) { GenericDocument& ParseInsitu(Ch* str) {
return ParseInsitu<parseFlags, Encoding>(str, limit); return ParseInsitu<parseFlags, Encoding>(str);
} }
//! Parse JSON text from a mutable string (with \ref kParseDefaultFlags) //! Parse JSON text from a mutable string (with \ref kParseDefaultFlags)
/*! \param str Mutable zero-terminated string to be parsed. /*! \param str Mutable zero-terminated string to be parsed.
\param limit Parsing stack size limit(in bytes). Pass 0 means no limit.
\return The document itself for fluent API. \return The document itself for fluent API.
*/ */
GenericDocument& ParseInsitu(Ch* str, size_t limit = 0) { GenericDocument& ParseInsitu(Ch* str) {
return ParseInsitu<kParseDefaultFlags, Encoding>(str, limit); return ParseInsitu<kParseDefaultFlags, Encoding>(str);
} }
//!@} //!@}
...@@ -1305,31 +1299,28 @@ public: ...@@ -1305,31 +1299,28 @@ public:
/*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag). /*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag).
\tparam SourceEncoding Transcoding from input Encoding \tparam SourceEncoding Transcoding from input Encoding
\param str Read-only zero-terminated string to be parsed. \param str Read-only zero-terminated string to be parsed.
\param limit Parsing stack size limit(in bytes). Pass 0 means no limit.
*/ */
template <unsigned parseFlags, typename SourceEncoding> template <unsigned parseFlags, typename SourceEncoding>
GenericDocument& Parse(const Ch* str, size_t limit = 0) { GenericDocument& Parse(const Ch* str) {
RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag)); RAPIDJSON_ASSERT(!(parseFlags & kParseInsituFlag));
GenericStringStream<SourceEncoding> s(str); GenericStringStream<SourceEncoding> s(str);
return ParseStream<parseFlags, SourceEncoding>(s, limit); return ParseStream<parseFlags, SourceEncoding>(s);
} }
//! Parse JSON text from a read-only string //! Parse JSON text from a read-only string
/*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag). /*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag).
\param str Read-only zero-terminated string to be parsed. \param str Read-only zero-terminated string to be parsed.
\param limit Parsing stack size limit(in bytes). Pass 0 means no limit.
*/ */
template <unsigned parseFlags> template <unsigned parseFlags>
GenericDocument& Parse(const Ch* str, size_t limit = 0) { GenericDocument& Parse(const Ch* str) {
return Parse<parseFlags, Encoding>(str, limit); return Parse<parseFlags, Encoding>(str);
} }
//! Parse JSON text from a read-only string (with \ref kParseDefaultFlags) //! Parse JSON text from a read-only string (with \ref kParseDefaultFlags)
/*! \param str Read-only zero-terminated string to be parsed. /*! \param str Read-only zero-terminated string to be parsed.
\param limit Parsing stack size limit(in bytes). Pass 0 means no limit.
*/ */
GenericDocument& Parse(const Ch* str, size_t limit = 0) { GenericDocument& Parse(const Ch* str) {
return Parse<kParseDefaultFlags>(str, limit); return Parse<kParseDefaultFlags>(str);
} }
//!@} //!@}
......
...@@ -40,7 +40,6 @@ inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErro ...@@ -40,7 +40,6 @@ inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErro
case kParseErrorTermination: return RAPIDJSON_ERROR_STRING("Terminate parsing due to Handler error."); case kParseErrorTermination: return RAPIDJSON_ERROR_STRING("Terminate parsing due to Handler error.");
case kParseErrorUnspecificSyntaxError: return RAPIDJSON_ERROR_STRING("Unspecific syntax error."); case kParseErrorUnspecificSyntaxError: return RAPIDJSON_ERROR_STRING("Unspecific syntax error.");
case kParseErrorStackSizeLimitExceeded: return RAPIDJSON_ERROR_STRING("Parsing stack size limit is exceeded.");
default: default:
return RAPIDJSON_ERROR_STRING("Unknown error."); return RAPIDJSON_ERROR_STRING("Unknown error.");
......
...@@ -60,7 +60,6 @@ enum ParseErrorCode { ...@@ -60,7 +60,6 @@ enum ParseErrorCode {
kParseErrorTermination, //!< Parsing was terminated. kParseErrorTermination, //!< Parsing was terminated.
kParseErrorUnspecificSyntaxError, //!< Unspecific syntax error. kParseErrorUnspecificSyntaxError, //!< Unspecific syntax error.
kParseErrorStackSizeLimitExceeded //!< Parsing stack size limit is exceeded.
}; };
//! Result of parsing (wraps ParseErrorCode) //! Result of parsing (wraps ParseErrorCode)
......
...@@ -28,20 +28,14 @@ public: ...@@ -28,20 +28,14 @@ public:
void Clear() { /*stack_top_ = 0;*/ stack_top_ = stack_; } void Clear() { /*stack_top_ = 0;*/ stack_top_ = stack_; }
// Optimization note: try to minimize the size of this function for force inline.
// Expansion is run very infrequently, so it is moved to another (probably non-inline) function.
template<typename T> template<typename T>
T* Push(size_t count = 1) { RAPIDJSON_FORCEINLINE T* Push(size_t count = 1) {
// Expand the stack if needed // Expand the stack if needed
if (stack_top_ + sizeof(T) * count >= stack_end_) { if (stack_top_ + sizeof(T) * count >= stack_end_)
size_t new_capacity = stack_capacity_ * 2; Expand<T>(count);
size_t size = GetSize();
size_t new_size = GetSize() + sizeof(T) * count;
if (new_capacity < new_size)
new_capacity = new_size;
stack_ = (char*)allocator_->Realloc(stack_, stack_capacity_, new_capacity);
stack_capacity_ = new_capacity;
stack_top_ = stack_ + size;
stack_end_ = stack_ + stack_capacity_;
}
T* ret = reinterpret_cast<T*>(stack_top_); T* ret = reinterpret_cast<T*>(stack_top_);
stack_top_ += sizeof(T) * count; stack_top_ += sizeof(T) * count;
return ret; return ret;
...@@ -69,6 +63,19 @@ public: ...@@ -69,6 +63,19 @@ public:
size_t GetCapacity() const { return stack_capacity_; } size_t GetCapacity() const { return stack_capacity_; }
private: private:
template<typename T>
void Expand(size_t count) {
size_t new_capacity = stack_capacity_ * 2;
size_t size = GetSize();
size_t new_size = GetSize() + sizeof(T) * count;
if (new_capacity < new_size)
new_capacity = new_size;
stack_ = (char*)allocator_->Realloc(stack_, stack_capacity_, new_capacity);
stack_capacity_ = new_capacity;
stack_top_ = stack_ + size;
stack_end_ = stack_ + stack_capacity_;
}
// Prohibit copy constructor & assignment operator. // Prohibit copy constructor & assignment operator.
Stack(const Stack&); Stack(const Stack&);
Stack& operator=(const Stack&); Stack& operator=(const Stack&);
......
...@@ -41,6 +41,8 @@ ...@@ -41,6 +41,8 @@
#ifndef RAPIDJSON_FORCEINLINE #ifndef RAPIDJSON_FORCEINLINE
#ifdef _MSC_VER #ifdef _MSC_VER
#define RAPIDJSON_FORCEINLINE __forceinline #define RAPIDJSON_FORCEINLINE __forceinline
#elif defined(__GNUC__) && __GNUC__ >= 4
#define RAPIDJSON_FORCEINLINE __attribute__((always_inline))
#else #else
#define RAPIDJSON_FORCEINLINE #define RAPIDJSON_FORCEINLINE
#endif #endif
......
...@@ -273,11 +273,10 @@ public: ...@@ -273,11 +273,10 @@ public:
typedef typename SourceEncoding::Ch Ch; //!< SourceEncoding character type typedef typename SourceEncoding::Ch Ch; //!< SourceEncoding character type
//! Constructor. //! Constructor.
/*! \param limit Parsing stack size limit(in bytes). Pass 0 means no limit. /*! \param allocator Optional allocator for allocating stack memory. (Only use for non-destructive parsing)
\param allocator Optional allocator for allocating stack memory. (Only use for non-destructive parsing)
\param stackCapacity stack capacity in bytes for storing a single decoded string. (Only use for non-destructive parsing) \param stackCapacity stack capacity in bytes for storing a single decoded string. (Only use for non-destructive parsing)
*/ */
GenericReader(size_t limit = 0, Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity) : stack_(allocator, stackCapacity), kStackSizeLimit_(limit), parseResult_() {} GenericReader(Allocator* allocator = 0, size_t stackCapacity = kDefaultStackCapacity) : stack_(allocator, stackCapacity), parseResult_() {}
//! Parse JSON text. //! Parse JSON text.
/*! \tparam parseFlags Combination of \ref ParseFlag. /*! \tparam parseFlags Combination of \ref ParseFlag.
...@@ -513,7 +512,7 @@ private: ...@@ -513,7 +512,7 @@ private:
typedef typename TargetEncoding::Ch Ch; typedef typename TargetEncoding::Ch Ch;
StackStream(internal::Stack<Allocator>& stack) : stack_(stack), length_(0) {} StackStream(internal::Stack<Allocator>& stack) : stack_(stack), length_(0) {}
void Put(Ch c) { RAPIDJSON_FORCEINLINE void Put(Ch c) {
*stack_.template Push<Ch>() = c; *stack_.template Push<Ch>() = c;
++length_; ++length_;
} }
...@@ -572,11 +571,6 @@ private: ...@@ -572,11 +571,6 @@ private:
is.Take(); is.Take();
Ch e = is.Take(); Ch e = is.Take();
if ((sizeof(Ch) == 1 || unsigned(e) < 256) && escape[(unsigned char)e]) { if ((sizeof(Ch) == 1 || unsigned(e) < 256) && escape[(unsigned char)e]) {
if (!(parseFlags & kParseInsituFlag)) {
if (!CheckStackSpaceQuota(sizeof(Ch))) {
RAPIDJSON_PARSE_ERROR(kParseErrorStackSizeLimitExceeded, is.Tell() - 1);
}
}
os.Put(escape[(unsigned char)e]); os.Put(escape[(unsigned char)e]);
} }
else if (e == 'u') { // Unicode else if (e == 'u') { // Unicode
...@@ -597,11 +591,6 @@ private: ...@@ -597,11 +591,6 @@ private:
} }
else if (c == '"') { // Closing double quote else if (c == '"') { // Closing double quote
is.Take(); is.Take();
if (!(parseFlags & kParseInsituFlag)) {
if (!CheckStackSpaceQuota(sizeof(Ch))) {
RAPIDJSON_PARSE_ERROR(kParseErrorStackSizeLimitExceeded, is.Tell() - 1);
}
}
os.Put('\0'); // null-terminate the string os.Put('\0'); // null-terminate the string
return; return;
} }
...@@ -829,44 +818,52 @@ private: ...@@ -829,44 +818,52 @@ private:
}; };
// Tokens // Tokens
enum IterativeParsingToken { enum Token {
IterativeParsingLeftBracketToken = 0, LeftBracketToken = 0,
IterativeParsingRightBracketToken, RightBracketToken,
IterativeParsingLeftCurlyBracketToken, LeftCurlyBracketToken,
IterativeParsingRightCurlyBracketToken, RightCurlyBracketToken,
IterativeParsingCommaToken, CommaToken,
IterativeParsingColonToken, ColonToken,
IterativeParsingStringToken, StringToken,
IterativeParsingFalseToken, FalseToken,
IterativeParsingTrueToken, TrueToken,
IterativeParsingNullToken, NullToken,
IterativeParsingNumberToken, NumberToken,
cIterativeParsingTokenCount kTokenCount
}; };
IterativeParsingToken Tokenize(Ch c) { RAPIDJSON_FORCEINLINE Token Tokenize(Ch c) {
switch (c) { #define N NumberToken
case '[': return IterativeParsingLeftBracketToken; #define N16 N,N,N,N,N,N,N,N,N,N,N,N,N,N,N,N
case ']': return IterativeParsingRightBracketToken; // Maps from ASCII to Token
case '{': return IterativeParsingLeftCurlyBracketToken; static const unsigned char tokenMap[256] = {
case '}': return IterativeParsingRightCurlyBracketToken; N16, // 00~0F
case ',': return IterativeParsingCommaToken; N16, // 10~1F
case ':': return IterativeParsingColonToken; N, N, StringToken, N, N, N, N, N, N, N, N, N, CommaToken, N, N, N, // 20~2F
case '"': return IterativeParsingStringToken; N, N, N, N, N, N, N, N, N, N, ColonToken, N, N, N, N, N, // 30~3F
case 'f': return IterativeParsingFalseToken; N16, // 40~4F
case 't': return IterativeParsingTrueToken; N, N, N, N, N, N, N, N, N, N, N, LeftBracketToken, N, RightBracketToken, N, N, // 50~5F
case 'n': return IterativeParsingNullToken; N, N, N, N, N, N, FalseToken, N, N, N, N, N, N, N, NullToken, N, // 60~6F
default: return IterativeParsingNumberToken; N, N, N, N, TrueToken, N, N, N, N, N, N, LeftCurlyBracketToken, N, RightCurlyBracketToken, N, N, // 70~7F
} N16, N16, N16, N16, N16, N16, N16, N16 // 80~FF
};
#undef N
#undef N16
if (sizeof(Ch) == 1 || unsigned(c) < 256)
return (Token)tokenMap[(unsigned char)c];
else
return NumberToken;
} }
IterativeParsingState Predict(IterativeParsingState state, IterativeParsingToken token) { RAPIDJSON_FORCEINLINE IterativeParsingState Predict(IterativeParsingState state, Token token) {
// current state x one lookahead token -> new state // current state x one lookahead token -> new state
static const char G[cIterativeParsingStateCount][cIterativeParsingTokenCount] = { static const char G[cIterativeParsingStateCount][kTokenCount] = {
// Start // Start
{ {
IterativeParsingArrayInitialState, // Left bracket IterativeParsingArrayInitialState, // Left bracket
...@@ -1025,11 +1022,7 @@ private: ...@@ -1025,11 +1022,7 @@ private:
// Make an advance in the token stream and state based on the candidate destination state which was returned by Transit(). // Make an advance in the token stream and state based on the candidate destination state which was returned by Transit().
// May return a new state on state pop. // May return a new state on state pop.
template <unsigned parseFlags, typename InputStream, typename Handler> template <unsigned parseFlags, typename InputStream, typename Handler>
IterativeParsingState Transit(IterativeParsingState src, IterativeParsingToken token, IterativeParsingState dst, InputStream& is, Handler& handler) { RAPIDJSON_FORCEINLINE IterativeParsingState Transit(IterativeParsingState src, Token token, IterativeParsingState dst, InputStream& is, Handler& handler) {
int c = 0;
IterativeParsingState n;
bool hr;
switch (dst) { switch (dst) {
case IterativeParsingStartState: case IterativeParsingStartState:
RAPIDJSON_ASSERT(false); RAPIDJSON_ASSERT(false);
...@@ -1043,27 +1036,20 @@ private: ...@@ -1043,27 +1036,20 @@ private:
case IterativeParsingObjectInitialState: case IterativeParsingObjectInitialState:
case IterativeParsingArrayInitialState: case IterativeParsingArrayInitialState:
{
// Push the state(Element or MemeberValue) if we are nested in another array or value of member. // Push the state(Element or MemeberValue) if we are nested in another array or value of member.
// In this way we can get the correct state on ObjectFinish or ArrayFinish by frame pop. // In this way we can get the correct state on ObjectFinish or ArrayFinish by frame pop.
n = src; IterativeParsingState n = src;
if (src == IterativeParsingArrayInitialState || src == IterativeParsingElementDelimiterState) if (src == IterativeParsingArrayInitialState || src == IterativeParsingElementDelimiterState)
n = IterativeParsingElementState; n = IterativeParsingElementState;
else if (src == IterativeParsingKeyValueDelimiterState) else if (src == IterativeParsingKeyValueDelimiterState)
n = IterativeParsingMemberValueState; n = IterativeParsingMemberValueState;
// Check stack space limit.
if (!CheckStackSpaceQuota(sizeof(IterativeParsingState) + sizeof(int))) {
RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorStackSizeLimitExceeded, is.Tell());
return IterativeParsingErrorState;
}
// Push current state. // Push current state.
*stack_.template Push<IterativeParsingState>(1) = n; *stack_.template Push<SizeType>(1) = n;
// Initialize and push the member/element count. // Initialize and push the member/element count.
*stack_.template Push<int>(1) = 0; *stack_.template Push<SizeType>(1) = 0;
// Call handler // Call handler
if (dst == IterativeParsingObjectInitialState) bool hr = (dst == IterativeParsingObjectInitialState) ? handler.StartObject() : handler.StartArray();
hr = handler.StartObject();
else
hr = handler.StartArray();
// On handler short circuits the parsing. // On handler short circuits the parsing.
if (!hr) { if (!hr) {
RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell());
...@@ -1073,6 +1059,7 @@ private: ...@@ -1073,6 +1059,7 @@ private:
is.Take(); is.Take();
return dst; return dst;
} }
}
case IterativeParsingMemberKeyState: case IterativeParsingMemberKeyState:
ParseString<parseFlags>(is, handler); ParseString<parseFlags>(is, handler);
...@@ -1082,7 +1069,7 @@ private: ...@@ -1082,7 +1069,7 @@ private:
return dst; return dst;
case IterativeParsingKeyValueDelimiterState: case IterativeParsingKeyValueDelimiterState:
if (token == IterativeParsingColonToken) { if (token == ColonToken) {
is.Take(); is.Take();
return dst; return dst;
} }
...@@ -1109,22 +1096,23 @@ private: ...@@ -1109,22 +1096,23 @@ private:
case IterativeParsingElementDelimiterState: case IterativeParsingElementDelimiterState:
is.Take(); is.Take();
// Update member/element count. // Update member/element count.
*stack_.template Top<int>() = *stack_.template Top<int>() + 1; *stack_.template Top<SizeType>() = *stack_.template Top<SizeType>() + 1;
return dst; return dst;
case IterativeParsingObjectFinishState: case IterativeParsingObjectFinishState:
{
// Get member count. // Get member count.
c = *stack_.template Pop<int>(1); SizeType c = *stack_.template Pop<SizeType>(1);
// If the object is not empty, count the last member. // If the object is not empty, count the last member.
if (src == IterativeParsingMemberValueState) if (src == IterativeParsingMemberValueState)
++c; ++c;
// Restore the state. // Restore the state.
n = *stack_.template Pop<IterativeParsingState>(1); IterativeParsingState n = static_cast<IterativeParsingState>(*stack_.template Pop<SizeType>(1));
// Transit to Finish state if this is the topmost scope. // Transit to Finish state if this is the topmost scope.
if (n == IterativeParsingStartState) if (n == IterativeParsingStartState)
n = IterativeParsingFinishState; n = IterativeParsingFinishState;
// Call handler // Call handler
hr = handler.EndObject(c); bool hr = handler.EndObject(c);
// On handler short circuits the parsing. // On handler short circuits the parsing.
if (!hr) { if (!hr) {
RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell());
...@@ -1134,20 +1122,22 @@ private: ...@@ -1134,20 +1122,22 @@ private:
is.Take(); is.Take();
return n; return n;
} }
}
case IterativeParsingArrayFinishState: case IterativeParsingArrayFinishState:
{
// Get element count. // Get element count.
c = *stack_.template Pop<int>(1); SizeType c = *stack_.template Pop<SizeType>(1);
// If the array is not empty, count the last element. // If the array is not empty, count the last element.
if (src == IterativeParsingElementState) if (src == IterativeParsingElementState)
++c; ++c;
// Restore the state. // Restore the state.
n = *stack_.template Pop<IterativeParsingState>(1); IterativeParsingState n = static_cast<IterativeParsingState>(*stack_.template Pop<SizeType>(1));
// Transit to Finish state if this is the topmost scope. // Transit to Finish state if this is the topmost scope.
if (n == IterativeParsingStartState) if (n == IterativeParsingStartState)
n = IterativeParsingFinishState; n = IterativeParsingFinishState;
// Call handler // Call handler
hr = handler.EndArray(c); bool hr = handler.EndArray(c);
// On handler short circuits the parsing. // On handler short circuits the parsing.
if (!hr) { if (!hr) {
RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell()); RAPIDJSON_PARSE_ERROR_NORETURN(kParseErrorTermination, is.Tell());
...@@ -1157,6 +1147,7 @@ private: ...@@ -1157,6 +1147,7 @@ private:
is.Take(); is.Take();
return n; return n;
} }
}
default: default:
RAPIDJSON_ASSERT(false); RAPIDJSON_ASSERT(false);
...@@ -1171,29 +1162,16 @@ private: ...@@ -1171,29 +1162,16 @@ private:
return; return;
} }
if (src == IterativeParsingStartState && is.Peek() == '\0') switch (src) {
RAPIDJSON_PARSE_ERROR(kParseErrorDocumentEmpty, is.Tell()); case IterativeParsingStartState: RAPIDJSON_PARSE_ERROR(is.Peek() == '\0' ? kParseErrorDocumentEmpty : kParseErrorDocumentRootNotObjectOrArray, is.Tell());
case IterativeParsingFinishState: RAPIDJSON_PARSE_ERROR(kParseErrorDocumentRootNotSingular, is.Tell());
else if (src == IterativeParsingStartState) case IterativeParsingObjectInitialState:
RAPIDJSON_PARSE_ERROR(kParseErrorDocumentRootNotObjectOrArray, is.Tell()); case IterativeParsingMemberDelimiterState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell());
case IterativeParsingMemberKeyState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell());
else if (src == IterativeParsingFinishState) case IterativeParsingMemberValueState: RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell());
RAPIDJSON_PARSE_ERROR(kParseErrorDocumentRootNotSingular, is.Tell()); case IterativeParsingElementState: RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell());
default: RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell());
else if (src == IterativeParsingObjectInitialState || src == IterativeParsingMemberDelimiterState) }
RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissName, is.Tell());
else if (src == IterativeParsingMemberKeyState)
RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissColon, is.Tell());
else if (src == IterativeParsingMemberValueState)
RAPIDJSON_PARSE_ERROR(kParseErrorObjectMissCommaOrCurlyBracket, is.Tell());
else if (src == IterativeParsingElementState)
RAPIDJSON_PARSE_ERROR(kParseErrorArrayMissCommaOrSquareBracket, is.Tell());
else
RAPIDJSON_PARSE_ERROR(kParseErrorUnspecificSyntaxError, is.Tell());
} }
template <unsigned parseFlags, typename InputStream, typename Handler> template <unsigned parseFlags, typename InputStream, typename Handler>
...@@ -1204,7 +1182,7 @@ private: ...@@ -1204,7 +1182,7 @@ private:
SkipWhitespace(is); SkipWhitespace(is);
while (is.Peek() != '\0') { while (is.Peek() != '\0') {
IterativeParsingToken t = Tokenize(is.Peek()); Token t = Tokenize(is.Peek());
IterativeParsingState n = Predict(state, t); IterativeParsingState n = Predict(state, t);
IterativeParsingState d = Transit<parseFlags>(state, t, n, is, handler); IterativeParsingState d = Transit<parseFlags>(state, t, n, is, handler);
...@@ -1224,13 +1202,8 @@ private: ...@@ -1224,13 +1202,8 @@ private:
return parseResult_; return parseResult_;
} }
bool CheckStackSpaceQuota(size_t size) const {
return kStackSizeLimit_ == 0 || (stack_.GetSize() + size <= kStackSizeLimit_);
}
static const size_t kDefaultStackCapacity = 256; //!< Default stack capacity in bytes for storing a single decoded string. static const size_t kDefaultStackCapacity = 256; //!< Default stack capacity in bytes for storing a single decoded string.
internal::Stack<Allocator> stack_; //!< A stack for storing decoded string temporarily during non-destructive parsing. internal::Stack<Allocator> stack_; //!< A stack for storing decoded string temporarily during non-destructive parsing.
const size_t kStackSizeLimit_; //!< Stack size limit(in bytes). A value of 0 means no limit.
ParseResult parseResult_; ParseResult parseResult_;
}; // class GenericReader }; // class GenericReader
......
...@@ -104,48 +104,37 @@ TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler_ValidateEncoding)) { ...@@ -104,48 +104,37 @@ TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler_ValidateEncoding)) {
} }
TEST_F(RapidJson, SIMD_SUFFIX(DocumentParseInsitu_MemoryPoolAllocator)) { TEST_F(RapidJson, SIMD_SUFFIX(DocumentParseInsitu_MemoryPoolAllocator)) {
//const size_t userBufferSize = 128 * 1024;
//char* userBuffer = (char*)malloc(userBufferSize);
for (size_t i = 0; i < kTrialCount; i++) { for (size_t i = 0; i < kTrialCount; i++) {
memcpy(temp_, json_, length_ + 1); memcpy(temp_, json_, length_ + 1);
//MemoryPoolAllocator<> allocator(userBuffer, userBufferSize);
//Document doc(&allocator);
Document doc; Document doc;
doc.ParseInsitu(temp_); doc.ParseInsitu(temp_);
ASSERT_TRUE(doc.IsObject()); ASSERT_TRUE(doc.IsObject());
//if (i == 0) {
// size_t size = doc.GetAllocator().Size();
// size_t capacity = doc.GetAllocator().Capacity();
// size_t stack_capacity = doc.GetStackCapacity();
// size_t actual = size - stack_capacity;
// std::cout << "Size:" << size << " Capacity:" << capacity << " Stack:" << stack_capacity << " Actual:" << actual << std::endl;
//}
} }
}
//free(userBuffer); TEST_F(RapidJson, SIMD_SUFFIX(DocumentParseIterativeInsitu_MemoryPoolAllocator)) {
for (size_t i = 0; i < kTrialCount; i++) {
memcpy(temp_, json_, length_ + 1);
Document doc;
doc.ParseInsitu<kParseIterativeFlag>(temp_);
ASSERT_TRUE(doc.IsObject());
}
} }
TEST_F(RapidJson, SIMD_SUFFIX(DocumentParse_MemoryPoolAllocator)) { TEST_F(RapidJson, SIMD_SUFFIX(DocumentParse_MemoryPoolAllocator)) {
//const size_t userBufferSize = 128 * 1024;
//char* userBuffer = (char*)malloc(userBufferSize);
for (size_t i = 0; i < kTrialCount; i++) { for (size_t i = 0; i < kTrialCount; i++) {
//MemoryPoolAllocator<> allocator(userBuffer, userBufferSize);
//Document doc(&allocator);
Document doc; Document doc;
doc.Parse(json_); doc.Parse(json_);
ASSERT_TRUE(doc.IsObject()); ASSERT_TRUE(doc.IsObject());
//if (i == 0) {
// size_t size = doc.GetAllocator().Size();
// size_t capacity = doc.GetAllocator().Capacity();
// size_t stack_capacity = doc.GetStackCapacity();
// size_t actual = size - stack_capacity;
// std::cout << "Size:" << size << " Capacity:" << capacity << " Stack:" << stack_capacity << " Actual:" << actual << std::endl;
//}
} }
}
//free(userBuffer); TEST_F(RapidJson, SIMD_SUFFIX(DocumentParseIterative_MemoryPoolAllocator)) {
for (size_t i = 0; i < kTrialCount; i++) {
Document doc;
doc.Parse<kParseIterativeFlag>(json_);
ASSERT_TRUE(doc.IsObject());
}
} }
TEST_F(RapidJson, SIMD_SUFFIX(DocumentParse_CrtAllocator)) { TEST_F(RapidJson, SIMD_SUFFIX(DocumentParse_CrtAllocator)) {
......
...@@ -932,18 +932,6 @@ TEST(Reader, IterativeParsing_ShortCircuit) { ...@@ -932,18 +932,6 @@ TEST(Reader, IterativeParsing_ShortCircuit) {
} }
} }
TEST(Reader, IterativeParsing_LimitStackSize) {
BaseReaderHandler<> handler;
Reader reader(20);
StringStream is("[[[]]]");
ParseResult r = reader.Parse<kParseIterativeFlag>(is, handler);
EXPECT_TRUE(reader.HasParseError());
EXPECT_EQ(kParseErrorStackSizeLimitExceeded, r.Code());
EXPECT_EQ(2u, r.Offset());
}
#ifdef __GNUC__ #ifdef __GNUC__
RAPIDJSON_DIAG_POP RAPIDJSON_DIAG_POP
#endif #endif
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment