Skip to content
Projects
Groups
Snippets
Help
Loading...
Sign in / Register
Toggle navigation
R
rapidjson
Project
Project
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Packages
Packages
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
submodule
rapidjson
Commits
47849a28
Commit
47849a28
authored
Jul 27, 2014
by
Milo Yip
Browse files
Options
Browse Files
Download
Plain Diff
Merge pull request #81 from miloyip/IterativeParseOptimization
Iterative parse optimization
parents
c4ce48cd
2a389ad6
Show whitespace changes
Inline
Side-by-side
Showing
8 changed files
with
124 additions
and
176 deletions
+124
-176
document.h
include/rapidjson/document.h
+18
-27
en.h
include/rapidjson/error/en.h
+0
-1
error.h
include/rapidjson/error/error.h
+0
-1
stack.h
include/rapidjson/internal/stack.h
+19
-12
rapidjson.h
include/rapidjson/rapidjson.h
+2
-0
reader.h
include/rapidjson/reader.h
+70
-97
rapidjsontest.cpp
test/perftest/rapidjsontest.cpp
+15
-26
readertest.cpp
test/unittest/readertest.cpp
+0
-12
No files found.
include/rapidjson/document.h
View file @
47849a28
...
@@ -1221,13 +1221,12 @@ public:
...
@@ -1221,13 +1221,12 @@ public:
\tparam SourceEncoding Encoding of input stream
\tparam SourceEncoding Encoding of input stream
\tparam InputStream Type of input stream, implementing Stream concept
\tparam InputStream Type of input stream, implementing Stream concept
\param is Input stream to be parsed.
\param is Input stream to be parsed.
\param limit Parsing stack size limit(in bytes). Pass 0 means no limit.
\return The document itself for fluent API.
\return The document itself for fluent API.
*/
*/
template
<
unsigned
parseFlags
,
typename
SourceEncoding
,
typename
InputStream
>
template
<
unsigned
parseFlags
,
typename
SourceEncoding
,
typename
InputStream
>
GenericDocument
&
ParseStream
(
InputStream
&
is
,
size_t
limit
=
0
)
{
GenericDocument
&
ParseStream
(
InputStream
&
is
)
{
ValueType
::
SetNull
();
// Remove existing root if exist
ValueType
::
SetNull
();
// Remove existing root if exist
GenericReader
<
SourceEncoding
,
Encoding
,
Allocator
>
reader
(
limit
,
&
GetAllocator
());
GenericReader
<
SourceEncoding
,
Encoding
,
Allocator
>
reader
(
&
GetAllocator
());
ClearStackOnExit
scope
(
*
this
);
ClearStackOnExit
scope
(
*
this
);
parseResult_
=
reader
.
template
Parse
<
parseFlags
>
(
is
,
*
this
);
parseResult_
=
reader
.
template
Parse
<
parseFlags
>
(
is
,
*
this
);
if
(
parseResult_
)
{
if
(
parseResult_
)
{
...
@@ -1241,23 +1240,21 @@ public:
...
@@ -1241,23 +1240,21 @@ public:
/*! \tparam parseFlags Combination of \ref ParseFlag.
/*! \tparam parseFlags Combination of \ref ParseFlag.
\tparam InputStream Type of input stream, implementing Stream concept
\tparam InputStream Type of input stream, implementing Stream concept
\param is Input stream to be parsed.
\param is Input stream to be parsed.
\param limit Parsing stack size limit(in bytes). Pass 0 means no limit.
\return The document itself for fluent API.
\return The document itself for fluent API.
*/
*/
template
<
unsigned
parseFlags
,
typename
InputStream
>
template
<
unsigned
parseFlags
,
typename
InputStream
>
GenericDocument
&
ParseStream
(
InputStream
&
is
,
size_t
limit
=
0
)
{
GenericDocument
&
ParseStream
(
InputStream
&
is
)
{
return
ParseStream
<
parseFlags
,
Encoding
,
InputStream
>
(
is
,
limit
);
return
ParseStream
<
parseFlags
,
Encoding
,
InputStream
>
(
is
);
}
}
//! Parse JSON text from an input stream (with \ref kParseDefaultFlags)
//! Parse JSON text from an input stream (with \ref kParseDefaultFlags)
/*! \tparam InputStream Type of input stream, implementing Stream concept
/*! \tparam InputStream Type of input stream, implementing Stream concept
\param is Input stream to be parsed.
\param is Input stream to be parsed.
\param limit Parsing stack size limit(in bytes). Pass 0 means no limit.
\return The document itself for fluent API.
\return The document itself for fluent API.
*/
*/
template
<
typename
InputStream
>
template
<
typename
InputStream
>
GenericDocument
&
ParseStream
(
InputStream
&
is
,
size_t
limit
=
0
)
{
GenericDocument
&
ParseStream
(
InputStream
&
is
)
{
return
ParseStream
<
kParseDefaultFlags
,
Encoding
,
InputStream
>
(
is
,
limit
);
return
ParseStream
<
kParseDefaultFlags
,
Encoding
,
InputStream
>
(
is
);
}
}
//!@}
//!@}
...
@@ -1268,33 +1265,30 @@ public:
...
@@ -1268,33 +1265,30 @@ public:
/*! \tparam parseFlags Combination of \ref ParseFlag.
/*! \tparam parseFlags Combination of \ref ParseFlag.
\tparam SourceEncoding Transcoding from input Encoding
\tparam SourceEncoding Transcoding from input Encoding
\param str Mutable zero-terminated string to be parsed.
\param str Mutable zero-terminated string to be parsed.
\param limit Parsing stack size limit(in bytes). Pass 0 means no limit.
\return The document itself for fluent API.
\return The document itself for fluent API.
*/
*/
template
<
unsigned
parseFlags
,
typename
SourceEncoding
>
template
<
unsigned
parseFlags
,
typename
SourceEncoding
>
GenericDocument
&
ParseInsitu
(
Ch
*
str
,
size_t
limit
=
0
)
{
GenericDocument
&
ParseInsitu
(
Ch
*
str
)
{
GenericInsituStringStream
<
Encoding
>
s
(
str
);
GenericInsituStringStream
<
Encoding
>
s
(
str
);
return
ParseStream
<
parseFlags
|
kParseInsituFlag
,
SourceEncoding
>
(
s
,
limit
);
return
ParseStream
<
parseFlags
|
kParseInsituFlag
,
SourceEncoding
>
(
s
);
}
}
//! Parse JSON text from a mutable string
//! Parse JSON text from a mutable string
/*! \tparam parseFlags Combination of \ref ParseFlag.
/*! \tparam parseFlags Combination of \ref ParseFlag.
\param str Mutable zero-terminated string to be parsed.
\param str Mutable zero-terminated string to be parsed.
\param limit Parsing stack size limit(in bytes). Pass 0 means no limit.
\return The document itself for fluent API.
\return The document itself for fluent API.
*/
*/
template
<
unsigned
parseFlags
>
template
<
unsigned
parseFlags
>
GenericDocument
&
ParseInsitu
(
Ch
*
str
,
size_t
limit
=
0
)
{
GenericDocument
&
ParseInsitu
(
Ch
*
str
)
{
return
ParseInsitu
<
parseFlags
,
Encoding
>
(
str
,
limit
);
return
ParseInsitu
<
parseFlags
,
Encoding
>
(
str
);
}
}
//! Parse JSON text from a mutable string (with \ref kParseDefaultFlags)
//! Parse JSON text from a mutable string (with \ref kParseDefaultFlags)
/*! \param str Mutable zero-terminated string to be parsed.
/*! \param str Mutable zero-terminated string to be parsed.
\param limit Parsing stack size limit(in bytes). Pass 0 means no limit.
\return The document itself for fluent API.
\return The document itself for fluent API.
*/
*/
GenericDocument
&
ParseInsitu
(
Ch
*
str
,
size_t
limit
=
0
)
{
GenericDocument
&
ParseInsitu
(
Ch
*
str
)
{
return
ParseInsitu
<
kParseDefaultFlags
,
Encoding
>
(
str
,
limit
);
return
ParseInsitu
<
kParseDefaultFlags
,
Encoding
>
(
str
);
}
}
//!@}
//!@}
...
@@ -1305,31 +1299,28 @@ public:
...
@@ -1305,31 +1299,28 @@ public:
/*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag).
/*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag).
\tparam SourceEncoding Transcoding from input Encoding
\tparam SourceEncoding Transcoding from input Encoding
\param str Read-only zero-terminated string to be parsed.
\param str Read-only zero-terminated string to be parsed.
\param limit Parsing stack size limit(in bytes). Pass 0 means no limit.
*/
*/
template
<
unsigned
parseFlags
,
typename
SourceEncoding
>
template
<
unsigned
parseFlags
,
typename
SourceEncoding
>
GenericDocument
&
Parse
(
const
Ch
*
str
,
size_t
limit
=
0
)
{
GenericDocument
&
Parse
(
const
Ch
*
str
)
{
RAPIDJSON_ASSERT
(
!
(
parseFlags
&
kParseInsituFlag
));
RAPIDJSON_ASSERT
(
!
(
parseFlags
&
kParseInsituFlag
));
GenericStringStream
<
SourceEncoding
>
s
(
str
);
GenericStringStream
<
SourceEncoding
>
s
(
str
);
return
ParseStream
<
parseFlags
,
SourceEncoding
>
(
s
,
limit
);
return
ParseStream
<
parseFlags
,
SourceEncoding
>
(
s
);
}
}
//! Parse JSON text from a read-only string
//! Parse JSON text from a read-only string
/*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag).
/*! \tparam parseFlags Combination of \ref ParseFlag (must not contain \ref kParseInsituFlag).
\param str Read-only zero-terminated string to be parsed.
\param str Read-only zero-terminated string to be parsed.
\param limit Parsing stack size limit(in bytes). Pass 0 means no limit.
*/
*/
template
<
unsigned
parseFlags
>
template
<
unsigned
parseFlags
>
GenericDocument
&
Parse
(
const
Ch
*
str
,
size_t
limit
=
0
)
{
GenericDocument
&
Parse
(
const
Ch
*
str
)
{
return
Parse
<
parseFlags
,
Encoding
>
(
str
,
limit
);
return
Parse
<
parseFlags
,
Encoding
>
(
str
);
}
}
//! Parse JSON text from a read-only string (with \ref kParseDefaultFlags)
//! Parse JSON text from a read-only string (with \ref kParseDefaultFlags)
/*! \param str Read-only zero-terminated string to be parsed.
/*! \param str Read-only zero-terminated string to be parsed.
\param limit Parsing stack size limit(in bytes). Pass 0 means no limit.
*/
*/
GenericDocument
&
Parse
(
const
Ch
*
str
,
size_t
limit
=
0
)
{
GenericDocument
&
Parse
(
const
Ch
*
str
)
{
return
Parse
<
kParseDefaultFlags
>
(
str
,
limit
);
return
Parse
<
kParseDefaultFlags
>
(
str
);
}
}
//!@}
//!@}
...
...
include/rapidjson/error/en.h
View file @
47849a28
...
@@ -40,7 +40,6 @@ inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErro
...
@@ -40,7 +40,6 @@ inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErro
case
kParseErrorTermination
:
return
RAPIDJSON_ERROR_STRING
(
"Terminate parsing due to Handler error."
);
case
kParseErrorTermination
:
return
RAPIDJSON_ERROR_STRING
(
"Terminate parsing due to Handler error."
);
case
kParseErrorUnspecificSyntaxError
:
return
RAPIDJSON_ERROR_STRING
(
"Unspecific syntax error."
);
case
kParseErrorUnspecificSyntaxError
:
return
RAPIDJSON_ERROR_STRING
(
"Unspecific syntax error."
);
case
kParseErrorStackSizeLimitExceeded
:
return
RAPIDJSON_ERROR_STRING
(
"Parsing stack size limit is exceeded."
);
default
:
default
:
return
RAPIDJSON_ERROR_STRING
(
"Unknown error."
);
return
RAPIDJSON_ERROR_STRING
(
"Unknown error."
);
...
...
include/rapidjson/error/error.h
View file @
47849a28
...
@@ -60,7 +60,6 @@ enum ParseErrorCode {
...
@@ -60,7 +60,6 @@ enum ParseErrorCode {
kParseErrorTermination
,
//!< Parsing was terminated.
kParseErrorTermination
,
//!< Parsing was terminated.
kParseErrorUnspecificSyntaxError
,
//!< Unspecific syntax error.
kParseErrorUnspecificSyntaxError
,
//!< Unspecific syntax error.
kParseErrorStackSizeLimitExceeded
//!< Parsing stack size limit is exceeded.
};
};
//! Result of parsing (wraps ParseErrorCode)
//! Result of parsing (wraps ParseErrorCode)
...
...
include/rapidjson/internal/stack.h
View file @
47849a28
...
@@ -28,20 +28,14 @@ public:
...
@@ -28,20 +28,14 @@ public:
void
Clear
()
{
/*stack_top_ = 0;*/
stack_top_
=
stack_
;
}
void
Clear
()
{
/*stack_top_ = 0;*/
stack_top_
=
stack_
;
}
// Optimization note: try to minimize the size of this function for force inline.
// Expansion is run very infrequently, so it is moved to another (probably non-inline) function.
template
<
typename
T
>
template
<
typename
T
>
T
*
Push
(
size_t
count
=
1
)
{
RAPIDJSON_FORCEINLINE
T
*
Push
(
size_t
count
=
1
)
{
// Expand the stack if needed
// Expand the stack if needed
if
(
stack_top_
+
sizeof
(
T
)
*
count
>=
stack_end_
)
{
if
(
stack_top_
+
sizeof
(
T
)
*
count
>=
stack_end_
)
size_t
new_capacity
=
stack_capacity_
*
2
;
Expand
<
T
>
(
count
);
size_t
size
=
GetSize
();
size_t
new_size
=
GetSize
()
+
sizeof
(
T
)
*
count
;
if
(
new_capacity
<
new_size
)
new_capacity
=
new_size
;
stack_
=
(
char
*
)
allocator_
->
Realloc
(
stack_
,
stack_capacity_
,
new_capacity
);
stack_capacity_
=
new_capacity
;
stack_top_
=
stack_
+
size
;
stack_end_
=
stack_
+
stack_capacity_
;
}
T
*
ret
=
reinterpret_cast
<
T
*>
(
stack_top_
);
T
*
ret
=
reinterpret_cast
<
T
*>
(
stack_top_
);
stack_top_
+=
sizeof
(
T
)
*
count
;
stack_top_
+=
sizeof
(
T
)
*
count
;
return
ret
;
return
ret
;
...
@@ -69,6 +63,19 @@ public:
...
@@ -69,6 +63,19 @@ public:
size_t
GetCapacity
()
const
{
return
stack_capacity_
;
}
size_t
GetCapacity
()
const
{
return
stack_capacity_
;
}
private
:
private
:
template
<
typename
T
>
void
Expand
(
size_t
count
)
{
size_t
new_capacity
=
stack_capacity_
*
2
;
size_t
size
=
GetSize
();
size_t
new_size
=
GetSize
()
+
sizeof
(
T
)
*
count
;
if
(
new_capacity
<
new_size
)
new_capacity
=
new_size
;
stack_
=
(
char
*
)
allocator_
->
Realloc
(
stack_
,
stack_capacity_
,
new_capacity
);
stack_capacity_
=
new_capacity
;
stack_top_
=
stack_
+
size
;
stack_end_
=
stack_
+
stack_capacity_
;
}
// Prohibit copy constructor & assignment operator.
// Prohibit copy constructor & assignment operator.
Stack
(
const
Stack
&
);
Stack
(
const
Stack
&
);
Stack
&
operator
=
(
const
Stack
&
);
Stack
&
operator
=
(
const
Stack
&
);
...
...
include/rapidjson/rapidjson.h
View file @
47849a28
...
@@ -41,6 +41,8 @@
...
@@ -41,6 +41,8 @@
#ifndef RAPIDJSON_FORCEINLINE
#ifndef RAPIDJSON_FORCEINLINE
#ifdef _MSC_VER
#ifdef _MSC_VER
#define RAPIDJSON_FORCEINLINE __forceinline
#define RAPIDJSON_FORCEINLINE __forceinline
#elif defined(__GNUC__) && __GNUC__ >= 4
#define RAPIDJSON_FORCEINLINE __attribute__((always_inline))
#else
#else
#define RAPIDJSON_FORCEINLINE
#define RAPIDJSON_FORCEINLINE
#endif
#endif
...
...
include/rapidjson/reader.h
View file @
47849a28
...
@@ -273,11 +273,10 @@ public:
...
@@ -273,11 +273,10 @@ public:
typedef
typename
SourceEncoding
::
Ch
Ch
;
//!< SourceEncoding character type
typedef
typename
SourceEncoding
::
Ch
Ch
;
//!< SourceEncoding character type
//! Constructor.
//! Constructor.
/*! \param limit Parsing stack size limit(in bytes). Pass 0 means no limit.
/*! \param allocator Optional allocator for allocating stack memory. (Only use for non-destructive parsing)
\param allocator Optional allocator for allocating stack memory. (Only use for non-destructive parsing)
\param stackCapacity stack capacity in bytes for storing a single decoded string. (Only use for non-destructive parsing)
\param stackCapacity stack capacity in bytes for storing a single decoded string. (Only use for non-destructive parsing)
*/
*/
GenericReader
(
size_t
limit
=
0
,
Allocator
*
allocator
=
0
,
size_t
stackCapacity
=
kDefaultStackCapacity
)
:
stack_
(
allocator
,
stackCapacity
),
kStackSizeLimit_
(
limit
),
parseResult_
()
{}
GenericReader
(
Allocator
*
allocator
=
0
,
size_t
stackCapacity
=
kDefaultStackCapacity
)
:
stack_
(
allocator
,
stackCapacity
),
parseResult_
()
{}
//! Parse JSON text.
//! Parse JSON text.
/*! \tparam parseFlags Combination of \ref ParseFlag.
/*! \tparam parseFlags Combination of \ref ParseFlag.
...
@@ -513,7 +512,7 @@ private:
...
@@ -513,7 +512,7 @@ private:
typedef
typename
TargetEncoding
::
Ch
Ch
;
typedef
typename
TargetEncoding
::
Ch
Ch
;
StackStream
(
internal
::
Stack
<
Allocator
>&
stack
)
:
stack_
(
stack
),
length_
(
0
)
{}
StackStream
(
internal
::
Stack
<
Allocator
>&
stack
)
:
stack_
(
stack
),
length_
(
0
)
{}
void
Put
(
Ch
c
)
{
RAPIDJSON_FORCEINLINE
void
Put
(
Ch
c
)
{
*
stack_
.
template
Push
<
Ch
>
()
=
c
;
*
stack_
.
template
Push
<
Ch
>
()
=
c
;
++
length_
;
++
length_
;
}
}
...
@@ -572,11 +571,6 @@ private:
...
@@ -572,11 +571,6 @@ private:
is
.
Take
();
is
.
Take
();
Ch
e
=
is
.
Take
();
Ch
e
=
is
.
Take
();
if
((
sizeof
(
Ch
)
==
1
||
unsigned
(
e
)
<
256
)
&&
escape
[(
unsigned
char
)
e
])
{
if
((
sizeof
(
Ch
)
==
1
||
unsigned
(
e
)
<
256
)
&&
escape
[(
unsigned
char
)
e
])
{
if
(
!
(
parseFlags
&
kParseInsituFlag
))
{
if
(
!
CheckStackSpaceQuota
(
sizeof
(
Ch
)))
{
RAPIDJSON_PARSE_ERROR
(
kParseErrorStackSizeLimitExceeded
,
is
.
Tell
()
-
1
);
}
}
os
.
Put
(
escape
[(
unsigned
char
)
e
]);
os
.
Put
(
escape
[(
unsigned
char
)
e
]);
}
}
else
if
(
e
==
'u'
)
{
// Unicode
else
if
(
e
==
'u'
)
{
// Unicode
...
@@ -597,11 +591,6 @@ private:
...
@@ -597,11 +591,6 @@ private:
}
}
else
if
(
c
==
'"'
)
{
// Closing double quote
else
if
(
c
==
'"'
)
{
// Closing double quote
is
.
Take
();
is
.
Take
();
if
(
!
(
parseFlags
&
kParseInsituFlag
))
{
if
(
!
CheckStackSpaceQuota
(
sizeof
(
Ch
)))
{
RAPIDJSON_PARSE_ERROR
(
kParseErrorStackSizeLimitExceeded
,
is
.
Tell
()
-
1
);
}
}
os
.
Put
(
'\0'
);
// null-terminate the string
os
.
Put
(
'\0'
);
// null-terminate the string
return
;
return
;
}
}
...
@@ -829,44 +818,52 @@ private:
...
@@ -829,44 +818,52 @@ private:
};
};
// Tokens
// Tokens
enum
IterativeParsing
Token
{
enum
Token
{
IterativeParsing
LeftBracketToken
=
0
,
LeftBracketToken
=
0
,
IterativeParsing
RightBracketToken
,
RightBracketToken
,
IterativeParsing
LeftCurlyBracketToken
,
LeftCurlyBracketToken
,
IterativeParsing
RightCurlyBracketToken
,
RightCurlyBracketToken
,
IterativeParsing
CommaToken
,
CommaToken
,
IterativeParsing
ColonToken
,
ColonToken
,
IterativeParsing
StringToken
,
StringToken
,
IterativeParsing
FalseToken
,
FalseToken
,
IterativeParsing
TrueToken
,
TrueToken
,
IterativeParsing
NullToken
,
NullToken
,
IterativeParsing
NumberToken
,
NumberToken
,
cIterativeParsing
TokenCount
k
TokenCount
};
};
IterativeParsingToken
Tokenize
(
Ch
c
)
{
RAPIDJSON_FORCEINLINE
Token
Tokenize
(
Ch
c
)
{
switch
(
c
)
{
#define N NumberToken
case
'['
:
return
IterativeParsingLeftBracketToken
;
#define N16 N,N,N,N,N,N,N,N,N,N,N,N,N,N,N,N
case
']'
:
return
IterativeParsingRightBracketToken
;
// Maps from ASCII to Token
case
'{'
:
return
IterativeParsingLeftCurlyBracketToken
;
static
const
unsigned
char
tokenMap
[
256
]
=
{
case
'}'
:
return
IterativeParsingRightCurlyBracketToken
;
N16
,
// 00~0F
case
','
:
return
IterativeParsingCommaToken
;
N16
,
// 10~1F
case
':'
:
return
IterativeParsingColonToken
;
N
,
N
,
StringToken
,
N
,
N
,
N
,
N
,
N
,
N
,
N
,
N
,
N
,
CommaToken
,
N
,
N
,
N
,
// 20~2F
case
'"'
:
return
IterativeParsingStringToken
;
N
,
N
,
N
,
N
,
N
,
N
,
N
,
N
,
N
,
N
,
ColonToken
,
N
,
N
,
N
,
N
,
N
,
// 30~3F
case
'f'
:
return
IterativeParsingFalseToken
;
N16
,
// 40~4F
case
't'
:
return
IterativeParsingTrueToken
;
N
,
N
,
N
,
N
,
N
,
N
,
N
,
N
,
N
,
N
,
N
,
LeftBracketToken
,
N
,
RightBracketToken
,
N
,
N
,
// 50~5F
case
'n'
:
return
IterativeParsingNullToken
;
N
,
N
,
N
,
N
,
N
,
N
,
FalseToken
,
N
,
N
,
N
,
N
,
N
,
N
,
N
,
NullToken
,
N
,
// 60~6F
default:
return
IterativeParsingNumberToken
;
N
,
N
,
N
,
N
,
TrueToken
,
N
,
N
,
N
,
N
,
N
,
N
,
LeftCurlyBracketToken
,
N
,
RightCurlyBracketToken
,
N
,
N
,
// 70~7F
}
N16
,
N16
,
N16
,
N16
,
N16
,
N16
,
N16
,
N16
// 80~FF
};
#undef N
#undef N16
if
(
sizeof
(
Ch
)
==
1
||
unsigned
(
c
)
<
256
)
return
(
Token
)
tokenMap
[(
unsigned
char
)
c
];
else
return
NumberToken
;
}
}
IterativeParsingState
Predict
(
IterativeParsingState
state
,
IterativeParsing
Token
token
)
{
RAPIDJSON_FORCEINLINE
IterativeParsingState
Predict
(
IterativeParsingState
state
,
Token
token
)
{
// current state x one lookahead token -> new state
// current state x one lookahead token -> new state
static
const
char
G
[
cIterativeParsingStateCount
][
cIterativeParsing
TokenCount
]
=
{
static
const
char
G
[
cIterativeParsingStateCount
][
k
TokenCount
]
=
{
// Start
// Start
{
{
IterativeParsingArrayInitialState
,
// Left bracket
IterativeParsingArrayInitialState
,
// Left bracket
...
@@ -1025,11 +1022,7 @@ private:
...
@@ -1025,11 +1022,7 @@ private:
// Make an advance in the token stream and state based on the candidate destination state which was returned by Transit().
// Make an advance in the token stream and state based on the candidate destination state which was returned by Transit().
// May return a new state on state pop.
// May return a new state on state pop.
template
<
unsigned
parseFlags
,
typename
InputStream
,
typename
Handler
>
template
<
unsigned
parseFlags
,
typename
InputStream
,
typename
Handler
>
IterativeParsingState
Transit
(
IterativeParsingState
src
,
IterativeParsingToken
token
,
IterativeParsingState
dst
,
InputStream
&
is
,
Handler
&
handler
)
{
RAPIDJSON_FORCEINLINE
IterativeParsingState
Transit
(
IterativeParsingState
src
,
Token
token
,
IterativeParsingState
dst
,
InputStream
&
is
,
Handler
&
handler
)
{
int
c
=
0
;
IterativeParsingState
n
;
bool
hr
;
switch
(
dst
)
{
switch
(
dst
)
{
case
IterativeParsingStartState
:
case
IterativeParsingStartState
:
RAPIDJSON_ASSERT
(
false
);
RAPIDJSON_ASSERT
(
false
);
...
@@ -1043,27 +1036,20 @@ private:
...
@@ -1043,27 +1036,20 @@ private:
case
IterativeParsingObjectInitialState
:
case
IterativeParsingObjectInitialState
:
case
IterativeParsingArrayInitialState
:
case
IterativeParsingArrayInitialState
:
{
// Push the state(Element or MemeberValue) if we are nested in another array or value of member.
// Push the state(Element or MemeberValue) if we are nested in another array or value of member.
// In this way we can get the correct state on ObjectFinish or ArrayFinish by frame pop.
// In this way we can get the correct state on ObjectFinish or ArrayFinish by frame pop.
n
=
src
;
IterativeParsingState
n
=
src
;
if
(
src
==
IterativeParsingArrayInitialState
||
src
==
IterativeParsingElementDelimiterState
)
if
(
src
==
IterativeParsingArrayInitialState
||
src
==
IterativeParsingElementDelimiterState
)
n
=
IterativeParsingElementState
;
n
=
IterativeParsingElementState
;
else
if
(
src
==
IterativeParsingKeyValueDelimiterState
)
else
if
(
src
==
IterativeParsingKeyValueDelimiterState
)
n
=
IterativeParsingMemberValueState
;
n
=
IterativeParsingMemberValueState
;
// Check stack space limit.
if
(
!
CheckStackSpaceQuota
(
sizeof
(
IterativeParsingState
)
+
sizeof
(
int
)))
{
RAPIDJSON_PARSE_ERROR_NORETURN
(
kParseErrorStackSizeLimitExceeded
,
is
.
Tell
());
return
IterativeParsingErrorState
;
}
// Push current state.
// Push current state.
*
stack_
.
template
Push
<
IterativeParsingStat
e
>
(
1
)
=
n
;
*
stack_
.
template
Push
<
SizeTyp
e
>
(
1
)
=
n
;
// Initialize and push the member/element count.
// Initialize and push the member/element count.
*
stack_
.
template
Push
<
int
>
(
1
)
=
0
;
*
stack_
.
template
Push
<
SizeType
>
(
1
)
=
0
;
// Call handler
// Call handler
if
(
dst
==
IterativeParsingObjectInitialState
)
bool
hr
=
(
dst
==
IterativeParsingObjectInitialState
)
?
handler
.
StartObject
()
:
handler
.
StartArray
();
hr
=
handler
.
StartObject
();
else
hr
=
handler
.
StartArray
();
// On handler short circuits the parsing.
// On handler short circuits the parsing.
if
(
!
hr
)
{
if
(
!
hr
)
{
RAPIDJSON_PARSE_ERROR_NORETURN
(
kParseErrorTermination
,
is
.
Tell
());
RAPIDJSON_PARSE_ERROR_NORETURN
(
kParseErrorTermination
,
is
.
Tell
());
...
@@ -1073,6 +1059,7 @@ private:
...
@@ -1073,6 +1059,7 @@ private:
is
.
Take
();
is
.
Take
();
return
dst
;
return
dst
;
}
}
}
case
IterativeParsingMemberKeyState
:
case
IterativeParsingMemberKeyState
:
ParseString
<
parseFlags
>
(
is
,
handler
);
ParseString
<
parseFlags
>
(
is
,
handler
);
...
@@ -1082,7 +1069,7 @@ private:
...
@@ -1082,7 +1069,7 @@ private:
return
dst
;
return
dst
;
case
IterativeParsingKeyValueDelimiterState
:
case
IterativeParsingKeyValueDelimiterState
:
if
(
token
==
IterativeParsing
ColonToken
)
{
if
(
token
==
ColonToken
)
{
is
.
Take
();
is
.
Take
();
return
dst
;
return
dst
;
}
}
...
@@ -1109,22 +1096,23 @@ private:
...
@@ -1109,22 +1096,23 @@ private:
case
IterativeParsingElementDelimiterState
:
case
IterativeParsingElementDelimiterState
:
is
.
Take
();
is
.
Take
();
// Update member/element count.
// Update member/element count.
*
stack_
.
template
Top
<
int
>
()
=
*
stack_
.
template
Top
<
int
>
()
+
1
;
*
stack_
.
template
Top
<
SizeType
>
()
=
*
stack_
.
template
Top
<
SizeType
>
()
+
1
;
return
dst
;
return
dst
;
case
IterativeParsingObjectFinishState
:
case
IterativeParsingObjectFinishState
:
{
// Get member count.
// Get member count.
c
=
*
stack_
.
template
Pop
<
int
>
(
1
);
SizeType
c
=
*
stack_
.
template
Pop
<
SizeType
>
(
1
);
// If the object is not empty, count the last member.
// If the object is not empty, count the last member.
if
(
src
==
IterativeParsingMemberValueState
)
if
(
src
==
IterativeParsingMemberValueState
)
++
c
;
++
c
;
// Restore the state.
// Restore the state.
n
=
*
stack_
.
template
Pop
<
IterativeParsingState
>
(
1
);
IterativeParsingState
n
=
static_cast
<
IterativeParsingState
>
(
*
stack_
.
template
Pop
<
SizeType
>
(
1
)
);
// Transit to Finish state if this is the topmost scope.
// Transit to Finish state if this is the topmost scope.
if
(
n
==
IterativeParsingStartState
)
if
(
n
==
IterativeParsingStartState
)
n
=
IterativeParsingFinishState
;
n
=
IterativeParsingFinishState
;
// Call handler
// Call handler
hr
=
handler
.
EndObject
(
c
);
bool
hr
=
handler
.
EndObject
(
c
);
// On handler short circuits the parsing.
// On handler short circuits the parsing.
if
(
!
hr
)
{
if
(
!
hr
)
{
RAPIDJSON_PARSE_ERROR_NORETURN
(
kParseErrorTermination
,
is
.
Tell
());
RAPIDJSON_PARSE_ERROR_NORETURN
(
kParseErrorTermination
,
is
.
Tell
());
...
@@ -1134,20 +1122,22 @@ private:
...
@@ -1134,20 +1122,22 @@ private:
is
.
Take
();
is
.
Take
();
return
n
;
return
n
;
}
}
}
case
IterativeParsingArrayFinishState
:
case
IterativeParsingArrayFinishState
:
{
// Get element count.
// Get element count.
c
=
*
stack_
.
template
Pop
<
int
>
(
1
);
SizeType
c
=
*
stack_
.
template
Pop
<
SizeType
>
(
1
);
// If the array is not empty, count the last element.
// If the array is not empty, count the last element.
if
(
src
==
IterativeParsingElementState
)
if
(
src
==
IterativeParsingElementState
)
++
c
;
++
c
;
// Restore the state.
// Restore the state.
n
=
*
stack_
.
template
Pop
<
IterativeParsingState
>
(
1
);
IterativeParsingState
n
=
static_cast
<
IterativeParsingState
>
(
*
stack_
.
template
Pop
<
SizeType
>
(
1
)
);
// Transit to Finish state if this is the topmost scope.
// Transit to Finish state if this is the topmost scope.
if
(
n
==
IterativeParsingStartState
)
if
(
n
==
IterativeParsingStartState
)
n
=
IterativeParsingFinishState
;
n
=
IterativeParsingFinishState
;
// Call handler
// Call handler
hr
=
handler
.
EndArray
(
c
);
bool
hr
=
handler
.
EndArray
(
c
);
// On handler short circuits the parsing.
// On handler short circuits the parsing.
if
(
!
hr
)
{
if
(
!
hr
)
{
RAPIDJSON_PARSE_ERROR_NORETURN
(
kParseErrorTermination
,
is
.
Tell
());
RAPIDJSON_PARSE_ERROR_NORETURN
(
kParseErrorTermination
,
is
.
Tell
());
...
@@ -1157,6 +1147,7 @@ private:
...
@@ -1157,6 +1147,7 @@ private:
is
.
Take
();
is
.
Take
();
return
n
;
return
n
;
}
}
}
default
:
default
:
RAPIDJSON_ASSERT
(
false
);
RAPIDJSON_ASSERT
(
false
);
...
@@ -1171,29 +1162,16 @@ private:
...
@@ -1171,29 +1162,16 @@ private:
return
;
return
;
}
}
if
(
src
==
IterativeParsingStartState
&&
is
.
Peek
()
==
'\0'
)
switch
(
src
)
{
RAPIDJSON_PARSE_ERROR
(
kParseErrorDocumentEmpty
,
is
.
Tell
());
case
IterativeParsingStartState
:
RAPIDJSON_PARSE_ERROR
(
is
.
Peek
()
==
'\0'
?
kParseErrorDocumentEmpty
:
kParseErrorDocumentRootNotObjectOrArray
,
is
.
Tell
());
case
IterativeParsingFinishState
:
RAPIDJSON_PARSE_ERROR
(
kParseErrorDocumentRootNotSingular
,
is
.
Tell
());
else
if
(
src
==
IterativeParsingStartState
)
case
IterativeParsingObjectInitialState
:
RAPIDJSON_PARSE_ERROR
(
kParseErrorDocumentRootNotObjectOrArray
,
is
.
Tell
());
case
IterativeParsingMemberDelimiterState
:
RAPIDJSON_PARSE_ERROR
(
kParseErrorObjectMissName
,
is
.
Tell
());
case
IterativeParsingMemberKeyState
:
RAPIDJSON_PARSE_ERROR
(
kParseErrorObjectMissColon
,
is
.
Tell
());
else
if
(
src
==
IterativeParsingFinishState
)
case
IterativeParsingMemberValueState
:
RAPIDJSON_PARSE_ERROR
(
kParseErrorObjectMissCommaOrCurlyBracket
,
is
.
Tell
());
RAPIDJSON_PARSE_ERROR
(
kParseErrorDocumentRootNotSingular
,
is
.
Tell
());
case
IterativeParsingElementState
:
RAPIDJSON_PARSE_ERROR
(
kParseErrorArrayMissCommaOrSquareBracket
,
is
.
Tell
());
default
:
RAPIDJSON_PARSE_ERROR
(
kParseErrorUnspecificSyntaxError
,
is
.
Tell
());
else
if
(
src
==
IterativeParsingObjectInitialState
||
src
==
IterativeParsingMemberDelimiterState
)
}
RAPIDJSON_PARSE_ERROR
(
kParseErrorObjectMissName
,
is
.
Tell
());
else
if
(
src
==
IterativeParsingMemberKeyState
)
RAPIDJSON_PARSE_ERROR
(
kParseErrorObjectMissColon
,
is
.
Tell
());
else
if
(
src
==
IterativeParsingMemberValueState
)
RAPIDJSON_PARSE_ERROR
(
kParseErrorObjectMissCommaOrCurlyBracket
,
is
.
Tell
());
else
if
(
src
==
IterativeParsingElementState
)
RAPIDJSON_PARSE_ERROR
(
kParseErrorArrayMissCommaOrSquareBracket
,
is
.
Tell
());
else
RAPIDJSON_PARSE_ERROR
(
kParseErrorUnspecificSyntaxError
,
is
.
Tell
());
}
}
template
<
unsigned
parseFlags
,
typename
InputStream
,
typename
Handler
>
template
<
unsigned
parseFlags
,
typename
InputStream
,
typename
Handler
>
...
@@ -1204,7 +1182,7 @@ private:
...
@@ -1204,7 +1182,7 @@ private:
SkipWhitespace
(
is
);
SkipWhitespace
(
is
);
while
(
is
.
Peek
()
!=
'\0'
)
{
while
(
is
.
Peek
()
!=
'\0'
)
{
IterativeParsing
Token
t
=
Tokenize
(
is
.
Peek
());
Token
t
=
Tokenize
(
is
.
Peek
());
IterativeParsingState
n
=
Predict
(
state
,
t
);
IterativeParsingState
n
=
Predict
(
state
,
t
);
IterativeParsingState
d
=
Transit
<
parseFlags
>
(
state
,
t
,
n
,
is
,
handler
);
IterativeParsingState
d
=
Transit
<
parseFlags
>
(
state
,
t
,
n
,
is
,
handler
);
...
@@ -1224,13 +1202,8 @@ private:
...
@@ -1224,13 +1202,8 @@ private:
return
parseResult_
;
return
parseResult_
;
}
}
bool
CheckStackSpaceQuota
(
size_t
size
)
const
{
return
kStackSizeLimit_
==
0
||
(
stack_
.
GetSize
()
+
size
<=
kStackSizeLimit_
);
}
static
const
size_t
kDefaultStackCapacity
=
256
;
//!< Default stack capacity in bytes for storing a single decoded string.
static
const
size_t
kDefaultStackCapacity
=
256
;
//!< Default stack capacity in bytes for storing a single decoded string.
internal
::
Stack
<
Allocator
>
stack_
;
//!< A stack for storing decoded string temporarily during non-destructive parsing.
internal
::
Stack
<
Allocator
>
stack_
;
//!< A stack for storing decoded string temporarily during non-destructive parsing.
const
size_t
kStackSizeLimit_
;
//!< Stack size limit(in bytes). A value of 0 means no limit.
ParseResult
parseResult_
;
ParseResult
parseResult_
;
};
// class GenericReader
};
// class GenericReader
...
...
test/perftest/rapidjsontest.cpp
View file @
47849a28
...
@@ -104,48 +104,37 @@ TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler_ValidateEncoding)) {
...
@@ -104,48 +104,37 @@ TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler_ValidateEncoding)) {
}
}
TEST_F
(
RapidJson
,
SIMD_SUFFIX
(
DocumentParseInsitu_MemoryPoolAllocator
))
{
TEST_F
(
RapidJson
,
SIMD_SUFFIX
(
DocumentParseInsitu_MemoryPoolAllocator
))
{
//const size_t userBufferSize = 128 * 1024;
//char* userBuffer = (char*)malloc(userBufferSize);
for
(
size_t
i
=
0
;
i
<
kTrialCount
;
i
++
)
{
for
(
size_t
i
=
0
;
i
<
kTrialCount
;
i
++
)
{
memcpy
(
temp_
,
json_
,
length_
+
1
);
memcpy
(
temp_
,
json_
,
length_
+
1
);
//MemoryPoolAllocator<> allocator(userBuffer, userBufferSize);
//Document doc(&allocator);
Document
doc
;
Document
doc
;
doc
.
ParseInsitu
(
temp_
);
doc
.
ParseInsitu
(
temp_
);
ASSERT_TRUE
(
doc
.
IsObject
());
ASSERT_TRUE
(
doc
.
IsObject
());
//if (i == 0) {
// size_t size = doc.GetAllocator().Size();
// size_t capacity = doc.GetAllocator().Capacity();
// size_t stack_capacity = doc.GetStackCapacity();
// size_t actual = size - stack_capacity;
// std::cout << "Size:" << size << " Capacity:" << capacity << " Stack:" << stack_capacity << " Actual:" << actual << std::endl;
//}
}
}
}
//free(userBuffer);
TEST_F
(
RapidJson
,
SIMD_SUFFIX
(
DocumentParseIterativeInsitu_MemoryPoolAllocator
))
{
for
(
size_t
i
=
0
;
i
<
kTrialCount
;
i
++
)
{
memcpy
(
temp_
,
json_
,
length_
+
1
);
Document
doc
;
doc
.
ParseInsitu
<
kParseIterativeFlag
>
(
temp_
);
ASSERT_TRUE
(
doc
.
IsObject
());
}
}
}
TEST_F
(
RapidJson
,
SIMD_SUFFIX
(
DocumentParse_MemoryPoolAllocator
))
{
TEST_F
(
RapidJson
,
SIMD_SUFFIX
(
DocumentParse_MemoryPoolAllocator
))
{
//const size_t userBufferSize = 128 * 1024;
//char* userBuffer = (char*)malloc(userBufferSize);
for
(
size_t
i
=
0
;
i
<
kTrialCount
;
i
++
)
{
for
(
size_t
i
=
0
;
i
<
kTrialCount
;
i
++
)
{
//MemoryPoolAllocator<> allocator(userBuffer, userBufferSize);
//Document doc(&allocator);
Document
doc
;
Document
doc
;
doc
.
Parse
(
json_
);
doc
.
Parse
(
json_
);
ASSERT_TRUE
(
doc
.
IsObject
());
ASSERT_TRUE
(
doc
.
IsObject
());
//if (i == 0) {
// size_t size = doc.GetAllocator().Size();
// size_t capacity = doc.GetAllocator().Capacity();
// size_t stack_capacity = doc.GetStackCapacity();
// size_t actual = size - stack_capacity;
// std::cout << "Size:" << size << " Capacity:" << capacity << " Stack:" << stack_capacity << " Actual:" << actual << std::endl;
//}
}
}
}
//free(userBuffer);
TEST_F
(
RapidJson
,
SIMD_SUFFIX
(
DocumentParseIterative_MemoryPoolAllocator
))
{
for
(
size_t
i
=
0
;
i
<
kTrialCount
;
i
++
)
{
Document
doc
;
doc
.
Parse
<
kParseIterativeFlag
>
(
json_
);
ASSERT_TRUE
(
doc
.
IsObject
());
}
}
}
TEST_F
(
RapidJson
,
SIMD_SUFFIX
(
DocumentParse_CrtAllocator
))
{
TEST_F
(
RapidJson
,
SIMD_SUFFIX
(
DocumentParse_CrtAllocator
))
{
...
...
test/unittest/readertest.cpp
View file @
47849a28
...
@@ -932,18 +932,6 @@ TEST(Reader, IterativeParsing_ShortCircuit) {
...
@@ -932,18 +932,6 @@ TEST(Reader, IterativeParsing_ShortCircuit) {
}
}
}
}
TEST
(
Reader
,
IterativeParsing_LimitStackSize
)
{
BaseReaderHandler
<>
handler
;
Reader
reader
(
20
);
StringStream
is
(
"[[[]]]"
);
ParseResult
r
=
reader
.
Parse
<
kParseIterativeFlag
>
(
is
,
handler
);
EXPECT_TRUE
(
reader
.
HasParseError
());
EXPECT_EQ
(
kParseErrorStackSizeLimitExceeded
,
r
.
Code
());
EXPECT_EQ
(
2u
,
r
.
Offset
());
}
#ifdef __GNUC__
#ifdef __GNUC__
RAPIDJSON_DIAG_POP
RAPIDJSON_DIAG_POP
#endif
#endif
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment