Commit 6f12e3e2 authored by kenton@google.com's avatar kenton@google.com

Accept form feeds as whitespace.

parent f9c5978f
...@@ -117,7 +117,7 @@ namespace { ...@@ -117,7 +117,7 @@ namespace {
} }
CHARACTER_CLASS(Whitespace, c == ' ' || c == '\n' || c == '\t' || CHARACTER_CLASS(Whitespace, c == ' ' || c == '\n' || c == '\t' ||
c == '\r' || c == '\v'); c == '\r' || c == '\v' || c == '\f');
CHARACTER_CLASS(Unprintable, c < ' ' && c > '\0'); CHARACTER_CLASS(Unprintable, c < ' ' && c > '\0');
......
...@@ -403,6 +403,13 @@ MultiTokenCase kMultiTokenCases[] = { ...@@ -403,6 +403,13 @@ MultiTokenCase kMultiTokenCases[] = {
{ Tokenizer::TYPE_SYMBOL, "\300", 0, 0 }, { Tokenizer::TYPE_SYMBOL, "\300", 0, 0 },
{ Tokenizer::TYPE_END , "" , 0, 1 }, { Tokenizer::TYPE_END , "" , 0, 1 },
}}, }},
// Test all whitespace chars
{ "foo\n\t\r\v\fbar", {
{ Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0 },
{ Tokenizer::TYPE_IDENTIFIER, "bar", 1, 11 },
{ Tokenizer::TYPE_END , "" , 1, 14 },
}},
}; };
TEST_2D(TokenizerTest, MultipleTokens, kMultiTokenCases, kBlockSizes) { TEST_2D(TokenizerTest, MultipleTokens, kMultiTokenCases, kBlockSizes) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment