Commit 6f12e3e2 authored by kenton@google.com's avatar kenton@google.com

Accept form feeds as whitespace.

parent f9c5978f
......@@ -117,7 +117,7 @@ namespace {
}
CHARACTER_CLASS(Whitespace, c == ' ' || c == '\n' || c == '\t' ||
c == '\r' || c == '\v');
c == '\r' || c == '\v' || c == '\f');
CHARACTER_CLASS(Unprintable, c < ' ' && c > '\0');
......
......@@ -403,6 +403,13 @@ MultiTokenCase kMultiTokenCases[] = {
{ Tokenizer::TYPE_SYMBOL, "\300", 0, 0 },
{ Tokenizer::TYPE_END , "" , 0, 1 },
}},
// Test all whitespace chars
{ "foo\n\t\r\v\fbar", {
{ Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0 },
{ Tokenizer::TYPE_IDENTIFIER, "bar", 1, 11 },
{ Tokenizer::TYPE_END , "" , 1, 14 },
}},
};
TEST_2D(TokenizerTest, MultipleTokens, kMultiTokenCases, kBlockSizes) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment