From 6f12e3e29a2f75af03c27723810e5a94b56a8642 Mon Sep 17 00:00:00 2001 From: "kenton@google.com" Date: Tue, 22 Dec 2009 18:11:09 +0000 Subject: [PATCH] Accept form feeds as whitespace. --- src/google/protobuf/io/tokenizer.cc | 2 +- src/google/protobuf/io/tokenizer_unittest.cc | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/google/protobuf/io/tokenizer.cc b/src/google/protobuf/io/tokenizer.cc index 75cbfed5f9..38fa351c41 100644 --- a/src/google/protobuf/io/tokenizer.cc +++ b/src/google/protobuf/io/tokenizer.cc @@ -117,7 +117,7 @@ namespace { } CHARACTER_CLASS(Whitespace, c == ' ' || c == '\n' || c == '\t' || - c == '\r' || c == '\v'); + c == '\r' || c == '\v' || c == '\f'); CHARACTER_CLASS(Unprintable, c < ' ' && c > '\0'); diff --git a/src/google/protobuf/io/tokenizer_unittest.cc b/src/google/protobuf/io/tokenizer_unittest.cc index 3598e1885a..358ec567e5 100644 --- a/src/google/protobuf/io/tokenizer_unittest.cc +++ b/src/google/protobuf/io/tokenizer_unittest.cc @@ -403,6 +403,13 @@ MultiTokenCase kMultiTokenCases[] = { { Tokenizer::TYPE_SYMBOL, "\300", 0, 0 }, { Tokenizer::TYPE_END , "" , 0, 1 }, }}, + + // Test all whitespace chars + { "foo\n\t\r\v\fbar", { + { Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0 }, + { Tokenizer::TYPE_IDENTIFIER, "bar", 1, 11 }, + { Tokenizer::TYPE_END , "" , 1, 14 }, + }}, }; TEST_2D(TokenizerTest, MultipleTokens, kMultiTokenCases, kBlockSizes) {