diff --git a/src/file_lists.cmake b/src/file_lists.cmake index 98fce372dd..1dbac5a9eb 100644 --- a/src/file_lists.cmake +++ b/src/file_lists.cmake @@ -358,6 +358,7 @@ set(libprotoc_srcs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/generator.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/helpers.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/import_writer.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/line_consumer.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/map_field.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/message.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/message_field.cc @@ -457,6 +458,7 @@ set(libprotoc_hdrs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/generator.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/helpers.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/import_writer.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/line_consumer.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/map_field.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/message.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/message_field.h @@ -684,8 +686,9 @@ set(compiler_test_files ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/doc_comment_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_serialization_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/plugin_unittest.cc - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/text_format_decode_data_unittest.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/line_consumer_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/names_unittest.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/text_format_decode_data_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/parser_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/python/plugin_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/ruby/ruby_generator_unittest.cc diff --git a/src/google/protobuf/compiler/objectivec/BUILD.bazel b/src/google/protobuf/compiler/objectivec/BUILD.bazel index de7eaeeca0..25fb412f24 100644 --- a/src/google/protobuf/compiler/objectivec/BUILD.bazel +++ b/src/google/protobuf/compiler/objectivec/BUILD.bazel @@ -29,6 +29,20 @@ cc_library( copts = COPTS, include_prefix = "google/protobuf/compiler/objectivec", visibility = ["//pkg:__pkg__"], + deps = [ + ":line_consumer", + "//src/google/protobuf/compiler:code_generator", + "//src/google/protobuf:protobuf_nowkt", + ], +) + +cc_library( + name = "line_consumer", + hdrs = ["line_consumer.h"], + srcs = ["line_consumer.cc"], + copts = COPTS, + include_prefix = "google/protobuf/compiler/objectivec", + visibility = ["//pkg:__pkg__"], deps = [ "//src/google/protobuf/compiler:code_generator", "//src/google/protobuf:protobuf_nowkt", @@ -77,6 +91,7 @@ cc_library( "//src/google/protobuf/compiler:__pkg__", ], deps = [ + ":line_consumer", ":names", "//src/google/protobuf:protobuf_nowkt", "//src/google/protobuf/compiler:code_generator", @@ -84,6 +99,17 @@ cc_library( ], ) +cc_test( + name = "line_consumer_unittest", + srcs = ["line_consumer_unittest.cc"], + deps = [ + ":line_consumer", + "//src/google/protobuf/io", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + cc_test( name = "names_unittest", srcs = ["names_unittest.cc"], diff --git a/src/google/protobuf/compiler/objectivec/import_writer.cc b/src/google/protobuf/compiler/objectivec/import_writer.cc index 01209bfb4c..e4c81978bf 100644 --- a/src/google/protobuf/compiler/objectivec/import_writer.cc +++ b/src/google/protobuf/compiler/objectivec/import_writer.cc @@ -29,6 +29,7 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "google/protobuf/compiler/objectivec/import_writer.h" +#include "google/protobuf/compiler/objectivec/line_consumer.h" #include "google/protobuf/compiler/objectivec/names.h" #include "google/protobuf/io/printer.h" diff --git a/src/google/protobuf/compiler/objectivec/line_consumer.cc b/src/google/protobuf/compiler/objectivec/line_consumer.cc new file mode 100644 index 0000000000..700a8fdc73 --- /dev/null +++ b/src/google/protobuf/compiler/objectivec/line_consumer.cc @@ -0,0 +1,224 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef _MSC_VER +#include +#endif +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" +#include "google/protobuf/compiler/objectivec/line_consumer.h" +#include "google/protobuf/io/io_win32.h" +#include "google/protobuf/io/zero_copy_stream_impl.h" + +// NOTE: src/google/protobuf/compiler/plugin.cc makes use of cerr for some +// error cases, so it seems to be ok to use as a back door for errors. + +namespace google { +namespace protobuf { +namespace compiler { +namespace objectivec { + +// is transitively included in this file. Import the functions explicitly +// in this port namespace to avoid ambiguous definition. +namespace posix { +#ifdef _WIN32 +using google::protobuf::io::win32::open; +#else // !_WIN32 +using ::open; +#endif // _WIN32 +} // namespace posix + +namespace { + +void TrimWhitespace(absl::string_view* input) { + while (!input->empty() && absl::ascii_isspace(*input->data())) { + input->remove_prefix(1); + } + while (!input->empty() && absl::ascii_isspace((*input)[input->length() - 1])) { + input->remove_suffix(1); + } +} + +bool ascii_isnewline(char c) { + return c == '\n' || c == '\r'; +} + +bool ReadLine(absl::string_view* input, absl::string_view* line) { + for (int len = 0; len < input->size(); ++len) { + if (ascii_isnewline((*input)[len])) { + *line = absl::string_view(input->data(), len); + ++len; // advance over the newline + *input = absl::string_view(input->data() + len, input->size() - len); + return true; + } + } + return false; // Ran out of input with no newline. +} + +void RemoveComment(absl::string_view* input) { + int offset = input->find('#'); + if (offset != absl::string_view::npos) { + input->remove_suffix(input->length() - offset); + } +} + +class Parser { + public: + Parser(LineConsumer* line_consumer) + : line_consumer_(line_consumer), line_(0) {} + + // Feeds in some input, parse what it can, returning success/failure. Calling + // again after an error is undefined. + bool ParseChunk(absl::string_view chunk, std::string* out_error); + + // Should be called to finish parsing (after all input has been provided via + // successful calls to ParseChunk(), calling after a ParseChunk() failure is + // undefined). Returns success/failure. + bool Finish(std::string* out_error); + + int last_line() const { return line_; } + + private: + LineConsumer* line_consumer_; + int line_; + std::string leftover_; +}; + +bool Parser::ParseChunk(absl::string_view chunk, std::string* out_error) { + absl::string_view full_chunk; + if (!leftover_.empty()) { + leftover_ += std::string(chunk); + full_chunk = absl::string_view(leftover_); + } else { + full_chunk = chunk; + } + + absl::string_view line; + while (ReadLine(&full_chunk, &line)) { + ++line_; + RemoveComment(&line); + TrimWhitespace(&line); + if (!line.empty() && !line_consumer_->ConsumeLine(line, out_error)) { + if (out_error->empty()) { + *out_error = "ConsumeLine failed without setting an error."; + } + leftover_.clear(); + return false; + } + } + + if (full_chunk.empty()) { + leftover_.clear(); + } else { + leftover_ = std::string(full_chunk); + } + return true; +} + +bool Parser::Finish(std::string* out_error) { + // If there is still something to go, flush it with a newline. + if (!leftover_.empty() && !ParseChunk("\n", out_error)) { + return false; + } + // This really should never fail if ParseChunk succeeded, but check to be sure. + if (!leftover_.empty()) { + *out_error = "ParseSimple Internal error: finished with pending data."; + return false; + } + return true; +} + +std::string FullErrorString(const std::string& name, int line_num, const std::string& msg) { + return std::string("error: ") + name + " Line " + absl::StrCat(line_num) + ", " + msg; +} + +} // namespace + +LineConsumer::LineConsumer() {} + +LineConsumer::~LineConsumer() {} + +bool ParseSimpleFile(const std::string& path, LineConsumer* line_consumer, + std::string* out_error) { + int fd; + do { + fd = posix::open(path.c_str(), O_RDONLY); + } while (fd < 0 && errno == EINTR); + if (fd < 0) { + *out_error = std::string("error: Unable to open \"") + path + "\", " + + strerror(errno); + return false; + } + io::FileInputStream file_stream(fd); + file_stream.SetCloseOnDelete(true); + + return ParseSimpleStream(file_stream, path, line_consumer, out_error); +} + +bool ParseSimpleStream(io::ZeroCopyInputStream& input_stream, + const std::string& stream_name, + LineConsumer* line_consumer, + std::string* out_error) { + std::string local_error; + Parser parser(line_consumer); + const void* buf; + int buf_len; + while (input_stream.Next(&buf, &buf_len)) { + if (buf_len == 0) { + continue; + } + + if (!parser.ParseChunk(absl::string_view(static_cast(buf), buf_len), + &local_error)) { + *out_error = FullErrorString(stream_name, parser.last_line(), local_error); + return false; + } + } + if (!parser.Finish(&local_error)) { + *out_error = FullErrorString(stream_name, parser.last_line(), local_error); + return false; + } + return true; +} + +} // namespace objectivec +} // namespace compiler +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/compiler/objectivec/line_consumer.h b/src/google/protobuf/compiler/objectivec/line_consumer.h new file mode 100644 index 0000000000..41fdb5b383 --- /dev/null +++ b/src/google/protobuf/compiler/objectivec/line_consumer.h @@ -0,0 +1,66 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_COMPILER_OBJECTIVEC_LINE_CONSUMER_H__ +#define GOOGLE_PROTOBUF_COMPILER_OBJECTIVEC_LINE_CONSUMER_H__ + +#include +#include + +#include "google/protobuf/io/zero_copy_stream.h" + +namespace google { +namespace protobuf { +namespace compiler { +namespace objectivec { + +// Helper for parsing simple files. +class LineConsumer { + public: + LineConsumer(); + virtual ~LineConsumer(); + virtual bool ConsumeLine(const absl::string_view& line, std::string* out_error) = 0; +}; + +bool ParseSimpleFile(const std::string& path, + LineConsumer* line_consumer, + std::string* out_error); + +bool ParseSimpleStream(io::ZeroCopyInputStream& input_stream, + const std::string& stream_name, + LineConsumer* line_consumer, + std::string* out_error); + +} // namespace objectivec +} // namespace compiler +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_COMPILER_OBJECTIVEC_LINE_CONSUMER_H__ diff --git a/src/google/protobuf/compiler/objectivec/line_consumer_unittest.cc b/src/google/protobuf/compiler/objectivec/line_consumer_unittest.cc new file mode 100644 index 0000000000..2513b6f4fe --- /dev/null +++ b/src/google/protobuf/compiler/objectivec/line_consumer_unittest.cc @@ -0,0 +1,174 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2014 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "google/protobuf/compiler/objectivec/line_consumer.h" +#include "google/protobuf/io/zero_copy_stream_impl_lite.h" +#include "absl/strings/str_cat.h" + +#include + +namespace google { +namespace protobuf { +namespace compiler { +namespace objectivec { +namespace { + +class TestLineCollector : public LineConsumer { + public: + explicit TestLineCollector(std::vector* inout_lines, + const std::string* reject_line = nullptr, + bool skip_msg = false) + : lines_(inout_lines), reject_(reject_line), skip_msg_(skip_msg) {} + + bool ConsumeLine(const absl::string_view& line, std::string* out_error) override { + if (reject_ && *reject_ == line) { + if (!skip_msg_) { + *out_error = std::string("Rejected '") + *reject_ + "'"; + } + return false; + } + if (lines_) { + lines_->emplace_back(line); + } + return true; + } + + private: + std::vector* lines_; + const std::string* reject_; + bool skip_msg_; +}; + +const int kBlockSizes[] = {-1, 1, 2, 5, 64}; +const int kBlockSizeCount = ABSL_ARRAYSIZE(kBlockSizes); + +TEST(ObjCHelper, ParseSimple_BasicsSuccess) { + const std::vector>> tests = { + {"", {}}, + {"a", {"a"}}, + {"a c", {"a c"}}, + {" a c ", {"a c"}}, + {"\ta c ", {"a c"}}, + {"abc\n", {"abc"}}, + {"abc\nd f", {"abc", "d f"}}, + {"\n abc \n def \n\n", {"abc", "def"}}, + }; + + for (const auto& test : tests) { + for (int i = 0; i < kBlockSizeCount; i++) { + io::ArrayInputStream input(test.first.data(), test.first.size(), kBlockSizes[i]); + std::string err_str; + std::vector lines; + TestLineCollector collector(&lines); + EXPECT_TRUE(ParseSimpleStream(input, "dummy", &collector, &err_str)); + EXPECT_EQ(lines, test.second); + EXPECT_TRUE(err_str.empty()); + } + } +} + +TEST(ObjCHelper, ParseSimple_DropsComments) { + const std::vector>> tests = { + {"# nothing", {}}, + {"#", {}}, + {"##", {}}, + {"\n# nothing\n", {}}, + {"a # same line", {"a"}}, + {"a # same line\n", {"a"}}, + {"a\n# line\nc", {"a", "c"}}, + {"# n o t # h i n g #", {}}, + {"## n o # t h i n g #", {}}, + {"a# n o t # h i n g #", {"a"}}, + {"a\n## n o # t h i n g #", {"a"}}, + }; + + for (const auto& test : tests) { + for (int i = 0; i < kBlockSizeCount; i++) { + io::ArrayInputStream input(test.first.data(), test.first.size(), kBlockSizes[i]); + std::string err_str; + std::vector lines; + TestLineCollector collector(&lines); + EXPECT_TRUE(ParseSimpleStream(input, "dummy", &collector, &err_str)); + EXPECT_EQ(lines, test.second); + EXPECT_TRUE(err_str.empty()); + } + } +} + +TEST(ObjCHelper, ParseSimple_RejectLines) { + const std::vector> tests = { + std::make_tuple("a\nb\nc", "a", 1), + std::make_tuple("a\nb\nc", "b", 2), + std::make_tuple("a\nb\nc", "c", 3), + std::make_tuple("a\nb\nc\n", "c", 3), + }; + + for (const auto& test : tests) { + for (int i = 0; i < kBlockSizeCount; i++) { + io::ArrayInputStream input(std::get<0>(test).data(), std::get<0>(test).size(), + kBlockSizes[i]); + std::string err_str; + TestLineCollector collector(nullptr, &std::get<1>(test)); + EXPECT_FALSE(ParseSimpleStream(input, "dummy", &collector, &err_str)); + std::string expected_err = + absl::StrCat("error: dummy Line ", std::get<2>(test), ", Rejected '", std::get<1>(test), "'"); + EXPECT_EQ(err_str, expected_err); + } + } +} + +TEST(ObjCHelper, ParseSimple_RejectLinesNoMessage) { + const std::vector> tests = { + std::make_tuple("a\nb\nc", "a", 1), + std::make_tuple("a\nb\nc", "b", 2), + std::make_tuple("a\nb\nc", "c", 3), + std::make_tuple("a\nb\nc\n", "c", 3), + }; + + for (const auto& test : tests) { + for (int i = 0; i < kBlockSizeCount; i++) { + io::ArrayInputStream input(std::get<0>(test).data(), std::get<0>(test).size(), + kBlockSizes[i]); + std::string err_str; + TestLineCollector collector(nullptr, &std::get<1>(test), true /* skip msg */); + EXPECT_FALSE(ParseSimpleStream(input, "dummy", &collector, &err_str)); + std::string expected_err = + absl::StrCat("error: dummy Line ", std::get<2>(test), + ", ConsumeLine failed without setting an error."); + EXPECT_EQ(err_str, expected_err); + } + } +} + +} // namespace +} // namespace objectivec +} // namespace compiler +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/compiler/objectivec/names.cc b/src/google/protobuf/compiler/objectivec/names.cc index 7ccb713caa..172ed69a27 100644 --- a/src/google/protobuf/compiler/objectivec/names.cc +++ b/src/google/protobuf/compiler/objectivec/names.cc @@ -28,13 +28,6 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#ifndef _MSC_VER -#include -#endif -#include -#include -#include - #include #include #include @@ -43,12 +36,11 @@ #include #include "google/protobuf/compiler/code_generator.h" -#include "absl/strings/str_replace.h" #include "absl/strings/str_split.h" +#include "google/protobuf/compiler/objectivec/line_consumer.h" #include "google/protobuf/compiler/objectivec/names.h" #include "google/protobuf/compiler/objectivec/nsobject_methods.h" #include "google/protobuf/descriptor.pb.h" -#include "google/protobuf/io/io_win32.h" #include "google/protobuf/io/zero_copy_stream_impl.h" // NOTE: src/google/protobuf/compiler/plugin.cc makes use of cerr for some @@ -59,16 +51,6 @@ namespace protobuf { namespace compiler { namespace objectivec { -// is transitively included in this file. Import the functions explicitly -// in this port namespace to avoid ambiguous definition. -namespace posix { -#ifdef _WIN32 -using google::protobuf::io::win32::open; -#else // !_WIN32 -using ::open; -#endif // _WIN32 -} // namespace posix - namespace { bool BoolFromEnvVar(const char* env_var, bool default_value) { @@ -275,10 +257,6 @@ const char* const kUpperSegmentsList[] = {"url", "http", "https"}; std::unordered_set kUpperSegments = MakeWordsMap(kUpperSegmentsList, ABSL_ARRAYSIZE(kUpperSegmentsList)); -bool ascii_isnewline(char c) { - return c == '\n' || c == '\r'; -} - // Internal helper for name handing. // Do not expose this outside of helpers, stick to having functions for specific // cases (ClassName(), FieldName()), so there is always consistent suffix rules. @@ -878,25 +856,6 @@ bool IsProtobufLibraryBundledProtoFile(const FileDescriptor* file) { return false; } -bool ReadLine(absl::string_view* input, absl::string_view* line) { - for (int len = 0; len < input->size(); ++len) { - if (ascii_isnewline((*input)[len])) { - *line = absl::string_view(input->data(), len); - ++len; // advance over the newline - *input = absl::string_view(input->data() + len, input->size() - len); - return true; - } - } - return false; // Ran out of input with no newline. -} - -void RemoveComment(absl::string_view* input) { - int offset = input->find('#'); - if (offset != absl::string_view::npos) { - input->remove_suffix(input->length() - offset); - } -} - namespace { bool PackageToPrefixesCollector::ConsumeLine( @@ -1252,127 +1211,6 @@ bool DecodeDataBuilder::AddCharacter(const char desired, const char input) { } // namespace -namespace { - -class Parser { - public: - Parser(LineConsumer* line_consumer) - : line_consumer_(line_consumer), line_(0) {} - - // Feeds in some input, parse what it can, returning success/failure. Calling - // again after an error is undefined. - bool ParseChunk(absl::string_view chunk, std::string* out_error); - - // Should be called to finish parsing (after all input has been provided via - // successful calls to ParseChunk(), calling after a ParseChunk() failure is - // undefined). Returns success/failure. - bool Finish(std::string* out_error); - - int last_line() const { return line_; } - - private: - LineConsumer* line_consumer_; - int line_; - std::string leftover_; -}; - -bool Parser::ParseChunk(absl::string_view chunk, std::string* out_error) { - absl::string_view full_chunk; - if (!leftover_.empty()) { - leftover_ += std::string(chunk); - full_chunk = absl::string_view(leftover_); - } else { - full_chunk = chunk; - } - - absl::string_view line; - while (ReadLine(&full_chunk, &line)) { - ++line_; - RemoveComment(&line); - TrimWhitespace(&line); - if (!line.empty() && !line_consumer_->ConsumeLine(line, out_error)) { - if (out_error->empty()) { - *out_error = "ConsumeLine failed without setting an error."; - } - leftover_.clear(); - return false; - } - } - - if (full_chunk.empty()) { - leftover_.clear(); - } else { - leftover_ = std::string(full_chunk); - } - return true; -} - -bool Parser::Finish(std::string* out_error) { - // If there is still something to go, flush it with a newline. - if (!leftover_.empty() && !ParseChunk("\n", out_error)) { - return false; - } - // This really should never fail if ParseChunk succeeded, but check to be sure. - if (!leftover_.empty()) { - *out_error = "ParseSimple Internal error: finished with pending data."; - return false; - } - return true; -} - -std::string FullErrorString(const std::string& name, int line_num, const std::string& msg) { - return std::string("error: ") + name + " Line " + absl::StrCat(line_num) + ", " + msg; -} - -} // namespace - -LineConsumer::LineConsumer() {} - -LineConsumer::~LineConsumer() {} - -bool ParseSimpleFile(const std::string& path, LineConsumer* line_consumer, - std::string* out_error) { - int fd; - do { - fd = posix::open(path.c_str(), O_RDONLY); - } while (fd < 0 && errno == EINTR); - if (fd < 0) { - *out_error = std::string("error: Unable to open \"") + path + "\", " + - strerror(errno); - return false; - } - io::FileInputStream file_stream(fd); - file_stream.SetCloseOnDelete(true); - - return ParseSimpleStream(file_stream, path, line_consumer, out_error); -} - -bool ParseSimpleStream(io::ZeroCopyInputStream& input_stream, - const std::string& stream_name, - LineConsumer* line_consumer, - std::string* out_error) { - std::string local_error; - Parser parser(line_consumer); - const void* buf; - int buf_len; - while (input_stream.Next(&buf, &buf_len)) { - if (buf_len == 0) { - continue; - } - - if (!parser.ParseChunk(absl::string_view(static_cast(buf), buf_len), - &local_error)) { - *out_error = FullErrorString(stream_name, parser.last_line(), local_error); - return false; - } - } - if (!parser.Finish(&local_error)) { - *out_error = FullErrorString(stream_name, parser.last_line(), local_error); - return false; - } - return true; -} - } // namespace objectivec } // namespace compiler } // namespace protobuf diff --git a/src/google/protobuf/compiler/objectivec/names.h b/src/google/protobuf/compiler/objectivec/names.h index 103cf1de52..8b17b4dd65 100644 --- a/src/google/protobuf/compiler/objectivec/names.h +++ b/src/google/protobuf/compiler/objectivec/names.h @@ -172,23 +172,6 @@ bool PROTOC_EXPORT ValidateObjCClassPrefixes( bool PROTOC_EXPORT ValidateObjCClassPrefixes( const std::vector& files, std::string* out_error); -// Helper for parsing simple files. -class PROTOC_EXPORT LineConsumer { - public: - LineConsumer(); - virtual ~LineConsumer(); - virtual bool ConsumeLine(const absl::string_view& line, std::string* out_error) = 0; -}; - -bool PROTOC_EXPORT ParseSimpleFile(const std::string& path, - LineConsumer* line_consumer, - std::string* out_error); - -bool PROTOC_EXPORT ParseSimpleStream(io::ZeroCopyInputStream& input_stream, - const std::string& stream_name, - LineConsumer* line_consumer, - std::string* out_error); - } // namespace objectivec } // namespace compiler } // namespace protobuf diff --git a/src/google/protobuf/compiler/objectivec/names_unittest.cc b/src/google/protobuf/compiler/objectivec/names_unittest.cc index f472a76448..0aac79497c 100644 --- a/src/google/protobuf/compiler/objectivec/names_unittest.cc +++ b/src/google/protobuf/compiler/objectivec/names_unittest.cc @@ -39,133 +39,6 @@ namespace compiler { namespace objectivec { namespace { -class TestLineCollector : public LineConsumer { - public: - explicit TestLineCollector(std::vector* inout_lines, - const std::string* reject_line = nullptr, - bool skip_msg = false) - : lines_(inout_lines), reject_(reject_line), skip_msg_(skip_msg) {} - - bool ConsumeLine(const absl::string_view& line, std::string* out_error) override { - if (reject_ && *reject_ == line) { - if (!skip_msg_) { - *out_error = std::string("Rejected '") + *reject_ + "'"; - } - return false; - } - if (lines_) { - lines_->emplace_back(line); - } - return true; - } - - private: - std::vector* lines_; - const std::string* reject_; - bool skip_msg_; -}; - -const int kBlockSizes[] = {-1, 1, 2, 5, 64}; -const int kBlockSizeCount = ABSL_ARRAYSIZE(kBlockSizes); - -TEST(ObjCHelper, ParseSimple_BasicsSuccess) { - const std::vector>> tests = { - {"", {}}, - {"a", {"a"}}, - {"a c", {"a c"}}, - {" a c ", {"a c"}}, - {"\ta c ", {"a c"}}, - {"abc\n", {"abc"}}, - {"abc\nd f", {"abc", "d f"}}, - {"\n abc \n def \n\n", {"abc", "def"}}, - }; - - for (const auto& test : tests) { - for (int i = 0; i < kBlockSizeCount; i++) { - io::ArrayInputStream input(test.first.data(), test.first.size(), kBlockSizes[i]); - std::string err_str; - std::vector lines; - TestLineCollector collector(&lines); - EXPECT_TRUE(ParseSimpleStream(input, "dummy", &collector, &err_str)); - EXPECT_EQ(lines, test.second); - EXPECT_TRUE(err_str.empty()); - } - } -} - -TEST(ObjCHelper, ParseSimple_DropsComments) { - const std::vector>> tests = { - {"# nothing", {}}, - {"#", {}}, - {"##", {}}, - {"\n# nothing\n", {}}, - {"a # same line", {"a"}}, - {"a # same line\n", {"a"}}, - {"a\n# line\nc", {"a", "c"}}, - {"# n o t # h i n g #", {}}, - {"## n o # t h i n g #", {}}, - {"a# n o t # h i n g #", {"a"}}, - {"a\n## n o # t h i n g #", {"a"}}, - }; - - for (const auto& test : tests) { - for (int i = 0; i < kBlockSizeCount; i++) { - io::ArrayInputStream input(test.first.data(), test.first.size(), kBlockSizes[i]); - std::string err_str; - std::vector lines; - TestLineCollector collector(&lines); - EXPECT_TRUE(ParseSimpleStream(input, "dummy", &collector, &err_str)); - EXPECT_EQ(lines, test.second); - EXPECT_TRUE(err_str.empty()); - } - } -} - -TEST(ObjCHelper, ParseSimple_RejectLines) { - const std::vector> tests = { - std::make_tuple("a\nb\nc", "a", 1), - std::make_tuple("a\nb\nc", "b", 2), - std::make_tuple("a\nb\nc", "c", 3), - std::make_tuple("a\nb\nc\n", "c", 3), - }; - - for (const auto& test : tests) { - for (int i = 0; i < kBlockSizeCount; i++) { - io::ArrayInputStream input(std::get<0>(test).data(), std::get<0>(test).size(), - kBlockSizes[i]); - std::string err_str; - TestLineCollector collector(nullptr, &std::get<1>(test)); - EXPECT_FALSE(ParseSimpleStream(input, "dummy", &collector, &err_str)); - std::string expected_err = - absl::StrCat("error: dummy Line ", std::get<2>(test), ", Rejected '", std::get<1>(test), "'"); - EXPECT_EQ(err_str, expected_err); - } - } -} - -TEST(ObjCHelper, ParseSimple_RejectLinesNoMessage) { - const std::vector> tests = { - std::make_tuple("a\nb\nc", "a", 1), - std::make_tuple("a\nb\nc", "b", 2), - std::make_tuple("a\nb\nc", "c", 3), - std::make_tuple("a\nb\nc\n", "c", 3), - }; - - for (const auto& test : tests) { - for (int i = 0; i < kBlockSizeCount; i++) { - io::ArrayInputStream input(std::get<0>(test).data(), std::get<0>(test).size(), - kBlockSizes[i]); - std::string err_str; - TestLineCollector collector(nullptr, &std::get<1>(test), true /* skip msg */); - EXPECT_FALSE(ParseSimpleStream(input, "dummy", &collector, &err_str)); - std::string expected_err = - absl::StrCat("error: dummy Line ", std::get<2>(test), - ", ConsumeLine failed without setting an error."); - EXPECT_EQ(err_str, expected_err); - } - } -} - TEST(ObjCHelper, IsRetainedName) { EXPECT_TRUE(IsRetainedName("new")); EXPECT_TRUE(IsRetainedName("alloc"));