From bb2bba124c5f90120df0432b22f694372a1147e0 Mon Sep 17 00:00:00 2001 From: Thomas Van Lenten Date: Tue, 4 Oct 2022 14:53:21 -0400 Subject: [PATCH] [ObjC] Move LineConsumer to its own file. Make a new (private to the package) target for this so names can depend on it as an implementation detail as well as the other generator code. Split the names unittests to have the LineConsumer out on its own also. --- src/file_lists.cmake | 5 +- .../protobuf/compiler/objectivec/BUILD.bazel | 26 ++ .../compiler/objectivec/import_writer.cc | 1 + .../compiler/objectivec/line_consumer.cc | 224 ++++++++++++++++++ .../compiler/objectivec/line_consumer.h | 66 ++++++ .../objectivec/line_consumer_unittest.cc | 174 ++++++++++++++ .../protobuf/compiler/objectivec/names.cc | 164 +------------ .../protobuf/compiler/objectivec/names.h | 17 -- .../compiler/objectivec/names_unittest.cc | 127 ---------- 9 files changed, 496 insertions(+), 308 deletions(-) create mode 100644 src/google/protobuf/compiler/objectivec/line_consumer.cc create mode 100644 src/google/protobuf/compiler/objectivec/line_consumer.h create mode 100644 src/google/protobuf/compiler/objectivec/line_consumer_unittest.cc diff --git a/src/file_lists.cmake b/src/file_lists.cmake index 98fce372dd..1dbac5a9eb 100644 --- a/src/file_lists.cmake +++ b/src/file_lists.cmake @@ -358,6 +358,7 @@ set(libprotoc_srcs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/generator.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/helpers.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/import_writer.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/line_consumer.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/map_field.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/message.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/message_field.cc @@ -457,6 +458,7 @@ set(libprotoc_hdrs ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/generator.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/helpers.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/import_writer.h + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/line_consumer.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/map_field.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/message.h ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/message_field.h @@ -684,8 +686,9 @@ set(compiler_test_files ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/doc_comment_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/message_serialization_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/java/plugin_unittest.cc - ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/text_format_decode_data_unittest.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/line_consumer_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/names_unittest.cc + ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/objectivec/text_format_decode_data_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/parser_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/python/plugin_unittest.cc ${protobuf_SOURCE_DIR}/src/google/protobuf/compiler/ruby/ruby_generator_unittest.cc diff --git a/src/google/protobuf/compiler/objectivec/BUILD.bazel b/src/google/protobuf/compiler/objectivec/BUILD.bazel index de7eaeeca0..25fb412f24 100644 --- a/src/google/protobuf/compiler/objectivec/BUILD.bazel +++ b/src/google/protobuf/compiler/objectivec/BUILD.bazel @@ -29,6 +29,20 @@ cc_library( copts = COPTS, include_prefix = "google/protobuf/compiler/objectivec", visibility = ["//pkg:__pkg__"], + deps = [ + ":line_consumer", + "//src/google/protobuf/compiler:code_generator", + "//src/google/protobuf:protobuf_nowkt", + ], +) + +cc_library( + name = "line_consumer", + hdrs = ["line_consumer.h"], + srcs = ["line_consumer.cc"], + copts = COPTS, + include_prefix = "google/protobuf/compiler/objectivec", + visibility = ["//pkg:__pkg__"], deps = [ "//src/google/protobuf/compiler:code_generator", "//src/google/protobuf:protobuf_nowkt", @@ -77,6 +91,7 @@ cc_library( "//src/google/protobuf/compiler:__pkg__", ], deps = [ + ":line_consumer", ":names", "//src/google/protobuf:protobuf_nowkt", "//src/google/protobuf/compiler:code_generator", @@ -84,6 +99,17 @@ cc_library( ], ) +cc_test( + name = "line_consumer_unittest", + srcs = ["line_consumer_unittest.cc"], + deps = [ + ":line_consumer", + "//src/google/protobuf/io", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + cc_test( name = "names_unittest", srcs = ["names_unittest.cc"], diff --git a/src/google/protobuf/compiler/objectivec/import_writer.cc b/src/google/protobuf/compiler/objectivec/import_writer.cc index 01209bfb4c..e4c81978bf 100644 --- a/src/google/protobuf/compiler/objectivec/import_writer.cc +++ b/src/google/protobuf/compiler/objectivec/import_writer.cc @@ -29,6 +29,7 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "google/protobuf/compiler/objectivec/import_writer.h" +#include "google/protobuf/compiler/objectivec/line_consumer.h" #include "google/protobuf/compiler/objectivec/names.h" #include "google/protobuf/io/printer.h" diff --git a/src/google/protobuf/compiler/objectivec/line_consumer.cc b/src/google/protobuf/compiler/objectivec/line_consumer.cc new file mode 100644 index 0000000000..700a8fdc73 --- /dev/null +++ b/src/google/protobuf/compiler/objectivec/line_consumer.cc @@ -0,0 +1,224 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef _MSC_VER +#include +#endif +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" +#include "google/protobuf/compiler/objectivec/line_consumer.h" +#include "google/protobuf/io/io_win32.h" +#include "google/protobuf/io/zero_copy_stream_impl.h" + +// NOTE: src/google/protobuf/compiler/plugin.cc makes use of cerr for some +// error cases, so it seems to be ok to use as a back door for errors. + +namespace google { +namespace protobuf { +namespace compiler { +namespace objectivec { + +// is transitively included in this file. Import the functions explicitly +// in this port namespace to avoid ambiguous definition. +namespace posix { +#ifdef _WIN32 +using google::protobuf::io::win32::open; +#else // !_WIN32 +using ::open; +#endif // _WIN32 +} // namespace posix + +namespace { + +void TrimWhitespace(absl::string_view* input) { + while (!input->empty() && absl::ascii_isspace(*input->data())) { + input->remove_prefix(1); + } + while (!input->empty() && absl::ascii_isspace((*input)[input->length() - 1])) { + input->remove_suffix(1); + } +} + +bool ascii_isnewline(char c) { + return c == '\n' || c == '\r'; +} + +bool ReadLine(absl::string_view* input, absl::string_view* line) { + for (int len = 0; len < input->size(); ++len) { + if (ascii_isnewline((*input)[len])) { + *line = absl::string_view(input->data(), len); + ++len; // advance over the newline + *input = absl::string_view(input->data() + len, input->size() - len); + return true; + } + } + return false; // Ran out of input with no newline. +} + +void RemoveComment(absl::string_view* input) { + int offset = input->find('#'); + if (offset != absl::string_view::npos) { + input->remove_suffix(input->length() - offset); + } +} + +class Parser { + public: + Parser(LineConsumer* line_consumer) + : line_consumer_(line_consumer), line_(0) {} + + // Feeds in some input, parse what it can, returning success/failure. Calling + // again after an error is undefined. + bool ParseChunk(absl::string_view chunk, std::string* out_error); + + // Should be called to finish parsing (after all input has been provided via + // successful calls to ParseChunk(), calling after a ParseChunk() failure is + // undefined). Returns success/failure. + bool Finish(std::string* out_error); + + int last_line() const { return line_; } + + private: + LineConsumer* line_consumer_; + int line_; + std::string leftover_; +}; + +bool Parser::ParseChunk(absl::string_view chunk, std::string* out_error) { + absl::string_view full_chunk; + if (!leftover_.empty()) { + leftover_ += std::string(chunk); + full_chunk = absl::string_view(leftover_); + } else { + full_chunk = chunk; + } + + absl::string_view line; + while (ReadLine(&full_chunk, &line)) { + ++line_; + RemoveComment(&line); + TrimWhitespace(&line); + if (!line.empty() && !line_consumer_->ConsumeLine(line, out_error)) { + if (out_error->empty()) { + *out_error = "ConsumeLine failed without setting an error."; + } + leftover_.clear(); + return false; + } + } + + if (full_chunk.empty()) { + leftover_.clear(); + } else { + leftover_ = std::string(full_chunk); + } + return true; +} + +bool Parser::Finish(std::string* out_error) { + // If there is still something to go, flush it with a newline. + if (!leftover_.empty() && !ParseChunk("\n", out_error)) { + return false; + } + // This really should never fail if ParseChunk succeeded, but check to be sure. + if (!leftover_.empty()) { + *out_error = "ParseSimple Internal error: finished with pending data."; + return false; + } + return true; +} + +std::string FullErrorString(const std::string& name, int line_num, const std::string& msg) { + return std::string("error: ") + name + " Line " + absl::StrCat(line_num) + ", " + msg; +} + +} // namespace + +LineConsumer::LineConsumer() {} + +LineConsumer::~LineConsumer() {} + +bool ParseSimpleFile(const std::string& path, LineConsumer* line_consumer, + std::string* out_error) { + int fd; + do { + fd = posix::open(path.c_str(), O_RDONLY); + } while (fd < 0 && errno == EINTR); + if (fd < 0) { + *out_error = std::string("error: Unable to open \"") + path + "\", " + + strerror(errno); + return false; + } + io::FileInputStream file_stream(fd); + file_stream.SetCloseOnDelete(true); + + return ParseSimpleStream(file_stream, path, line_consumer, out_error); +} + +bool ParseSimpleStream(io::ZeroCopyInputStream& input_stream, + const std::string& stream_name, + LineConsumer* line_consumer, + std::string* out_error) { + std::string local_error; + Parser parser(line_consumer); + const void* buf; + int buf_len; + while (input_stream.Next(&buf, &buf_len)) { + if (buf_len == 0) { + continue; + } + + if (!parser.ParseChunk(absl::string_view(static_cast(buf), buf_len), + &local_error)) { + *out_error = FullErrorString(stream_name, parser.last_line(), local_error); + return false; + } + } + if (!parser.Finish(&local_error)) { + *out_error = FullErrorString(stream_name, parser.last_line(), local_error); + return false; + } + return true; +} + +} // namespace objectivec +} // namespace compiler +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/compiler/objectivec/line_consumer.h b/src/google/protobuf/compiler/objectivec/line_consumer.h new file mode 100644 index 0000000000..41fdb5b383 --- /dev/null +++ b/src/google/protobuf/compiler/objectivec/line_consumer.h @@ -0,0 +1,66 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_COMPILER_OBJECTIVEC_LINE_CONSUMER_H__ +#define GOOGLE_PROTOBUF_COMPILER_OBJECTIVEC_LINE_CONSUMER_H__ + +#include +#include + +#include "google/protobuf/io/zero_copy_stream.h" + +namespace google { +namespace protobuf { +namespace compiler { +namespace objectivec { + +// Helper for parsing simple files. +class LineConsumer { + public: + LineConsumer(); + virtual ~LineConsumer(); + virtual bool ConsumeLine(const absl::string_view& line, std::string* out_error) = 0; +}; + +bool ParseSimpleFile(const std::string& path, + LineConsumer* line_consumer, + std::string* out_error); + +bool ParseSimpleStream(io::ZeroCopyInputStream& input_stream, + const std::string& stream_name, + LineConsumer* line_consumer, + std::string* out_error); + +} // namespace objectivec +} // namespace compiler +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_COMPILER_OBJECTIVEC_LINE_CONSUMER_H__ diff --git a/src/google/protobuf/compiler/objectivec/line_consumer_unittest.cc b/src/google/protobuf/compiler/objectivec/line_consumer_unittest.cc new file mode 100644 index 0000000000..2513b6f4fe --- /dev/null +++ b/src/google/protobuf/compiler/objectivec/line_consumer_unittest.cc @@ -0,0 +1,174 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2014 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "google/protobuf/compiler/objectivec/line_consumer.h" +#include "google/protobuf/io/zero_copy_stream_impl_lite.h" +#include "absl/strings/str_cat.h" + +#include + +namespace google { +namespace protobuf { +namespace compiler { +namespace objectivec { +namespace { + +class TestLineCollector : public LineConsumer { + public: + explicit TestLineCollector(std::vector* inout_lines, + const std::string* reject_line = nullptr, + bool skip_msg = false) + : lines_(inout_lines), reject_(reject_line), skip_msg_(skip_msg) {} + + bool ConsumeLine(const absl::string_view& line, std::string* out_error) override { + if (reject_ && *reject_ == line) { + if (!skip_msg_) { + *out_error = std::string("Rejected '") + *reject_ + "'"; + } + return false; + } + if (lines_) { + lines_->emplace_back(line); + } + return true; + } + + private: + std::vector* lines_; + const std::string* reject_; + bool skip_msg_; +}; + +const int kBlockSizes[] = {-1, 1, 2, 5, 64}; +const int kBlockSizeCount = ABSL_ARRAYSIZE(kBlockSizes); + +TEST(ObjCHelper, ParseSimple_BasicsSuccess) { + const std::vector>> tests = { + {"", {}}, + {"a", {"a"}}, + {"a c", {"a c"}}, + {" a c ", {"a c"}}, + {"\ta c ", {"a c"}}, + {"abc\n", {"abc"}}, + {"abc\nd f", {"abc", "d f"}}, + {"\n abc \n def \n\n", {"abc", "def"}}, + }; + + for (const auto& test : tests) { + for (int i = 0; i < kBlockSizeCount; i++) { + io::ArrayInputStream input(test.first.data(), test.first.size(), kBlockSizes[i]); + std::string err_str; + std::vector lines; + TestLineCollector collector(&lines); + EXPECT_TRUE(ParseSimpleStream(input, "dummy", &collector, &err_str)); + EXPECT_EQ(lines, test.second); + EXPECT_TRUE(err_str.empty()); + } + } +} + +TEST(ObjCHelper, ParseSimple_DropsComments) { + const std::vector>> tests = { + {"# nothing", {}}, + {"#", {}}, + {"##", {}}, + {"\n# nothing\n", {}}, + {"a # same line", {"a"}}, + {"a # same line\n", {"a"}}, + {"a\n# line\nc", {"a", "c"}}, + {"# n o t # h i n g #", {}}, + {"## n o # t h i n g #", {}}, + {"a# n o t # h i n g #", {"a"}}, + {"a\n## n o # t h i n g #", {"a"}}, + }; + + for (const auto& test : tests) { + for (int i = 0; i < kBlockSizeCount; i++) { + io::ArrayInputStream input(test.first.data(), test.first.size(), kBlockSizes[i]); + std::string err_str; + std::vector lines; + TestLineCollector collector(&lines); + EXPECT_TRUE(ParseSimpleStream(input, "dummy", &collector, &err_str)); + EXPECT_EQ(lines, test.second); + EXPECT_TRUE(err_str.empty()); + } + } +} + +TEST(ObjCHelper, ParseSimple_RejectLines) { + const std::vector> tests = { + std::make_tuple("a\nb\nc", "a", 1), + std::make_tuple("a\nb\nc", "b", 2), + std::make_tuple("a\nb\nc", "c", 3), + std::make_tuple("a\nb\nc\n", "c", 3), + }; + + for (const auto& test : tests) { + for (int i = 0; i < kBlockSizeCount; i++) { + io::ArrayInputStream input(std::get<0>(test).data(), std::get<0>(test).size(), + kBlockSizes[i]); + std::string err_str; + TestLineCollector collector(nullptr, &std::get<1>(test)); + EXPECT_FALSE(ParseSimpleStream(input, "dummy", &collector, &err_str)); + std::string expected_err = + absl::StrCat("error: dummy Line ", std::get<2>(test), ", Rejected '", std::get<1>(test), "'"); + EXPECT_EQ(err_str, expected_err); + } + } +} + +TEST(ObjCHelper, ParseSimple_RejectLinesNoMessage) { + const std::vector> tests = { + std::make_tuple("a\nb\nc", "a", 1), + std::make_tuple("a\nb\nc", "b", 2), + std::make_tuple("a\nb\nc", "c", 3), + std::make_tuple("a\nb\nc\n", "c", 3), + }; + + for (const auto& test : tests) { + for (int i = 0; i < kBlockSizeCount; i++) { + io::ArrayInputStream input(std::get<0>(test).data(), std::get<0>(test).size(), + kBlockSizes[i]); + std::string err_str; + TestLineCollector collector(nullptr, &std::get<1>(test), true /* skip msg */); + EXPECT_FALSE(ParseSimpleStream(input, "dummy", &collector, &err_str)); + std::string expected_err = + absl::StrCat("error: dummy Line ", std::get<2>(test), + ", ConsumeLine failed without setting an error."); + EXPECT_EQ(err_str, expected_err); + } + } +} + +} // namespace +} // namespace objectivec +} // namespace compiler +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/compiler/objectivec/names.cc b/src/google/protobuf/compiler/objectivec/names.cc index 7ccb713caa..172ed69a27 100644 --- a/src/google/protobuf/compiler/objectivec/names.cc +++ b/src/google/protobuf/compiler/objectivec/names.cc @@ -28,13 +28,6 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#ifndef _MSC_VER -#include -#endif -#include -#include -#include - #include #include #include @@ -43,12 +36,11 @@ #include #include "google/protobuf/compiler/code_generator.h" -#include "absl/strings/str_replace.h" #include "absl/strings/str_split.h" +#include "google/protobuf/compiler/objectivec/line_consumer.h" #include "google/protobuf/compiler/objectivec/names.h" #include "google/protobuf/compiler/objectivec/nsobject_methods.h" #include "google/protobuf/descriptor.pb.h" -#include "google/protobuf/io/io_win32.h" #include "google/protobuf/io/zero_copy_stream_impl.h" // NOTE: src/google/protobuf/compiler/plugin.cc makes use of cerr for some @@ -59,16 +51,6 @@ namespace protobuf { namespace compiler { namespace objectivec { -// is transitively included in this file. Import the functions explicitly -// in this port namespace to avoid ambiguous definition. -namespace posix { -#ifdef _WIN32 -using google::protobuf::io::win32::open; -#else // !_WIN32 -using ::open; -#endif // _WIN32 -} // namespace posix - namespace { bool BoolFromEnvVar(const char* env_var, bool default_value) { @@ -275,10 +257,6 @@ const char* const kUpperSegmentsList[] = {"url", "http", "https"}; std::unordered_set kUpperSegments = MakeWordsMap(kUpperSegmentsList, ABSL_ARRAYSIZE(kUpperSegmentsList)); -bool ascii_isnewline(char c) { - return c == '\n' || c == '\r'; -} - // Internal helper for name handing. // Do not expose this outside of helpers, stick to having functions for specific // cases (ClassName(), FieldName()), so there is always consistent suffix rules. @@ -878,25 +856,6 @@ bool IsProtobufLibraryBundledProtoFile(const FileDescriptor* file) { return false; } -bool ReadLine(absl::string_view* input, absl::string_view* line) { - for (int len = 0; len < input->size(); ++len) { - if (ascii_isnewline((*input)[len])) { - *line = absl::string_view(input->data(), len); - ++len; // advance over the newline - *input = absl::string_view(input->data() + len, input->size() - len); - return true; - } - } - return false; // Ran out of input with no newline. -} - -void RemoveComment(absl::string_view* input) { - int offset = input->find('#'); - if (offset != absl::string_view::npos) { - input->remove_suffix(input->length() - offset); - } -} - namespace { bool PackageToPrefixesCollector::ConsumeLine( @@ -1252,127 +1211,6 @@ bool DecodeDataBuilder::AddCharacter(const char desired, const char input) { } // namespace -namespace { - -class Parser { - public: - Parser(LineConsumer* line_consumer) - : line_consumer_(line_consumer), line_(0) {} - - // Feeds in some input, parse what it can, returning success/failure. Calling - // again after an error is undefined. - bool ParseChunk(absl::string_view chunk, std::string* out_error); - - // Should be called to finish parsing (after all input has been provided via - // successful calls to ParseChunk(), calling after a ParseChunk() failure is - // undefined). Returns success/failure. - bool Finish(std::string* out_error); - - int last_line() const { return line_; } - - private: - LineConsumer* line_consumer_; - int line_; - std::string leftover_; -}; - -bool Parser::ParseChunk(absl::string_view chunk, std::string* out_error) { - absl::string_view full_chunk; - if (!leftover_.empty()) { - leftover_ += std::string(chunk); - full_chunk = absl::string_view(leftover_); - } else { - full_chunk = chunk; - } - - absl::string_view line; - while (ReadLine(&full_chunk, &line)) { - ++line_; - RemoveComment(&line); - TrimWhitespace(&line); - if (!line.empty() && !line_consumer_->ConsumeLine(line, out_error)) { - if (out_error->empty()) { - *out_error = "ConsumeLine failed without setting an error."; - } - leftover_.clear(); - return false; - } - } - - if (full_chunk.empty()) { - leftover_.clear(); - } else { - leftover_ = std::string(full_chunk); - } - return true; -} - -bool Parser::Finish(std::string* out_error) { - // If there is still something to go, flush it with a newline. - if (!leftover_.empty() && !ParseChunk("\n", out_error)) { - return false; - } - // This really should never fail if ParseChunk succeeded, but check to be sure. - if (!leftover_.empty()) { - *out_error = "ParseSimple Internal error: finished with pending data."; - return false; - } - return true; -} - -std::string FullErrorString(const std::string& name, int line_num, const std::string& msg) { - return std::string("error: ") + name + " Line " + absl::StrCat(line_num) + ", " + msg; -} - -} // namespace - -LineConsumer::LineConsumer() {} - -LineConsumer::~LineConsumer() {} - -bool ParseSimpleFile(const std::string& path, LineConsumer* line_consumer, - std::string* out_error) { - int fd; - do { - fd = posix::open(path.c_str(), O_RDONLY); - } while (fd < 0 && errno == EINTR); - if (fd < 0) { - *out_error = std::string("error: Unable to open \"") + path + "\", " + - strerror(errno); - return false; - } - io::FileInputStream file_stream(fd); - file_stream.SetCloseOnDelete(true); - - return ParseSimpleStream(file_stream, path, line_consumer, out_error); -} - -bool ParseSimpleStream(io::ZeroCopyInputStream& input_stream, - const std::string& stream_name, - LineConsumer* line_consumer, - std::string* out_error) { - std::string local_error; - Parser parser(line_consumer); - const void* buf; - int buf_len; - while (input_stream.Next(&buf, &buf_len)) { - if (buf_len == 0) { - continue; - } - - if (!parser.ParseChunk(absl::string_view(static_cast(buf), buf_len), - &local_error)) { - *out_error = FullErrorString(stream_name, parser.last_line(), local_error); - return false; - } - } - if (!parser.Finish(&local_error)) { - *out_error = FullErrorString(stream_name, parser.last_line(), local_error); - return false; - } - return true; -} - } // namespace objectivec } // namespace compiler } // namespace protobuf diff --git a/src/google/protobuf/compiler/objectivec/names.h b/src/google/protobuf/compiler/objectivec/names.h index 103cf1de52..8b17b4dd65 100644 --- a/src/google/protobuf/compiler/objectivec/names.h +++ b/src/google/protobuf/compiler/objectivec/names.h @@ -172,23 +172,6 @@ bool PROTOC_EXPORT ValidateObjCClassPrefixes( bool PROTOC_EXPORT ValidateObjCClassPrefixes( const std::vector& files, std::string* out_error); -// Helper for parsing simple files. -class PROTOC_EXPORT LineConsumer { - public: - LineConsumer(); - virtual ~LineConsumer(); - virtual bool ConsumeLine(const absl::string_view& line, std::string* out_error) = 0; -}; - -bool PROTOC_EXPORT ParseSimpleFile(const std::string& path, - LineConsumer* line_consumer, - std::string* out_error); - -bool PROTOC_EXPORT ParseSimpleStream(io::ZeroCopyInputStream& input_stream, - const std::string& stream_name, - LineConsumer* line_consumer, - std::string* out_error); - } // namespace objectivec } // namespace compiler } // namespace protobuf diff --git a/src/google/protobuf/compiler/objectivec/names_unittest.cc b/src/google/protobuf/compiler/objectivec/names_unittest.cc index f472a76448..0aac79497c 100644 --- a/src/google/protobuf/compiler/objectivec/names_unittest.cc +++ b/src/google/protobuf/compiler/objectivec/names_unittest.cc @@ -39,133 +39,6 @@ namespace compiler { namespace objectivec { namespace { -class TestLineCollector : public LineConsumer { - public: - explicit TestLineCollector(std::vector* inout_lines, - const std::string* reject_line = nullptr, - bool skip_msg = false) - : lines_(inout_lines), reject_(reject_line), skip_msg_(skip_msg) {} - - bool ConsumeLine(const absl::string_view& line, std::string* out_error) override { - if (reject_ && *reject_ == line) { - if (!skip_msg_) { - *out_error = std::string("Rejected '") + *reject_ + "'"; - } - return false; - } - if (lines_) { - lines_->emplace_back(line); - } - return true; - } - - private: - std::vector* lines_; - const std::string* reject_; - bool skip_msg_; -}; - -const int kBlockSizes[] = {-1, 1, 2, 5, 64}; -const int kBlockSizeCount = ABSL_ARRAYSIZE(kBlockSizes); - -TEST(ObjCHelper, ParseSimple_BasicsSuccess) { - const std::vector>> tests = { - {"", {}}, - {"a", {"a"}}, - {"a c", {"a c"}}, - {" a c ", {"a c"}}, - {"\ta c ", {"a c"}}, - {"abc\n", {"abc"}}, - {"abc\nd f", {"abc", "d f"}}, - {"\n abc \n def \n\n", {"abc", "def"}}, - }; - - for (const auto& test : tests) { - for (int i = 0; i < kBlockSizeCount; i++) { - io::ArrayInputStream input(test.first.data(), test.first.size(), kBlockSizes[i]); - std::string err_str; - std::vector lines; - TestLineCollector collector(&lines); - EXPECT_TRUE(ParseSimpleStream(input, "dummy", &collector, &err_str)); - EXPECT_EQ(lines, test.second); - EXPECT_TRUE(err_str.empty()); - } - } -} - -TEST(ObjCHelper, ParseSimple_DropsComments) { - const std::vector>> tests = { - {"# nothing", {}}, - {"#", {}}, - {"##", {}}, - {"\n# nothing\n", {}}, - {"a # same line", {"a"}}, - {"a # same line\n", {"a"}}, - {"a\n# line\nc", {"a", "c"}}, - {"# n o t # h i n g #", {}}, - {"## n o # t h i n g #", {}}, - {"a# n o t # h i n g #", {"a"}}, - {"a\n## n o # t h i n g #", {"a"}}, - }; - - for (const auto& test : tests) { - for (int i = 0; i < kBlockSizeCount; i++) { - io::ArrayInputStream input(test.first.data(), test.first.size(), kBlockSizes[i]); - std::string err_str; - std::vector lines; - TestLineCollector collector(&lines); - EXPECT_TRUE(ParseSimpleStream(input, "dummy", &collector, &err_str)); - EXPECT_EQ(lines, test.second); - EXPECT_TRUE(err_str.empty()); - } - } -} - -TEST(ObjCHelper, ParseSimple_RejectLines) { - const std::vector> tests = { - std::make_tuple("a\nb\nc", "a", 1), - std::make_tuple("a\nb\nc", "b", 2), - std::make_tuple("a\nb\nc", "c", 3), - std::make_tuple("a\nb\nc\n", "c", 3), - }; - - for (const auto& test : tests) { - for (int i = 0; i < kBlockSizeCount; i++) { - io::ArrayInputStream input(std::get<0>(test).data(), std::get<0>(test).size(), - kBlockSizes[i]); - std::string err_str; - TestLineCollector collector(nullptr, &std::get<1>(test)); - EXPECT_FALSE(ParseSimpleStream(input, "dummy", &collector, &err_str)); - std::string expected_err = - absl::StrCat("error: dummy Line ", std::get<2>(test), ", Rejected '", std::get<1>(test), "'"); - EXPECT_EQ(err_str, expected_err); - } - } -} - -TEST(ObjCHelper, ParseSimple_RejectLinesNoMessage) { - const std::vector> tests = { - std::make_tuple("a\nb\nc", "a", 1), - std::make_tuple("a\nb\nc", "b", 2), - std::make_tuple("a\nb\nc", "c", 3), - std::make_tuple("a\nb\nc\n", "c", 3), - }; - - for (const auto& test : tests) { - for (int i = 0; i < kBlockSizeCount; i++) { - io::ArrayInputStream input(std::get<0>(test).data(), std::get<0>(test).size(), - kBlockSizes[i]); - std::string err_str; - TestLineCollector collector(nullptr, &std::get<1>(test), true /* skip msg */); - EXPECT_FALSE(ParseSimpleStream(input, "dummy", &collector, &err_str)); - std::string expected_err = - absl::StrCat("error: dummy Line ", std::get<2>(test), - ", ConsumeLine failed without setting an error."); - EXPECT_EQ(err_str, expected_err); - } - } -} - TEST(ObjCHelper, IsRetainedName) { EXPECT_TRUE(IsRetainedName("new")); EXPECT_TRUE(IsRetainedName("alloc"));