Make a new (private to the package) target for this so names can depend on it as an implementation detail as well as the other generator code. Split the names unittests to have the LineConsumer out on its own also.pull/10720/head
Thomas Van Lenten
2 years ago
parent
5ace0e8450
commit
bb2bba124c
9 changed files with 496 additions and 308 deletions
@ -0,0 +1,224 @@ |
||||
// Protocol Buffers - Google's data interchange format
|
||||
// Copyright 2008 Google Inc. All rights reserved.
|
||||
// https://developers.google.com/protocol-buffers/
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are
|
||||
// met:
|
||||
//
|
||||
// * Redistributions of source code must retain the above copyright
|
||||
// notice, this list of conditions and the following disclaimer.
|
||||
// * Redistributions in binary form must reproduce the above
|
||||
// copyright notice, this list of conditions and the following disclaimer
|
||||
// in the documentation and/or other materials provided with the
|
||||
// distribution.
|
||||
// * Neither the name of Google Inc. nor the names of its
|
||||
// contributors may be used to endorse or promote products derived from
|
||||
// this software without specific prior written permission.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
#ifndef _MSC_VER |
||||
#include <unistd.h> |
||||
#endif |
||||
#include <errno.h> |
||||
#include <fcntl.h> |
||||
#include <stdlib.h> |
||||
|
||||
#include <climits> |
||||
#include <fstream> |
||||
#include <iostream> |
||||
#include <sstream> |
||||
#include <vector> |
||||
|
||||
#include "absl/strings/str_cat.h" |
||||
#include "absl/strings/str_split.h" |
||||
#include "google/protobuf/compiler/objectivec/line_consumer.h" |
||||
#include "google/protobuf/io/io_win32.h" |
||||
#include "google/protobuf/io/zero_copy_stream_impl.h" |
||||
|
||||
// NOTE: src/google/protobuf/compiler/plugin.cc makes use of cerr for some
|
||||
// error cases, so it seems to be ok to use as a back door for errors.
|
||||
|
||||
namespace google { |
||||
namespace protobuf { |
||||
namespace compiler { |
||||
namespace objectivec { |
||||
|
||||
// <io.h> is transitively included in this file. Import the functions explicitly
|
||||
// in this port namespace to avoid ambiguous definition.
|
||||
namespace posix { |
||||
#ifdef _WIN32 |
||||
using google::protobuf::io::win32::open; |
||||
#else // !_WIN32
|
||||
using ::open; |
||||
#endif // _WIN32
|
||||
} // namespace posix
|
||||
|
||||
namespace { |
||||
|
||||
void TrimWhitespace(absl::string_view* input) { |
||||
while (!input->empty() && absl::ascii_isspace(*input->data())) { |
||||
input->remove_prefix(1); |
||||
} |
||||
while (!input->empty() && absl::ascii_isspace((*input)[input->length() - 1])) { |
||||
input->remove_suffix(1); |
||||
} |
||||
} |
||||
|
||||
bool ascii_isnewline(char c) { |
||||
return c == '\n' || c == '\r'; |
||||
} |
||||
|
||||
bool ReadLine(absl::string_view* input, absl::string_view* line) { |
||||
for (int len = 0; len < input->size(); ++len) { |
||||
if (ascii_isnewline((*input)[len])) { |
||||
*line = absl::string_view(input->data(), len); |
||||
++len; // advance over the newline
|
||||
*input = absl::string_view(input->data() + len, input->size() - len); |
||||
return true; |
||||
} |
||||
} |
||||
return false; // Ran out of input with no newline.
|
||||
} |
||||
|
||||
void RemoveComment(absl::string_view* input) { |
||||
int offset = input->find('#'); |
||||
if (offset != absl::string_view::npos) { |
||||
input->remove_suffix(input->length() - offset); |
||||
} |
||||
} |
||||
|
||||
class Parser { |
||||
public: |
||||
Parser(LineConsumer* line_consumer) |
||||
: line_consumer_(line_consumer), line_(0) {} |
||||
|
||||
// Feeds in some input, parse what it can, returning success/failure. Calling
|
||||
// again after an error is undefined.
|
||||
bool ParseChunk(absl::string_view chunk, std::string* out_error); |
||||
|
||||
// Should be called to finish parsing (after all input has been provided via
|
||||
// successful calls to ParseChunk(), calling after a ParseChunk() failure is
|
||||
// undefined). Returns success/failure.
|
||||
bool Finish(std::string* out_error); |
||||
|
||||
int last_line() const { return line_; } |
||||
|
||||
private: |
||||
LineConsumer* line_consumer_; |
||||
int line_; |
||||
std::string leftover_; |
||||
}; |
||||
|
||||
bool Parser::ParseChunk(absl::string_view chunk, std::string* out_error) { |
||||
absl::string_view full_chunk; |
||||
if (!leftover_.empty()) { |
||||
leftover_ += std::string(chunk); |
||||
full_chunk = absl::string_view(leftover_); |
||||
} else { |
||||
full_chunk = chunk; |
||||
} |
||||
|
||||
absl::string_view line; |
||||
while (ReadLine(&full_chunk, &line)) { |
||||
++line_; |
||||
RemoveComment(&line); |
||||
TrimWhitespace(&line); |
||||
if (!line.empty() && !line_consumer_->ConsumeLine(line, out_error)) { |
||||
if (out_error->empty()) { |
||||
*out_error = "ConsumeLine failed without setting an error."; |
||||
} |
||||
leftover_.clear(); |
||||
return false; |
||||
} |
||||
} |
||||
|
||||
if (full_chunk.empty()) { |
||||
leftover_.clear(); |
||||
} else { |
||||
leftover_ = std::string(full_chunk); |
||||
} |
||||
return true; |
||||
} |
||||
|
||||
bool Parser::Finish(std::string* out_error) { |
||||
// If there is still something to go, flush it with a newline.
|
||||
if (!leftover_.empty() && !ParseChunk("\n", out_error)) { |
||||
return false; |
||||
} |
||||
// This really should never fail if ParseChunk succeeded, but check to be sure.
|
||||
if (!leftover_.empty()) { |
||||
*out_error = "ParseSimple Internal error: finished with pending data."; |
||||
return false; |
||||
} |
||||
return true; |
||||
} |
||||
|
||||
std::string FullErrorString(const std::string& name, int line_num, const std::string& msg) { |
||||
return std::string("error: ") + name + " Line " + absl::StrCat(line_num) + ", " + msg; |
||||
} |
||||
|
||||
} // namespace
|
||||
|
||||
LineConsumer::LineConsumer() {} |
||||
|
||||
LineConsumer::~LineConsumer() {} |
||||
|
||||
bool ParseSimpleFile(const std::string& path, LineConsumer* line_consumer, |
||||
std::string* out_error) { |
||||
int fd; |
||||
do { |
||||
fd = posix::open(path.c_str(), O_RDONLY); |
||||
} while (fd < 0 && errno == EINTR); |
||||
if (fd < 0) { |
||||
*out_error = std::string("error: Unable to open \"") + path + "\", " + |
||||
strerror(errno); |
||||
return false; |
||||
} |
||||
io::FileInputStream file_stream(fd); |
||||
file_stream.SetCloseOnDelete(true); |
||||
|
||||
return ParseSimpleStream(file_stream, path, line_consumer, out_error); |
||||
} |
||||
|
||||
bool ParseSimpleStream(io::ZeroCopyInputStream& input_stream, |
||||
const std::string& stream_name, |
||||
LineConsumer* line_consumer, |
||||
std::string* out_error) { |
||||
std::string local_error; |
||||
Parser parser(line_consumer); |
||||
const void* buf; |
||||
int buf_len; |
||||
while (input_stream.Next(&buf, &buf_len)) { |
||||
if (buf_len == 0) { |
||||
continue; |
||||
} |
||||
|
||||
if (!parser.ParseChunk(absl::string_view(static_cast<const char*>(buf), buf_len), |
||||
&local_error)) { |
||||
*out_error = FullErrorString(stream_name, parser.last_line(), local_error); |
||||
return false; |
||||
} |
||||
} |
||||
if (!parser.Finish(&local_error)) { |
||||
*out_error = FullErrorString(stream_name, parser.last_line(), local_error); |
||||
return false; |
||||
} |
||||
return true; |
||||
} |
||||
|
||||
} // namespace objectivec
|
||||
} // namespace compiler
|
||||
} // namespace protobuf
|
||||
} // namespace google
|
||||
@ -0,0 +1,66 @@ |
||||
// Protocol Buffers - Google's data interchange format
|
||||
// Copyright 2008 Google Inc. All rights reserved.
|
||||
// https://developers.google.com/protocol-buffers/
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are
|
||||
// met:
|
||||
//
|
||||
// * Redistributions of source code must retain the above copyright
|
||||
// notice, this list of conditions and the following disclaimer.
|
||||
// * Redistributions in binary form must reproduce the above
|
||||
// copyright notice, this list of conditions and the following disclaimer
|
||||
// in the documentation and/or other materials provided with the
|
||||
// distribution.
|
||||
// * Neither the name of Google Inc. nor the names of its
|
||||
// contributors may be used to endorse or promote products derived from
|
||||
// this software without specific prior written permission.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
#ifndef GOOGLE_PROTOBUF_COMPILER_OBJECTIVEC_LINE_CONSUMER_H__ |
||||
#define GOOGLE_PROTOBUF_COMPILER_OBJECTIVEC_LINE_CONSUMER_H__ |
||||
|
||||
#include <string> |
||||
#include <vector> |
||||
|
||||
#include "google/protobuf/io/zero_copy_stream.h" |
||||
|
||||
namespace google { |
||||
namespace protobuf { |
||||
namespace compiler { |
||||
namespace objectivec { |
||||
|
||||
// Helper for parsing simple files.
|
||||
class LineConsumer { |
||||
public: |
||||
LineConsumer(); |
||||
virtual ~LineConsumer(); |
||||
virtual bool ConsumeLine(const absl::string_view& line, std::string* out_error) = 0; |
||||
}; |
||||
|
||||
bool ParseSimpleFile(const std::string& path, |
||||
LineConsumer* line_consumer, |
||||
std::string* out_error); |
||||
|
||||
bool ParseSimpleStream(io::ZeroCopyInputStream& input_stream, |
||||
const std::string& stream_name, |
||||
LineConsumer* line_consumer, |
||||
std::string* out_error); |
||||
|
||||
} // namespace objectivec
|
||||
} // namespace compiler
|
||||
} // namespace protobuf
|
||||
} // namespace google
|
||||
|
||||
#endif // GOOGLE_PROTOBUF_COMPILER_OBJECTIVEC_LINE_CONSUMER_H__
|
||||
@ -0,0 +1,174 @@ |
||||
// Protocol Buffers - Google's data interchange format
|
||||
// Copyright 2014 Google Inc. All rights reserved.
|
||||
// https://developers.google.com/protocol-buffers/
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are
|
||||
// met:
|
||||
//
|
||||
// * Redistributions of source code must retain the above copyright
|
||||
// notice, this list of conditions and the following disclaimer.
|
||||
// * Redistributions in binary form must reproduce the above
|
||||
// copyright notice, this list of conditions and the following disclaimer
|
||||
// in the documentation and/or other materials provided with the
|
||||
// distribution.
|
||||
// * Neither the name of Google Inc. nor the names of its
|
||||
// contributors may be used to endorse or promote products derived from
|
||||
// this software without specific prior written permission.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
#include "google/protobuf/compiler/objectivec/line_consumer.h" |
||||
#include "google/protobuf/io/zero_copy_stream_impl_lite.h" |
||||
#include "absl/strings/str_cat.h" |
||||
|
||||
#include <gtest/gtest.h> |
||||
|
||||
namespace google { |
||||
namespace protobuf { |
||||
namespace compiler { |
||||
namespace objectivec { |
||||
namespace { |
||||
|
||||
class TestLineCollector : public LineConsumer { |
||||
public: |
||||
explicit TestLineCollector(std::vector<std::string>* inout_lines, |
||||
const std::string* reject_line = nullptr, |
||||
bool skip_msg = false) |
||||
: lines_(inout_lines), reject_(reject_line), skip_msg_(skip_msg) {} |
||||
|
||||
bool ConsumeLine(const absl::string_view& line, std::string* out_error) override { |
||||
if (reject_ && *reject_ == line) { |
||||
if (!skip_msg_) { |
||||
*out_error = std::string("Rejected '") + *reject_ + "'"; |
||||
} |
||||
return false; |
||||
} |
||||
if (lines_) { |
||||
lines_->emplace_back(line); |
||||
} |
||||
return true; |
||||
} |
||||
|
||||
private: |
||||
std::vector<std::string>* lines_; |
||||
const std::string* reject_; |
||||
bool skip_msg_; |
||||
}; |
||||
|
||||
const int kBlockSizes[] = {-1, 1, 2, 5, 64}; |
||||
const int kBlockSizeCount = ABSL_ARRAYSIZE(kBlockSizes); |
||||
|
||||
TEST(ObjCHelper, ParseSimple_BasicsSuccess) { |
||||
const std::vector<std::pair<std::string, std::vector<std::string>>> tests = { |
||||
{"", {}}, |
||||
{"a", {"a"}}, |
||||
{"a c", {"a c"}}, |
||||
{" a c ", {"a c"}}, |
||||
{"\ta c ", {"a c"}}, |
||||
{"abc\n", {"abc"}}, |
||||
{"abc\nd f", {"abc", "d f"}}, |
||||
{"\n abc \n def \n\n", {"abc", "def"}}, |
||||
}; |
||||
|
||||
for (const auto& test : tests) { |
||||
for (int i = 0; i < kBlockSizeCount; i++) { |
||||
io::ArrayInputStream input(test.first.data(), test.first.size(), kBlockSizes[i]); |
||||
std::string err_str; |
||||
std::vector<std::string> lines; |
||||
TestLineCollector collector(&lines); |
||||
EXPECT_TRUE(ParseSimpleStream(input, "dummy", &collector, &err_str)); |
||||
EXPECT_EQ(lines, test.second); |
||||
EXPECT_TRUE(err_str.empty()); |
||||
} |
||||
} |
||||
} |
||||
|
||||
TEST(ObjCHelper, ParseSimple_DropsComments) { |
||||
const std::vector<std::pair<std::string, std::vector<std::string>>> tests = { |
||||
{"# nothing", {}}, |
||||
{"#", {}}, |
||||
{"##", {}}, |
||||
{"\n# nothing\n", {}}, |
||||
{"a # same line", {"a"}}, |
||||
{"a # same line\n", {"a"}}, |
||||
{"a\n# line\nc", {"a", "c"}}, |
||||
{"# n o t # h i n g #", {}}, |
||||
{"## n o # t h i n g #", {}}, |
||||
{"a# n o t # h i n g #", {"a"}}, |
||||
{"a\n## n o # t h i n g #", {"a"}}, |
||||
}; |
||||
|
||||
for (const auto& test : tests) { |
||||
for (int i = 0; i < kBlockSizeCount; i++) { |
||||
io::ArrayInputStream input(test.first.data(), test.first.size(), kBlockSizes[i]); |
||||
std::string err_str; |
||||
std::vector<std::string> lines; |
||||
TestLineCollector collector(&lines); |
||||
EXPECT_TRUE(ParseSimpleStream(input, "dummy", &collector, &err_str)); |
||||
EXPECT_EQ(lines, test.second); |
||||
EXPECT_TRUE(err_str.empty()); |
||||
} |
||||
} |
||||
} |
||||
|
||||
TEST(ObjCHelper, ParseSimple_RejectLines) { |
||||
const std::vector<std::tuple<std::string, std::string, int>> tests = { |
||||
std::make_tuple("a\nb\nc", "a", 1), |
||||
std::make_tuple("a\nb\nc", "b", 2), |
||||
std::make_tuple("a\nb\nc", "c", 3), |
||||
std::make_tuple("a\nb\nc\n", "c", 3), |
||||
}; |
||||
|
||||
for (const auto& test : tests) { |
||||
for (int i = 0; i < kBlockSizeCount; i++) { |
||||
io::ArrayInputStream input(std::get<0>(test).data(), std::get<0>(test).size(), |
||||
kBlockSizes[i]); |
||||
std::string err_str; |
||||
TestLineCollector collector(nullptr, &std::get<1>(test)); |
||||
EXPECT_FALSE(ParseSimpleStream(input, "dummy", &collector, &err_str)); |
||||
std::string expected_err = |
||||
absl::StrCat("error: dummy Line ", std::get<2>(test), ", Rejected '", std::get<1>(test), "'"); |
||||
EXPECT_EQ(err_str, expected_err); |
||||
} |
||||
} |
||||
} |
||||
|
||||
TEST(ObjCHelper, ParseSimple_RejectLinesNoMessage) { |
||||
const std::vector<std::tuple<std::string, std::string, int>> tests = { |
||||
std::make_tuple("a\nb\nc", "a", 1), |
||||
std::make_tuple("a\nb\nc", "b", 2), |
||||
std::make_tuple("a\nb\nc", "c", 3), |
||||
std::make_tuple("a\nb\nc\n", "c", 3), |
||||
}; |
||||
|
||||
for (const auto& test : tests) { |
||||
for (int i = 0; i < kBlockSizeCount; i++) { |
||||
io::ArrayInputStream input(std::get<0>(test).data(), std::get<0>(test).size(), |
||||
kBlockSizes[i]); |
||||
std::string err_str; |
||||
TestLineCollector collector(nullptr, &std::get<1>(test), true /* skip msg */); |
||||
EXPECT_FALSE(ParseSimpleStream(input, "dummy", &collector, &err_str)); |
||||
std::string expected_err = |
||||
absl::StrCat("error: dummy Line ", std::get<2>(test), |
||||
", ConsumeLine failed without setting an error."); |
||||
EXPECT_EQ(err_str, expected_err); |
||||
} |
||||
} |
||||
} |
||||
|
||||
} // namespace
|
||||
} // namespace objectivec
|
||||
} // namespace compiler
|
||||
} // namespace protobuf
|
||||
} // namespace google
|
||||
Loading…
Reference in new issue