Chiebot-Mirror
/
protobuf
mirror of https://github.com/protocolbuffers/protobuf.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

internal change

Browse Source 
PiperOrigin-RevId: 537990149
pull/12985/head
Protobuf Team Bot 2 years ago committed by Copybara-Service
parent a10c2b77c3
commit 6809f6c386
5 changed files with 115 additions and 64 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      src/google/protobuf/redaction_metric_test.cc
  2. 55
      src/google/protobuf/text_format.cc
  3. 12
      src/google/protobuf/text_format.h
  4. 105
      src/google/protobuf/text_format_unittest.cc
  5. 3
      src/google/protobuf/unittest.proto

4
src/google/protobuf/redaction_metric_test.cc
Unescape View File

@ -33,6 +33,7 @@
// separate file to more accurately test their values.
#include <cstdint>
#include <string>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
@ -48,10 +49,11 @@ namespace {
using ::testing::HasSubstr;
TEST(TextFormatParsingMetricsTest, MetricsTest) {
std::string value_replacement = "[REDACTED]";
protobuf_unittest::RedactedFields proto;
proto.set_optional_redacted_string("foo");
int64_t before = internal::GetRedactedFieldCount();
EXPECT_THAT(absl::StrCat(proto), HasSubstr("[REDACTED]"));
EXPECT_THAT(absl::StrCat(proto), HasSubstr(value_replacement));
int64_t after = internal::GetRedactedFieldCount();
EXPECT_EQ(after, before + 1);
}

55
src/google/protobuf/text_format.cc
Unescape View File

@ -82,6 +82,8 @@ using internal::ScopedReflectionMode;
namespace {
const absl::string_view kFieldValueReplacement = "[REDACTED]";
inline bool IsHexNumber(const std::string& str) {
return (str.length() >= 2 && str[0] == '0' &&
(str[1] == 'x' || str[1] == 'X'));
@ -115,6 +117,10 @@ const char kDebugStringSilentMarkerForDetection[] = "\t ";
// Controls insertion of kDebugStringSilentMarker into DebugString() output.
PROTOBUF_EXPORT std::atomic<bool> enable_debug_text_format_marker;
// Controls insertion of a marker making debug strings non-parseable, and
// redacting annotated fields.
PROTOBUF_EXPORT std::atomic<bool> enable_debug_text_redaction{true};
int64_t GetRedactedFieldCount() {
return num_redacted_field.load(std::memory_order_relaxed);
}
@ -190,9 +196,10 @@ std::string StringifyMessage(const Message& message, Option option) {
break;
}
printer.SetExpandAny(true);
printer.SetRedactDebugString(true);
printer.SetRedactDebugString(
internal::enable_debug_text_redaction.load(std::memory_order_relaxed));
printer.SetRandomizeDebugString(true);
printer.SetRootMessageFullName(message.GetDescriptor()->full_name());
printer.SetReportSensitiveFields(true);
std::string result;
printer.PrintToString(message, &result);
@ -2099,12 +2106,12 @@ TextFormat::Printer::Printer()
insert_silent_marker_(false),
redact_debug_string_(false),
randomize_debug_string_(false),
report_sensitive_fields_(false),
hide_unknown_fields_(false),
print_message_fields_in_index_order_(false),
expand_any_(false),
truncate_string_field_longer_than_(0LL),
finder_(nullptr),
root_message_full_name_("") {
finder_(nullptr) {
SetUseUtf8StringEscaping(false);
}
@ -2863,27 +2870,35 @@ void TextFormat::Printer::PrintUnknownFields(
}
}
namespace {
// Check if the field is sensitive and should be redacted.
bool ShouldRedactField(const FieldDescriptor* field) {
if (field->options().debug_redact()) return true;
return false;
}
} // namespace
bool TextFormat::Printer::TryRedactFieldValue(
const Message& message, const FieldDescriptor* field,
BaseTextGenerator* generator, bool insert_value_separator) const {
auto do_redact = [&](absl::string_view replacement) {
IncrementRedactedFieldCounter();
if (insert_value_separator) {
generator->PrintMaybeWithMarker(MarkerToken(), ": ");
}
generator->PrintString(replacement);
if (insert_value_separator) {
if (single_line_mode_) {
generator->PrintLiteral(" ");
} else {
generator->PrintLiteral("\n");
if (ShouldRedactField(field)) {
if (redact_debug_string_) {
IncrementRedactedFieldCounter();
if (insert_value_separator) {
generator->PrintMaybeWithMarker(MarkerToken(), ": ");
}
generator->PrintString(kFieldValueReplacement);
if (insert_value_separator) {
if (single_line_mode_) {
generator->PrintLiteral(" ");
} else {
generator->PrintLiteral("\n");
}
}
return true;
}
};
if (redact_debug_string_ && field->options().debug_redact()) {
do_redact("[REDACTED]");
return true;
}
return false;
}

12
src/google/protobuf/text_format.h
Unescape View File

@ -45,6 +45,7 @@
#include "google/protobuf/port.h"
#include "absl/container/flat_hash_map.h"
#include "absl/container/flat_hash_set.h"
#include "absl/strings/cord.h"
#include "absl/strings/string_view.h"
#include "google/protobuf/descriptor.h"
@ -68,6 +69,8 @@ PROTOBUF_EXPORT extern const char kDebugStringSilentMarker[1];
PROTOBUF_EXPORT extern const char kDebugStringSilentMarkerForDetection[3];
PROTOBUF_EXPORT extern std::atomic<bool> enable_debug_text_format_marker;
PROTOBUF_EXPORT extern std::atomic<bool> enable_debug_text_detection;
PROTOBUF_EXPORT extern std::atomic<bool> enable_debug_text_redaction;
PROTOBUF_EXPORT int64_t GetRedactedFieldCount();
} // namespace internal
@ -445,9 +448,10 @@ class PROTOBUF_EXPORT TextFormat {
randomize_debug_string_ = randomize;
}
// Sets the.
void SetRootMessageFullName(absl::string_view root_message_full_name) {
root_message_full_name_ = std::string(root_message_full_name);
// Sets whether sensitive fields found in the message will be reported or
// not.
void SetReportSensitiveFields(bool report) {
report_sensitive_fields_ = report;
}
// Forward declaration of an internal class used to print the text
@ -521,6 +525,7 @@ class PROTOBUF_EXPORT TextFormat {
bool insert_silent_marker_;
bool redact_debug_string_;
bool randomize_debug_string_;
bool report_sensitive_fields_;
bool hide_unknown_fields_;
bool print_message_fields_in_index_order_;
bool expand_any_;
@ -536,7 +541,6 @@ class PROTOBUF_EXPORT TextFormat {
custom_message_printers_;
const Finder* finder_;
std::string root_message_full_name_;
};
// Parses a text-format protocol message from the given input stream to

105
src/google/protobuf/text_format_unittest.cc
Unescape View File

@ -168,8 +168,8 @@ TEST_F(TextFormatTest, ShortFormat) {
unittest::RedactedFields proto;
unittest::TestNestedMessageRedaction redacted_nested_proto;
unittest::TestNestedMessageRedaction unredacted_nested_proto;
redacted_nested_proto.set_optional_nested_string("hello");
unredacted_nested_proto.set_optional_nested_string("world");
redacted_nested_proto.set_optional_unredacted_nested_string("hello");
unredacted_nested_proto.set_optional_unredacted_nested_string("world");
proto.set_optional_redacted_string("foo");
proto.set_optional_unredacted_string("bar");
proto.add_repeated_redacted_string("1");
@ -186,38 +186,53 @@ TEST_F(TextFormatTest, ShortFormat) {
proto.add_repeated_unredacted_message();
unittest::TestNestedMessageRedaction* unredacted_message_2 =
proto.add_repeated_unredacted_message();
redacted_message_1->set_optional_nested_string("5");
redacted_message_2->set_optional_nested_string("6");
unredacted_message_1->set_optional_nested_string("7");
unredacted_message_2->set_optional_nested_string("8");
redacted_message_1->set_optional_unredacted_nested_string("5");
redacted_message_2->set_optional_unredacted_nested_string("6");
unredacted_message_1->set_optional_unredacted_nested_string("7");
unredacted_message_2->set_optional_unredacted_nested_string("8");
(*proto.mutable_map_redacted_string())["abc"] = "def";
(*proto.mutable_map_unredacted_string())["ghi"] = "jkl";
EXPECT_THAT(
google::protobuf::ShortFormat(proto),
testing::MatchesRegex(
"optional_redacted_string: \\[REDACTED\\] "
"optional_unredacted_string: \"bar\" "
"repeated_redacted_string: \\[REDACTED\\] "
"repeated_redacted_string: \\[REDACTED\\] "
"repeated_unredacted_string: \"3\" "
"repeated_unredacted_string: \"4\" "
"optional_redacted_message: \\[REDACTED\\] "
"optional_unredacted_message \\{ "
"optional_nested_string: \"world\" \\} "
"repeated_redacted_message: \\[REDACTED\\] "
"repeated_unredacted_message \\{ optional_nested_string: \"7\" \\} "
"repeated_unredacted_message \\{ optional_nested_string: \"8\" \\} "
"map_redacted_string: \\[REDACTED\\] "
"map_unredacted_string \\{ key: \"ghi\" value: \"jkl\" \\}"));
std::string value_replacement = "\\[REDACTED\\]";
EXPECT_THAT(google::protobuf::ShortFormat(proto),
testing::MatchesRegex(
"optional_redacted_string: " +
value_replacement +
" "
"optional_unredacted_string: \"bar\" "
"repeated_redacted_string: " +
value_replacement +
" "
"repeated_redacted_string: " +
value_replacement +
" "
"repeated_unredacted_string: \"3\" "
"repeated_unredacted_string: \"4\" "
"optional_redacted_message: " +
value_replacement +
" "
"optional_unredacted_message \\{ "
"optional_unredacted_nested_string: \"world\" \\} "
"repeated_redacted_message: " +
value_replacement +
" "
"repeated_unredacted_message "
"\\{ optional_unredacted_nested_string: \"7\" \\} "
"repeated_unredacted_message "
"\\{ optional_unredacted_nested_string: \"8\" \\} "
"map_redacted_string: " +
value_replacement +
" "
"map_unredacted_string \\{ key: \"ghi\" value: \"jkl\" \\}"));
}
TEST_F(TextFormatTest, Utf8Format) {
unittest::RedactedFields proto;
unittest::TestNestedMessageRedaction redacted_nested_proto;
unittest::TestNestedMessageRedaction unredacted_nested_proto;
redacted_nested_proto.set_optional_nested_string("\350\260\267\346\255\214");
unredacted_nested_proto.set_optional_nested_string(
redacted_nested_proto.set_optional_unredacted_nested_string(
"\350\260\267\346\255\214");
unredacted_nested_proto.set_optional_unredacted_nested_string(
"\350\260\267\346\255\214");
proto.set_optional_redacted_string("foo");
proto.set_optional_unredacted_string("bar");
@ -235,30 +250,44 @@ TEST_F(TextFormatTest, Utf8Format) {
proto.add_repeated_unredacted_message();
unittest::TestNestedMessageRedaction* unredacted_message_2 =
proto.add_repeated_unredacted_message();
redacted_message_1->set_optional_nested_string("5");
redacted_message_2->set_optional_nested_string("6");
unredacted_message_1->set_optional_nested_string("7");
unredacted_message_2->set_optional_nested_string("8");
redacted_message_1->set_optional_unredacted_nested_string("5");
redacted_message_2->set_optional_unredacted_nested_string("6");
unredacted_message_1->set_optional_unredacted_nested_string("7");
unredacted_message_2->set_optional_unredacted_nested_string("8");
(*proto.mutable_map_redacted_string())["abc"] = "def";
(*proto.mutable_map_unredacted_string())["ghi"] = "jkl";
std::string value_replacement = "\\[REDACTED\\]";
EXPECT_THAT(google::protobuf::Utf8Format(proto),
testing::MatchesRegex(
"optional_redacted_string: \\[REDACTED\\]\n"
"optional_redacted_string: " +
value_replacement +
"\n"
"optional_unredacted_string: \"bar\"\n"
"repeated_redacted_string: \\[REDACTED\\]\n"
"repeated_redacted_string: \\[REDACTED\\]\n"
"repeated_redacted_string: " +
value_replacement +
"\n"
"repeated_redacted_string: " +
value_replacement +
"\n"
"repeated_unredacted_string: \"3\"\n"
"repeated_unredacted_string: \"4\"\n"
"optional_redacted_message: \\[REDACTED\\]\n"
"optional_redacted_message: " +
value_replacement +
"\n"
"optional_unredacted_message \\{\n "
"optional_nested_string: \"\xE8\xB0\xB7\xE6\xAD\x8C\"\n\\}\n"
"repeated_redacted_message: \\[REDACTED\\]\n"
"optional_unredacted_nested_string: "
"\"\xE8\xB0\xB7\xE6\xAD\x8C\"\n\\}\n"
"repeated_redacted_message: " +
value_replacement +
"\n"
"repeated_unredacted_message \\{\n "
"optional_nested_string: \"7\"\n\\}\n"
"optional_unredacted_nested_string: \"7\"\n\\}\n"
"repeated_unredacted_message \\{\n "
"optional_nested_string: \"8\"\n\\}\n"
"map_redacted_string: \\[REDACTED\\]\n"
"optional_unredacted_nested_string: \"8\"\n\\}\n"
"map_redacted_string: " +
value_replacement +
"\n"
"map_unredacted_string \\{\n "
"key: \"ghi\"\n value: \"jkl\"\n\\}\n"));
}

3
src/google/protobuf/unittest.proto
Unescape View File

@ -1664,7 +1664,8 @@ message BadFieldNames{
}
message TestNestedMessageRedaction {
optional string optional_nested_string = 1;
optional string optional_unredacted_nested_string = 1;
optional string optional_redacted_nested_string = 2 [debug_redact = true];
}
message RedactedFields {

Write Preview
Loading…
Cancel
Save