Merge pull request #10341 from fowles/sync-stage

Integrate from Piper for C++, Java, and Python
pull/10354/head
Matt Fowles Kulukundis 2 years ago committed by GitHub
commit eb95655721
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      cmake/tests.cmake
  2. 3
      python/google/protobuf/message.py
  3. 42
      src/google/protobuf/arenaz_sampler.cc
  4. 12
      src/google/protobuf/arenaz_sampler.h
  5. 90
      src/google/protobuf/arenaz_sampler_test.cc
  6. 5
      src/google/protobuf/compiler/cpp/message.cc
  7. 1
      src/google/protobuf/generated_message_reflection.cc
  8. 2
      src/google/protobuf/io/tokenizer.cc
  9. 3
      src/google/protobuf/io/tokenizer_unittest.cc
  10. 4
      src/google/protobuf/message.cc
  11. 3
      src/google/protobuf/message.h
  12. 2
      src/google/protobuf/port_def.inc
  13. 10
      src/google/protobuf/unittest.proto
  14. 8
      src/google/protobuf/util/json_format_proto3.proto
  15. 124
      src/google/protobuf/util/json_util_test.cc

@ -130,7 +130,7 @@ if (MSVC)
/wd4146 # unary minus operator applied to unsigned type, result still unsigned
)
endif()
target_link_libraries(tests protobuf-lite-test-common protobuf-test-common ${protobuf_LIB_PROTOC} ${protobuf_LIB_PROTOBUF_LITE} GTest::gmock_main)
target_link_libraries(tests protobuf-lite-test-common protobuf-test-common ${protobuf_LIB_PROTOC} ${protobuf_LIB_PROTOBUF} GTest::gmock_main)
set(test_plugin_files
${test_plugin_files}

@ -74,7 +74,8 @@ class Message(object):
__slots__ = []
#: The :class:`google.protobuf.descriptor.Descriptor` for this message type.
#: The :class:`google.protobuf.Descriptor`
# for this message type.
DESCRIPTOR = None
def __deepcopy__(self, memo=None):

@ -56,13 +56,20 @@ namespace {
PROTOBUF_CONSTINIT std::atomic<bool> g_arenaz_enabled{true};
PROTOBUF_CONSTINIT std::atomic<int32_t> g_arenaz_sample_parameter{1 << 10};
PROTOBUF_CONSTINIT std::atomic<ThreadSafeArenazConfigListener>
g_arenaz_config_listener{nullptr};
PROTOBUF_THREAD_LOCAL absl::profiling_internal::ExponentialBiased
g_exponential_biased_generator;
void TriggerThreadSafeArenazConfigListener() {
auto* listener = g_arenaz_config_listener.load(std::memory_order_acquire);
if (listener != nullptr) listener();
}
} // namespace
PROTOBUF_THREAD_LOCAL SamplingState global_sampling_state = {
.next_sample = int64_t{1} << 10, .sample_stride = int64_t{1} << 10};
/*next_sample=*/0, /*sample_stride=*/0};
ThreadSafeArenaStats::ThreadSafeArenaStats() { PrepareForSampling(0); }
ThreadSafeArenaStats::~ThreadSafeArenaStats() = default;
@ -118,11 +125,29 @@ ThreadSafeArenaStats* SampleSlow(SamplingState& sampling_state) {
return GlobalThreadSafeArenazSampler().Register(old_stride);
}
void SetThreadSafeArenazConfigListener(ThreadSafeArenazConfigListener l) {
g_arenaz_config_listener.store(l, std::memory_order_release);
}
bool IsThreadSafeArenazEnabled() {
return g_arenaz_enabled.load(std::memory_order_acquire);
}
void SetThreadSafeArenazEnabled(bool enabled) {
SetThreadSafeArenazEnabledInternal(enabled);
TriggerThreadSafeArenazConfigListener();
}
void SetThreadSafeArenazEnabledInternal(bool enabled) {
g_arenaz_enabled.store(enabled, std::memory_order_release);
}
void SetThreadSafeArenazSampleParameter(int32_t rate) {
SetThreadSafeArenazSampleParameterInternal(rate);
TriggerThreadSafeArenazConfigListener();
}
void SetThreadSafeArenazSampleParameterInternal(int32_t rate) {
if (rate > 0) {
g_arenaz_sample_parameter.store(rate, std::memory_order_release);
} else {
@ -136,6 +161,11 @@ int32_t ThreadSafeArenazSampleParameter() {
}
void SetThreadSafeArenazMaxSamples(int32_t max) {
SetThreadSafeArenazMaxSamplesInternal(max);
TriggerThreadSafeArenazConfigListener();
}
void SetThreadSafeArenazMaxSamplesInternal(int32_t max) {
if (max > 0) {
GlobalThreadSafeArenazSampler().SetMaxSamples(max);
} else {
@ -144,6 +174,10 @@ void SetThreadSafeArenazMaxSamples(int32_t max) {
}
}
size_t ThreadSafeArenazMaxSamples() {
return GlobalThreadSafeArenazSampler().GetMaxSamples();
}
void SetThreadSafeArenazGlobalNextSample(int64_t next_sample) {
if (next_sample >= 0) {
global_sampling_state.next_sample = next_sample;
@ -160,10 +194,16 @@ ThreadSafeArenaStats* SampleSlow(int64_t* next_sample) {
return nullptr;
}
void SetThreadSafeArenazConfigListener(ThreadSafeArenazConfigListener) {}
void SetThreadSafeArenazEnabled(bool enabled) {}
void SetThreadSafeArenazEnabledInternal(bool enabled) {}
bool IsThreadSafeArenazEnabled() { return false; }
void SetThreadSafeArenazSampleParameter(int32_t rate) {}
void SetThreadSafeArenazSampleParameterInternal(int32_t rate) {}
int32_t ThreadSafeArenazSampleParameter() { return 0; }
void SetThreadSafeArenazMaxSamples(int32_t max) {}
void SetThreadSafeArenazMaxSamplesInternal(int32_t max) {}
size_t ThreadSafeArenazMaxSamples() { return 0; }
void SetThreadSafeArenazGlobalNextSample(int64_t next_sample) {}
#endif // defined(PROTOBUF_ARENAZ_SAMPLE)

@ -199,17 +199,29 @@ inline ThreadSafeArenaStatsHandle Sample() {
// Returns a global Sampler.
ThreadSafeArenazSampler& GlobalThreadSafeArenazSampler();
using ThreadSafeArenazConfigListener = void (*)();
void SetThreadSafeArenazConfigListener(ThreadSafeArenazConfigListener l);
// Enables or disables sampling for thread safe arenas.
void SetThreadSafeArenazEnabled(bool enabled);
void SetThreadSafeArenazEnabledInternal(bool enabled);
// Returns true if sampling is on, false otherwise.
bool IsThreadSafeArenazEnabled();
// Sets the rate at which thread safe arena will be sampled.
void SetThreadSafeArenazSampleParameter(int32_t rate);
void SetThreadSafeArenazSampleParameterInternal(int32_t rate);
// Returns the rate at which thread safe arena will be sampled.
int32_t ThreadSafeArenazSampleParameter();
// Sets a soft max for the number of samples that will be kept.
void SetThreadSafeArenazMaxSamples(int32_t max);
void SetThreadSafeArenazMaxSamplesInternal(int32_t max);
// Returns the max number of samples that will be kept.
size_t ThreadSafeArenazMaxSamples();
// Sets the current value for when arenas should be next sampled.
void SetThreadSafeArenazGlobalNextSample(int64_t next_sample);

@ -375,6 +375,7 @@ TEST(ThreadSafeArenazSamplerTest, MultiThread) {
SetThreadSafeArenazEnabled(true);
// Setting 1 as the parameter value means one in every two arenas would be
// sampled, on average.
int32_t oldparam = ThreadSafeArenazSampleParameter();
SetThreadSafeArenazSampleParameter(1);
SetThreadSafeArenazGlobalNextSample(0);
auto& sampler = GlobalThreadSafeArenazSampler();
@ -402,6 +403,95 @@ TEST(ThreadSafeArenazSamplerTest, MultiThread) {
}
}
EXPECT_GT(count, 0);
SetThreadSafeArenazSampleParameter(oldparam);
}
class SampleFirstArenaThread : public Thread {
protected:
void Run() override {
google::protobuf::Arena arena;
google::protobuf::ArenaSafeUniquePtr<
protobuf_test_messages::proto2::TestAllTypesProto2>
message = google::protobuf::MakeArenaSafeUnique<
protobuf_test_messages::proto2::TestAllTypesProto2>(&arena);
GOOGLE_CHECK(message != nullptr);
arena_created_.Notify();
samples_counted_.WaitForNotification();
}
public:
explicit SampleFirstArenaThread(const thread::Options& options)
: Thread(options, "SampleFirstArenaThread") {}
absl::Notification arena_created_;
absl::Notification samples_counted_;
};
// Test that the first arena created on a thread may and may not be chosen for
// sampling.
TEST(ThreadSafeArenazSamplerTest, SampleFirstArena) {
SetThreadSafeArenazEnabled(true);
auto& sampler = GlobalThreadSafeArenazSampler();
enum class SampleResult {
kSampled,
kUnsampled,
kSpoiled,
};
auto count_samples = [&]() {
int count = 0;
sampler.Iterate([&](const ThreadSafeArenaStats& h) { ++count; });
return count;
};
auto run_sample_experiment = [&]() {
int before = count_samples();
thread::Options options;
options.set_joinable(true);
SampleFirstArenaThread t(options);
t.Start();
t.arena_created_.WaitForNotification();
int during = count_samples();
t.samples_counted_.Notify();
t.Join();
int after = count_samples();
// If we didn't get back where we were, some other thread may have
// created an arena and produced an invalid experiment run.
if (before != after) return SampleResult::kSpoiled;
switch (during - before) {
case 1:
return SampleResult::kSampled;
case 0:
return SampleResult::kUnsampled;
default:
return SampleResult::kSpoiled;
}
};
constexpr int kTrials = 10000;
bool sampled = false;
bool unsampled = false;
for (int i = 0; i < kTrials; ++i) {
switch (run_sample_experiment()) {
case SampleResult::kSampled:
sampled = true;
break;
case SampleResult::kUnsampled:
unsampled = true;
break;
default:
break;
}
// This is the success criteria for the entire test. At some point
// we sampled the first arena and at some point we did not.
if (sampled && unsampled) return;
}
EXPECT_TRUE(sampled);
EXPECT_TRUE(unsampled);
}
#endif // defined(PROTOBUF_ARENAZ_SAMPLE)

@ -2205,11 +2205,12 @@ void MessageGenerator::GenerateClassMethods(io::Printer* printer) {
" if (IsSplitMessageDefault()) {\n"
" void* chunk = "
"::PROTOBUF_NAMESPACE_ID::internal::CreateSplitMessageGeneric("
"GetArenaForAllocation(), &$1$, sizeof(Impl_::Split));\n"
"GetArenaForAllocation(), &$1$, sizeof(Impl_::Split), this, &$2$);\n"
" $split$ = reinterpret_cast<Impl_::Split*>(chunk);\n"
" }\n"
"}\n",
DefaultInstanceName(descriptor_, options_, /*split=*/true));
DefaultInstanceName(descriptor_, options_, /*split=*/true),
DefaultInstanceName(descriptor_, options_, /*split=*/false));
}
GenerateVerify(printer);

@ -2514,6 +2514,7 @@ const Type& Reflection::GetRawNonOneof(const Message& message,
}
void Reflection::PrepareSplitMessageForWrite(Message* message) const {
GOOGLE_DCHECK_NE(message, schema_.default_instance_);
void** split = MutableSplitField(message);
const void* default_split = GetSplitField(schema_.default_instance_);
if (*split == default_split) {

@ -406,7 +406,7 @@ void Tokenizer::ConsumeString(char delimiter) {
case '\n': {
if (!allow_multiline_strings_) {
AddError("String literals cannot cross line boundaries.");
AddError("Multiline strings are not allowed. Did you miss a \"?.");
return;
}
NextChar();

@ -1067,7 +1067,8 @@ ErrorCase kErrorCases[] = {
{"'\\X' foo", true, "0:2: Invalid escape sequence in string literal.\n"},
{"'\\x' foo", true, "0:3: Expected hex digits for escape sequence.\n"},
{"'foo", false, "0:4: Unexpected end of string.\n"},
{"'bar\nfoo", true, "0:4: String literals cannot cross line boundaries.\n"},
{"'bar\nfoo", true,
"0:4: Multiline strings are not allowed. Did you miss a \"?.\n"},
{"'\\u01' foo", true,
"0:5: Expected four hex digits for \\u escape sequence.\n"},
{"'\\u01' foo", true,

@ -215,7 +215,9 @@ uint64_t Message::GetInvariantPerBuild(uint64_t salt) {
namespace internal {
void* CreateSplitMessageGeneric(Arena* arena, const void* default_split,
size_t size) {
size_t size, const void* message,
const void* default_message) {
GOOGLE_DCHECK_NE(message, default_message);
void* split =
(arena == nullptr) ? ::operator new(size) : arena->AllocateAligned(size);
memcpy(split, default_split, size);

@ -412,7 +412,8 @@ class PROTOBUF_EXPORT Message : public MessageLite {
namespace internal {
// Creates and returns an allocation for a split message.
void* CreateSplitMessageGeneric(Arena* arena, const void* default_split,
size_t size);
size_t size, const void* message,
const void* default_message);
// Forward-declare interfaces used to implement RepeatedFieldRef.
// These are protobuf internals that users shouldn't care about.

@ -699,7 +699,7 @@
// uses Clang 12.0.5.
# elif !defined(__CYGWIN__) && \
__has_cpp_attribute(clang::require_constant_initialization) && \
((defined(__APPLE__) && PROTOBUF_CLANG_MIN(13, 0)) || \
((defined(__APPLE__) && PROTOBUF_CLANG_MIN(13, 0)) || \
(!defined(__APPLE__) && PROTOBUF_CLANG_MIN(12, 0)))
# define PROTOBUF_CONSTINIT [[clang::require_constant_initialization]]
# define PROTOBUF_CONSTEXPR constexpr

@ -346,6 +346,16 @@ extend TestAllExtensions {
optional bytes oneof_bytes_extension = 114;
}
message TestMixedFieldsAndExtensions {
optional int32 a = 1;
repeated fixed32 b = 3;
extensions 2, 4;
extend TestMixedFieldsAndExtensions {
optional int32 c = 2;
repeated fixed32 d = 4;
}
}
message TestGroup {
optional group OptionalGroup = 16 {
optional int32 a = 17;

@ -189,6 +189,14 @@ message TestCustomJsonName {
int32 value = 1 [json_name = "@value"];
}
message TestEvilJson {
int32 regular_value = 1 [json_name = "regular_name"];
int32 script = 2 [json_name = "</script>"];
int32 quotes = 3 [json_name = "unbalanced\"quotes"];
int32 script_and_quotes = 4
[json_name = "\"<script>alert('hello!);</script>"];
}
message TestExtensions {
.protobuf_unittest.TestAllExtensions extensions = 1;
}

@ -64,6 +64,11 @@
// Must be included last.
#include <google/protobuf/port_def.inc>
bool IsJson2() {
// Pay no attention to the person behind the curtain.
return false;
}
namespace google {
namespace protobuf {
namespace util {
@ -76,6 +81,7 @@ using ::proto3::TestOneof;
using ::proto3::TestWrapper;
using ::proto_util_converter::testing::MapIn;
using ::testing::ElementsAre;
using ::testing::Not;
using ::testing::SizeIs;
// TODO(b/234474291): Use the gtest versions once that's available in OSS.
@ -274,20 +280,37 @@ TEST_P(JsonTest, TestDefaultValues) {
// The ESF parser actually gets this wrong, and serializes floats whose
// default value is non-finite as 0. We make sure to reproduce this bug.
EXPECT_THAT(
ToJson(protobuf_unittest::TestExtremeDefaultValues(), options),
IsOkAndHolds(
R"({"escapedBytes":"XDAwMFwwMDFcMDA3XDAxMFwwMTRcblxyXHRcMDEzXFxcJ1wiXDM3Ng==")"
R"(,"largeUint32":4294967295,"largeUint64":"18446744073709551615",)"
R"("smallInt32":-2147483647,"smallInt64":"-9223372036854775807")"
R"(,"reallySmallInt32":-2147483648,"reallySmallInt64":"-9223372036854775808",)"
R"("utf8String":"","zeroFloat":0,"oneFloat":1,"smallFloat":1.5,)"
R"("negativeOneFloat":-1,"negativeFloat":-1.5,"largeFloat":2e+08,)"
R"("smallNegativeFloat":-8e-28,"infDouble":0,"negInfDouble":0)"
R"(,"nanDouble":0,"infFloat":0,"negInfFloat":0,"nanFloat":0)"
R"(,"cppTrigraph":"? ? ?? ?? ??? ??/ ??-","stringWithZero":"hel\u0000lo")"
R"(,"bytesWithZero":"d29yXDAwMGxk","stringPieceWithZero":"ab\u0000c")"
R"(,"cordWithZero":"12\u00003","replacementString":"${unknown}"})"));
if (IsJson2()) {
EXPECT_THAT(
ToJson(protobuf_unittest::TestExtremeDefaultValues(), options),
IsOkAndHolds(
R"({"escapedBytes":"XDAwMFwwMDFcMDA3XDAxMFwwMTRcblxyXHRcMDEzXFxcJ1wiXDM3Ng==")"
R"(,"largeUint32":4294967295,"largeUint64":"18446744073709551615",)"
R"("smallInt32":-2147483647,"smallInt64":"-9223372036854775807",)"
R"("utf8String":"","zeroFloat":0,"oneFloat":1,"smallFloat":1.5,)"
R"("negativeOneFloat":-1,"negativeFloat":-1.5,"largeFloat":2e+08,)"
R"("smallNegativeFloat":-8e-28,"infDouble":0,"negInfDouble":0,)"
R"("nanDouble":0,"infFloat":0,"negInfFloat":0,"nanFloat":0,)"
R"("cppTrigraph":"? ? ?? ?? ??? ??/ ??-","reallySmallInt32":-2147483648)"
R"(,"reallySmallInt64":"-9223372036854775808","stringWithZero":"hel\u0000lo")"
R"(,"bytesWithZero":"d29yXDAwMGxk","stringPieceWithZero":"ab\u0000c")"
R"(,"cordWithZero":"12\u00003","replacementString":"${unknown}"})"));
} else {
EXPECT_THAT(
ToJson(protobuf_unittest::TestExtremeDefaultValues(), options),
IsOkAndHolds(
R"({"escapedBytes":"XDAwMFwwMDFcMDA3XDAxMFwwMTRcblxyXHRcMDEzXFxcJ1wiXDM3Ng==")"
R"(,"largeUint32":4294967295,"largeUint64":"18446744073709551615",)"
R"("smallInt32":-2147483647,"smallInt64":"-9223372036854775807")"
R"(,"reallySmallInt32":-2147483648,"reallySmallInt64":"-9223372036854775808",)"
R"("utf8String":"","zeroFloat":0,"oneFloat":1,"smallFloat":1.5,)"
R"("negativeOneFloat":-1,"negativeFloat":-1.5,"largeFloat":2e+08,)"
R"("smallNegativeFloat":-8e-28,"infDouble":0,"negInfDouble":0)"
R"(,"nanDouble":0,"infFloat":0,"negInfFloat":0,"nanFloat":0)"
R"(,"cppTrigraph":"? ? ?? ?? ??? ??/ ??-","stringWithZero":"hel\u0000lo")"
R"(,"bytesWithZero":"d29yXDAwMGxk","stringPieceWithZero":"ab\u0000c")"
R"(,"cordWithZero":"12\u00003","replacementString":"${unknown}"})"));
}
}
TEST_P(JsonTest, TestPreserveProtoFieldNames) {
@ -762,6 +785,20 @@ TEST_P(JsonTest, TestParsingBrokenAny) {
}
)json"),
StatusIs(util::StatusCode::kInvalidArgument));
TestAny m2;
m2.mutable_value();
EXPECT_THAT(ToJson(m2), IsOkAndHolds(R"({"value":{}})"));
m2.mutable_value()->set_value("garbage");
// The ESF parser does not return InvalidArgument for this error.
EXPECT_THAT(ToJson(m2), Not(StatusIs(util::StatusCode::kOk)));
m2.Clear();
m2.mutable_value()->set_type_url("type.googleapis.com/proto3.TestMessage");
EXPECT_THAT(
ToJson(m2),
IsOkAndHolds(
R"({"value":{"@type":"type.googleapis.com/proto3.TestMessage"}})"));
}
TEST_P(JsonTest, TestFlatList) {
@ -952,6 +989,37 @@ TEST_P(JsonTest, TestParsingEnumIgnoreCase) {
EXPECT_EQ(m.enum_value(), proto3::BAR);
}
// This functionality is not correctly implemented by the ESF parser, so
// the test is only turned on when testing json2.
TEST_P(JsonTest, Extensions) {
if (GetParam() == Codec::kResolver || !IsJson2()) {
GTEST_SKIP();
}
auto m = ToProto<protobuf_unittest::TestMixedFieldsAndExtensions>(R"json({
"[protobuf_unittest.TestMixedFieldsAndExtensions.c]": 42,
"a": 5,
"b": [1, 2, 3],
"[protobuf_unittest.TestMixedFieldsAndExtensions.d]": [1, 1, 2, 3, 5, 8, 13]
})json");
ASSERT_OK(m);
EXPECT_EQ(m->a(), 5);
EXPECT_THAT(m->b(), ElementsAre(1, 2, 3));
EXPECT_EQ(m->GetExtension(protobuf_unittest::TestMixedFieldsAndExtensions::c),
42);
EXPECT_THAT(
m->GetRepeatedExtension(protobuf_unittest::TestMixedFieldsAndExtensions::d),
ElementsAre(1, 1, 2, 3, 5, 8, 13));
EXPECT_THAT(
ToJson(*m),
IsOkAndHolds(
R"({"a":5,)"
R"("[protobuf_unittest.TestMixedFieldsAndExtensions.c]":42,)"
R"("b":[1,2,3],)"
R"("[protobuf_unittest.TestMixedFieldsAndExtensions.d]":[1,1,2,3,5,8,13]})"));
}
// Parsing does NOT work like MergeFrom: existing repeated field values are
// clobbered, not appended to.
TEST_P(JsonTest, TestOverwriteRepeated) {
@ -1158,6 +1226,34 @@ TEST_P(JsonTest, HtmlEscape) {
m.set_string_value("</script>");
EXPECT_THAT(ToJson(m),
IsOkAndHolds(R"({"stringValue":"\u003c/script\u003e"})"));
proto3::TestEvilJson m2;
JsonPrintOptions opts;
opts.always_print_primitive_fields = true;
EXPECT_THAT(
ToJson(m2, opts),
IsOkAndHolds(
R"({"regular_name":0,"\u003c/script\u003e":0,)"
R"("unbalanced\"quotes":0,)"
R"("\"\u003cscript\u003ealert('hello!);\u003c/script\u003e":0})"));
}
TEST_P(JsonTest, FieldOrder) {
// $ protoscope -s <<< "3: 3 22: 2 1: 1 22: 2"
std::string out;
util::Status s = BinaryToJsonString(
resolver_.get(), "type.googleapis.com/proto3.TestMessage",
"\x18\x03\xb0\x01\x02\x08\x01\xb0\x01\x02", &out);
ASSERT_OK(s);
if (IsJson2()) {
EXPECT_EQ(
out,
R"({"boolValue":true,"int64Value":"3","repeatedInt32Value":[2,2]})");
} else {
EXPECT_EQ(
out,
R"({"int64Value":"3","repeatedInt32Value":[2],"boolValue":true,"repeatedInt32Value":[2]})");
}
}
} // namespace

Loading…
Cancel
Save