Merge pull request #12097 from protocolbuffers/cherrypicks

Cherrypick fixes in preparation for 22.1
pull/12099/head
deannagarcia 2 years ago committed by GitHub
commit ec1a192689
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 7
      .bazelrc
  2. 10
      .github/workflows/test_cpp.yml
  3. 2
      .github/workflows/test_csharp.yml
  4. 2
      .github/workflows/test_java.yml
  5. 4
      .github/workflows/test_php.yml
  6. 4
      .github/workflows/test_python.yml
  7. 5
      .github/workflows/test_ruby.yml
  8. 18
      BUILD.bazel
  9. 2
      CMakeLists.txt
  10. 7
      build_defs/cpp_opts.bzl
  11. 8
      ci/common.bazelrc
  12. 2
      pkg/BUILD.bazel
  13. 4
      src/google/protobuf/compiler/BUILD.bazel
  14. 3
      src/google/protobuf/io/tokenizer.cc
  15. 48
      src/google/protobuf/io/tokenizer_unittest.cc

@ -17,13 +17,6 @@ build:msan --copt=-fsanitize-memory-use-after-dtor
build:msan --action_env=MSAN_OPTIONS=poison_in_dtor=1
build:msan --copt=-DMEMORY_SANITIZER=1
# Use our instrumented LLVM libc++ in Kokoro.
build:kokoro-msan --config=msan
build:kokoro-msan --linkopt=-L/opt/libcxx_msan/lib
build:kokoro-msan --linkopt=-Wl,-rpath,/opt/libcxx_msan/lib
build:kokoro-msan --cxxopt=-stdlib=libc++ --linkopt=-stdlib=libc++
build:tsan --config=san-common --copt=-fsanitize=thread --linkopt=-fsanitize=thread
build:tsan --copt=-DTHREAD_SANITIZER=1

@ -17,25 +17,25 @@ jobs:
- { name: Optimized, flags: --config=opt }
- { name: Debug, flags: --config=dbg }
- { name: ASAN, flags: --config=asan }
- { name: MSAN, flags: --config=kokoro-msan }
- { name: MSAN, flags: --config=docker-msan }
- { name: TSAN, flags: --config=tsan }
- { name: UBSAN, flags: --config=ubsan }
- { name: No-RTTI, flags: --cxxopt=-fno-rtti }
include:
# Set defaults
- image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize@sha256:dbd2f15fb69734d72c3fd10cb819bbe2ce4890acf49e9a2f9403983fe48e8807
- image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize@sha256:309dae3122031447d714414814d262e5f31cb93c0e248e9c02f9d8cdafd7e3b9
- targets: //pkg/... //src/... @com_google_protobuf_examples//...
# Override cases with custom images
- config: { name: "TCMalloc" }
image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/tcmalloc@sha256:9d975616c3fd44d5a091aeb60ee94f37e22fb367d471d258fc18cb4a2387c943"
image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/tcmalloc@sha256:4df3b4749e787ba0a671ec0b783d0f1ba05f60be4c9e9fd72c875550a0cde1ea"
targets: "//src/..."
- config: { name: "aarch64" }
targets: "//src/... //src/google/protobuf/compiler:protoc_aarch64_test"
image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-e863f8ec6b1dfe41f7dc573bac9c8072a0a68b1b"
image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-508417e5215994ade7585d28ba3aad681a25fa5d"
- config: { name: "Bazel4" }
targets: "//src/..."
image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:4.2.3-3b71de326b62f67bf754c4dc4016d6a2fa9dd664"
image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:4.2.3-6361b3a6e5c97e9951d03a4de28542fc45f1adab"
name: Linux ${{ matrix.config.name }}
runs-on: ubuntu-latest
steps:

@ -20,7 +20,7 @@ jobs:
- name: Run tests
uses: ./.github/actions/bazel-docker
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/csharp:3.1.415-6.0.100-65526ea124d1034eac33e7c37cc6d65c5bef054f
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/csharp:3.1.415-6.0.100-508417e5215994ade7585d28ba3aad681a25fa5d
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: csharp_linux
bazel: test //csharp/... --action_env=DOTNET_CLI_TELEMETRY_OPTOUT=1 --test_env=DOTNET_CLI_HOME=/home/bazel

@ -28,7 +28,7 @@ jobs:
targets: //java/... //java/internal:java_version
- name: aarch64
version: 'aarch64'
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-e863f8ec6b1dfe41f7dc573bac9c8072a0a68b1b
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-508417e5215994ade7585d28ba3aad681a25fa5d
targets: //java/... //src/google/protobuf/compiler:protoc_aarch64_test
name: Linux ${{ matrix.name }}

@ -45,7 +45,7 @@ jobs:
- name: Run tests
uses: ./.github/actions/docker
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-6e95c0e221e4bd52e3b4dc1398c6336985196931
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-508417e5215994ade7585d28ba3aad681a25fa5d
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
command: ${{ matrix.command }}
@ -80,6 +80,7 @@ jobs:
id: cross-compile
uses: ./.github/actions/cross-compile-protoc
with:
image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-6361b3a6e5c97e9951d03a4de28542fc45f1adab
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
architecture: linux-i386
@ -111,6 +112,7 @@ jobs:
id: cross-compile
uses: ./.github/actions/cross-compile-protoc
with:
image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-6361b3a6e5c97e9951d03a4de28542fc45f1adab
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
architecture: linux-aarch64

@ -27,7 +27,7 @@ jobs:
targets: //python/... //python:aarch64_test
# TODO(b/262628111) Enable this once conformance tests are fixed.
flags: --define=use_fast_cpp_protos=true --test_tag_filters=-conformance
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-e863f8ec6b1dfe41f7dc573bac9c8072a0a68b1b
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-508417e5215994ade7585d28ba3aad681a25fa5d
name: Linux ${{ matrix.type }} ${{ matrix.version }}
runs-on: ubuntu-latest
@ -39,7 +39,7 @@ jobs:
- name: Run tests
uses: ./.github/actions/bazel-docker
with:
image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/python:{0}-65526ea124d1034eac33e7c37cc6d65c5bef054f', matrix.version) }}
image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/python:{0}-508417e5215994ade7585d28ba3aad681a25fa5d', matrix.version) }}
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: python_linux/${{ matrix.type }}_${{ matrix.version }}
bazel: test ${{ matrix.targets }} ${{ matrix.flags }} --test_env=KOKORO_PYTHON_VERSION

@ -34,7 +34,7 @@ jobs:
- name: Run tests
uses: ./.github/actions/bazel-docker
with:
image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:{0}-{1}-75e79f791b96e056086f43ace729cf3ebf9a9f5d', matrix.ruby, matrix.bazel) }}
image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:{0}-{1}-508417e5215994ade7585d28ba3aad681a25fa5d', matrix.ruby, matrix.bazel) }}
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: ruby_linux/${{ matrix.ruby }}_${{ matrix.bazel }}
bazel: test //ruby/... //ruby/tests:ruby_version --test_env=KOKORO_RUBY_VERSION
@ -52,6 +52,7 @@ jobs:
id: cross-compile
uses: ./.github/actions/cross-compile-protoc
with:
image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-6361b3a6e5c97e9951d03a4de28542fc45f1adab
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
architecture: linux-aarch64
@ -125,7 +126,7 @@ jobs:
- name: Run tests
uses: ./.github/actions/bazel-docker
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:${{ matrix.ruby }}-${{ matrix.bazel }}-75e79f791b96e056086f43ace729cf3ebf9a9f5d
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:${{ matrix.ruby }}-${{ matrix.bazel }}-508417e5215994ade7585d28ba3aad681a25fa5d
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: ruby_install/${{ matrix.ruby }}_${{ matrix.bazel }}
bash: >

@ -4,7 +4,7 @@ load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_proto_library")
load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix")
load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain", "proto_library")
load("@rules_java//java:defs.bzl", "java_lite_proto_library", "java_proto_library")
load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS", "PROTOC_LINK_OPTS")
load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS")
load(":protobuf.bzl", "internal_objc_proto_library", "internal_php_proto_library", "internal_py_proto_library", "internal_ruby_proto_library")
licenses(["notice"])
@ -168,7 +168,21 @@ alias(
cc_binary(
name = "protoc",
copts = COPTS,
linkopts = LINK_OPTS + PROTOC_LINK_OPTS,
linkopts = LINK_OPTS,
visibility = ["//visibility:public"],
deps = ["//src/google/protobuf/compiler:protoc_lib"],
)
cc_binary(
name = "protoc_static",
copts = COPTS,
linkopts = LINK_OPTS,
features = select({
# This isn't possible on mac because there is no static library for lcrt0.o
"@platforms//os:osx": [],
# When cross-compiling we need to statically link all C++ libraries.
"//conditions:default": ["fully_static_link"],
}),
visibility = ["//visibility:public"],
deps = ["//src/google/protobuf/compiler:protoc_lib"],
)

@ -125,7 +125,7 @@ message(STATUS "${protobuf_VERSION_PRERELEASE}")
# Package version
set(protobuf_VERSION
"${protobuf_VERSION_MAJOR}.${protobuf_VERSION_MINOR}.${protobuf_VERSION_PATCH}")
"${protobuf_VERSION_MINOR}.${protobuf_VERSION_PATCH}")
if(protobuf_VERSION_PRERELEASE)
set(protobuf_VERSION "${protobuf_VERSION}.${protobuf_VERSION_PRERELEASE}")

@ -41,10 +41,3 @@ LINK_OPTS = select({
"-lm",
],
})
# When cross-compiling for Windows we need to statically link pthread and the C++ library.
PROTOC_LINK_OPTS = select({
"//build_defs:config_win32": ["-static"],
"//build_defs:config_win64": ["-static"],
"//conditions:default": [],
})

@ -19,10 +19,10 @@ build:msan --action_env=MSAN_OPTIONS=poison_in_dtor=1
build:msan --copt=-DMEMORY_SANITIZER=1
# Use our instrumented LLVM libc++ in Kokoro.
build:kokoro-msan --config=msan
build:kokoro-msan --linkopt=-L/opt/libcxx_msan/lib
build:kokoro-msan --linkopt=-Wl,-rpath,/opt/libcxx_msan/lib
build:kokoro-msan --cxxopt=-stdlib=libc++ --linkopt=-stdlib=libc++
build:docker-msan --config=msan
build:docker-msan --linkopt=-L/opt/libcxx_msan/lib --linkopt=-lc++abi
build:docker-msan --linkopt=-Wl,-rpath,/opt/libcxx_msan/lib
build:docker-msan --cxxopt=-stdlib=libc++ --linkopt=-stdlib=libc++
build:tsan --config=san-common --copt=-fsanitize=thread --linkopt=-fsanitize=thread

@ -59,7 +59,7 @@ Please refer to our official github site for more installation instructions:
pkg_files(
name = "protoc_files",
srcs = ["//:protoc"],
srcs = ["//:protoc_static"],
attributes = pkg_attributes(mode = "0555"),
prefix = "bin/",
visibility = ["//visibility:private"],

@ -11,7 +11,7 @@ load(
)
load("@rules_proto//proto:defs.bzl", "proto_library")
load("//build_defs:arch_tests.bzl", "aarch64_test", "x86_64_test")
load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS", "PROTOC_LINK_OPTS")
load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS")
proto_library(
name = "plugin_proto",
@ -123,7 +123,7 @@ cc_library(
cc_binary(
name = "protoc_nowkt",
copts = COPTS,
linkopts = LINK_OPTS + PROTOC_LINK_OPTS,
linkopts = LINK_OPTS,
visibility = [
"//src/google/protobuf:__pkg__",
],

@ -942,7 +942,8 @@ bool Tokenizer::NextWithComments(std::string* prev_trailing_comments,
// makes no sense to attach a comment to the following token.
collector.Flush();
}
if (prev_line == line_ || trailing_comment_end_line == line_) {
if (result &&
(prev_line == line_ || trailing_comment_end_line == line_)) {
// When previous token and this one are on the same line, or
// even if a multi-line trailing comment ends on the same line
// as this token, it's unclear to what token the comment

@ -643,6 +643,7 @@ inline std::ostream& operator<<(std::ostream& out,
return out << absl::CEscape(test_case.input);
}
// clang-format off
DocCommentCase kDocCommentCases[] = {
{"prev next",
@ -650,6 +651,18 @@ DocCommentCase kDocCommentCases[] = {
{},
""},
{"prev // no next token\n",
" no next token\n",
{},
""},
{"prev // no next token and no trailing newline",
" no next token and no trailing newline",
{},
""},
{"prev /* detached */ next",
"",
@ -780,7 +793,7 @@ DocCommentCase kDocCommentCases[] = {
prev /* a single block comment
that spans multiple lines
is detached if it ends
on the same line as next */ next"
on the same line as next */ next
)pb",
"",
@ -791,7 +804,7 @@ DocCommentCase kDocCommentCases[] = {
""},
{R"pb(
prev /* trailing */ /* leading */ next"
prev /* trailing */ /* leading */ next
)pb",
" trailing ",
@ -802,13 +815,26 @@ DocCommentCase kDocCommentCases[] = {
prev /* multi-line
trailing */ /* an oddly
placed detached */ /* an oddly
placed leading */ next"
placed leading */ next
)pb",
" multi-line\ntrailing ",
{" an oddly\nplaced detached "},
" an oddly\nplaced leading "},
{R"pb(
prev // trailing with newline
// detached
/* another detached */
// leading but no next token to attach it to
)pb",
" trailing with newline\n",
{" detached\n", " another detached ",
" leading but no next token to attach it to\n"},
""},
};
// clang-format on
TEST_2D(TokenizerTest, DocComments, kDocCommentCases, kBlockSizes) {
// Set up the tokenizer.
@ -822,8 +848,8 @@ TEST_2D(TokenizerTest, DocComments, kDocCommentCases, kBlockSizes) {
kDocCommentCases_case.input.size(), kBlockSizes_case);
Tokenizer tokenizer2(&input2, &error_collector);
tokenizer.Next();
tokenizer2.Next();
EXPECT_TRUE(tokenizer.Next());
EXPECT_TRUE(tokenizer2.Next());
EXPECT_EQ("prev", tokenizer.current().text);
EXPECT_EQ("prev", tokenizer2.current().text);
@ -831,11 +857,13 @@ TEST_2D(TokenizerTest, DocComments, kDocCommentCases, kBlockSizes) {
std::string prev_trailing_comments;
std::vector<std::string> detached_comments;
std::string next_leading_comments;
tokenizer.NextWithComments(&prev_trailing_comments, &detached_comments,
&next_leading_comments);
tokenizer2.NextWithComments(NULL, NULL, NULL);
EXPECT_EQ("next", tokenizer.current().text);
EXPECT_EQ("next", tokenizer2.current().text);
bool has_next = tokenizer.NextWithComments(
&prev_trailing_comments, &detached_comments, &next_leading_comments);
EXPECT_EQ(has_next, tokenizer2.NextWithComments(nullptr, nullptr, nullptr));
if (has_next) {
EXPECT_EQ("next", tokenizer.current().text);
EXPECT_EQ("next", tokenizer2.current().text);
}
EXPECT_EQ(kDocCommentCases_case.prev_trailing_comments,
prev_trailing_comments);

Loading…
Cancel
Save