diff --git a/.bazelrc b/.bazelrc index de30d6b48c..fb29fe10d9 100644 --- a/.bazelrc +++ b/.bazelrc @@ -17,13 +17,6 @@ build:msan --copt=-fsanitize-memory-use-after-dtor build:msan --action_env=MSAN_OPTIONS=poison_in_dtor=1 build:msan --copt=-DMEMORY_SANITIZER=1 -# Use our instrumented LLVM libc++ in Kokoro. -build:kokoro-msan --config=msan -build:kokoro-msan --linkopt=-L/opt/libcxx_msan/lib -build:kokoro-msan --linkopt=-Wl,-rpath,/opt/libcxx_msan/lib -build:kokoro-msan --cxxopt=-stdlib=libc++ --linkopt=-stdlib=libc++ - - build:tsan --config=san-common --copt=-fsanitize=thread --linkopt=-fsanitize=thread build:tsan --copt=-DTHREAD_SANITIZER=1 diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index f846d80a8d..d5cf8b8a79 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -17,25 +17,25 @@ jobs: - { name: Optimized, flags: --config=opt } - { name: Debug, flags: --config=dbg } - { name: ASAN, flags: --config=asan } - - { name: MSAN, flags: --config=kokoro-msan } + - { name: MSAN, flags: --config=docker-msan } - { name: TSAN, flags: --config=tsan } - { name: UBSAN, flags: --config=ubsan } - { name: No-RTTI, flags: --cxxopt=-fno-rtti } include: # Set defaults - - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize@sha256:dbd2f15fb69734d72c3fd10cb819bbe2ce4890acf49e9a2f9403983fe48e8807 + - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize@sha256:309dae3122031447d714414814d262e5f31cb93c0e248e9c02f9d8cdafd7e3b9 - targets: //pkg/... //src/... @com_google_protobuf_examples//... # Override cases with custom images - config: { name: "TCMalloc" } - image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/tcmalloc@sha256:9d975616c3fd44d5a091aeb60ee94f37e22fb367d471d258fc18cb4a2387c943" + image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/tcmalloc@sha256:4df3b4749e787ba0a671ec0b783d0f1ba05f60be4c9e9fd72c875550a0cde1ea" targets: "//src/..." - config: { name: "aarch64" } targets: "//src/... //src/google/protobuf/compiler:protoc_aarch64_test" - image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-e863f8ec6b1dfe41f7dc573bac9c8072a0a68b1b" + image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-508417e5215994ade7585d28ba3aad681a25fa5d" - config: { name: "Bazel4" } targets: "//src/..." - image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:4.2.3-3b71de326b62f67bf754c4dc4016d6a2fa9dd664" + image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:4.2.3-6361b3a6e5c97e9951d03a4de28542fc45f1adab" name: Linux ${{ matrix.config.name }} runs-on: ubuntu-latest steps: diff --git a/.github/workflows/test_csharp.yml b/.github/workflows/test_csharp.yml index 022d4f1f6c..1fe79d4641 100644 --- a/.github/workflows/test_csharp.yml +++ b/.github/workflows/test_csharp.yml @@ -20,7 +20,7 @@ jobs: - name: Run tests uses: ./.github/actions/bazel-docker with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/csharp:3.1.415-6.0.100-65526ea124d1034eac33e7c37cc6d65c5bef054f + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/csharp:3.1.415-6.0.100-508417e5215994ade7585d28ba3aad681a25fa5d credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: csharp_linux bazel: test //csharp/... --action_env=DOTNET_CLI_TELEMETRY_OPTOUT=1 --test_env=DOTNET_CLI_HOME=/home/bazel diff --git a/.github/workflows/test_java.yml b/.github/workflows/test_java.yml index b415fdef4a..34347a9554 100644 --- a/.github/workflows/test_java.yml +++ b/.github/workflows/test_java.yml @@ -28,7 +28,7 @@ jobs: targets: //java/... //java/internal:java_version - name: aarch64 version: 'aarch64' - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-e863f8ec6b1dfe41f7dc573bac9c8072a0a68b1b + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-508417e5215994ade7585d28ba3aad681a25fa5d targets: //java/... //src/google/protobuf/compiler:protoc_aarch64_test name: Linux ${{ matrix.name }} diff --git a/.github/workflows/test_php.yml b/.github/workflows/test_php.yml index 2d93d8f2b2..fda266fb2a 100644 --- a/.github/workflows/test_php.yml +++ b/.github/workflows/test_php.yml @@ -45,7 +45,7 @@ jobs: - name: Run tests uses: ./.github/actions/docker with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-6e95c0e221e4bd52e3b4dc1398c6336985196931 + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-508417e5215994ade7585d28ba3aad681a25fa5d credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: ${{ matrix.command }} @@ -80,6 +80,7 @@ jobs: id: cross-compile uses: ./.github/actions/cross-compile-protoc with: + image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-6361b3a6e5c97e9951d03a4de28542fc45f1adab credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} architecture: linux-i386 @@ -111,6 +112,7 @@ jobs: id: cross-compile uses: ./.github/actions/cross-compile-protoc with: + image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-6361b3a6e5c97e9951d03a4de28542fc45f1adab credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} architecture: linux-aarch64 diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml index e7b76a4209..18f29855c8 100644 --- a/.github/workflows/test_python.yml +++ b/.github/workflows/test_python.yml @@ -27,7 +27,7 @@ jobs: targets: //python/... //python:aarch64_test # TODO(b/262628111) Enable this once conformance tests are fixed. flags: --define=use_fast_cpp_protos=true --test_tag_filters=-conformance - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-e863f8ec6b1dfe41f7dc573bac9c8072a0a68b1b + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-508417e5215994ade7585d28ba3aad681a25fa5d name: Linux ${{ matrix.type }} ${{ matrix.version }} runs-on: ubuntu-latest @@ -39,7 +39,7 @@ jobs: - name: Run tests uses: ./.github/actions/bazel-docker with: - image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/python:{0}-65526ea124d1034eac33e7c37cc6d65c5bef054f', matrix.version) }} + image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/python:{0}-508417e5215994ade7585d28ba3aad681a25fa5d', matrix.version) }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: python_linux/${{ matrix.type }}_${{ matrix.version }} bazel: test ${{ matrix.targets }} ${{ matrix.flags }} --test_env=KOKORO_PYTHON_VERSION diff --git a/.github/workflows/test_ruby.yml b/.github/workflows/test_ruby.yml index e22bf8a356..31b8f79b9f 100644 --- a/.github/workflows/test_ruby.yml +++ b/.github/workflows/test_ruby.yml @@ -34,7 +34,7 @@ jobs: - name: Run tests uses: ./.github/actions/bazel-docker with: - image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:{0}-{1}-75e79f791b96e056086f43ace729cf3ebf9a9f5d', matrix.ruby, matrix.bazel) }} + image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:{0}-{1}-508417e5215994ade7585d28ba3aad681a25fa5d', matrix.ruby, matrix.bazel) }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: ruby_linux/${{ matrix.ruby }}_${{ matrix.bazel }} bazel: test //ruby/... //ruby/tests:ruby_version --test_env=KOKORO_RUBY_VERSION @@ -52,6 +52,7 @@ jobs: id: cross-compile uses: ./.github/actions/cross-compile-protoc with: + image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-6361b3a6e5c97e9951d03a4de28542fc45f1adab credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} architecture: linux-aarch64 @@ -125,7 +126,7 @@ jobs: - name: Run tests uses: ./.github/actions/bazel-docker with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:${{ matrix.ruby }}-${{ matrix.bazel }}-75e79f791b96e056086f43ace729cf3ebf9a9f5d + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:${{ matrix.ruby }}-${{ matrix.bazel }}-508417e5215994ade7585d28ba3aad681a25fa5d credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: ruby_install/${{ matrix.ruby }}_${{ matrix.bazel }} bash: > diff --git a/BUILD.bazel b/BUILD.bazel index 69a1ba96c8..bc229ddf67 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -4,7 +4,7 @@ load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_proto_library") load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain", "proto_library") load("@rules_java//java:defs.bzl", "java_lite_proto_library", "java_proto_library") -load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS", "PROTOC_LINK_OPTS") +load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS") load(":protobuf.bzl", "internal_objc_proto_library", "internal_php_proto_library", "internal_py_proto_library", "internal_ruby_proto_library") licenses(["notice"]) @@ -168,7 +168,21 @@ alias( cc_binary( name = "protoc", copts = COPTS, - linkopts = LINK_OPTS + PROTOC_LINK_OPTS, + linkopts = LINK_OPTS, + visibility = ["//visibility:public"], + deps = ["//src/google/protobuf/compiler:protoc_lib"], +) + +cc_binary( + name = "protoc_static", + copts = COPTS, + linkopts = LINK_OPTS, + features = select({ + # This isn't possible on mac because there is no static library for lcrt0.o + "@platforms//os:osx": [], + # When cross-compiling we need to statically link all C++ libraries. + "//conditions:default": ["fully_static_link"], + }), visibility = ["//visibility:public"], deps = ["//src/google/protobuf/compiler:protoc_lib"], ) diff --git a/CMakeLists.txt b/CMakeLists.txt index c71a74d4a1..2c3e002697 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -125,7 +125,7 @@ message(STATUS "${protobuf_VERSION_PRERELEASE}") # Package version set(protobuf_VERSION - "${protobuf_VERSION_MAJOR}.${protobuf_VERSION_MINOR}.${protobuf_VERSION_PATCH}") + "${protobuf_VERSION_MINOR}.${protobuf_VERSION_PATCH}") if(protobuf_VERSION_PRERELEASE) set(protobuf_VERSION "${protobuf_VERSION}.${protobuf_VERSION_PRERELEASE}") diff --git a/build_defs/cpp_opts.bzl b/build_defs/cpp_opts.bzl index 33e157875b..f95c3e2579 100644 --- a/build_defs/cpp_opts.bzl +++ b/build_defs/cpp_opts.bzl @@ -41,10 +41,3 @@ LINK_OPTS = select({ "-lm", ], }) - -# When cross-compiling for Windows we need to statically link pthread and the C++ library. -PROTOC_LINK_OPTS = select({ - "//build_defs:config_win32": ["-static"], - "//build_defs:config_win64": ["-static"], - "//conditions:default": [], -}) diff --git a/ci/common.bazelrc b/ci/common.bazelrc index 421bcc0934..e5345cfd69 100644 --- a/ci/common.bazelrc +++ b/ci/common.bazelrc @@ -19,10 +19,10 @@ build:msan --action_env=MSAN_OPTIONS=poison_in_dtor=1 build:msan --copt=-DMEMORY_SANITIZER=1 # Use our instrumented LLVM libc++ in Kokoro. -build:kokoro-msan --config=msan -build:kokoro-msan --linkopt=-L/opt/libcxx_msan/lib -build:kokoro-msan --linkopt=-Wl,-rpath,/opt/libcxx_msan/lib -build:kokoro-msan --cxxopt=-stdlib=libc++ --linkopt=-stdlib=libc++ +build:docker-msan --config=msan +build:docker-msan --linkopt=-L/opt/libcxx_msan/lib --linkopt=-lc++abi +build:docker-msan --linkopt=-Wl,-rpath,/opt/libcxx_msan/lib +build:docker-msan --cxxopt=-stdlib=libc++ --linkopt=-stdlib=libc++ build:tsan --config=san-common --copt=-fsanitize=thread --linkopt=-fsanitize=thread diff --git a/pkg/BUILD.bazel b/pkg/BUILD.bazel index a3a11399ad..3d871ae90f 100644 --- a/pkg/BUILD.bazel +++ b/pkg/BUILD.bazel @@ -59,7 +59,7 @@ Please refer to our official github site for more installation instructions: pkg_files( name = "protoc_files", - srcs = ["//:protoc"], + srcs = ["//:protoc_static"], attributes = pkg_attributes(mode = "0555"), prefix = "bin/", visibility = ["//visibility:private"], diff --git a/src/google/protobuf/compiler/BUILD.bazel b/src/google/protobuf/compiler/BUILD.bazel index 64081c9a42..197721101a 100644 --- a/src/google/protobuf/compiler/BUILD.bazel +++ b/src/google/protobuf/compiler/BUILD.bazel @@ -11,7 +11,7 @@ load( ) load("@rules_proto//proto:defs.bzl", "proto_library") load("//build_defs:arch_tests.bzl", "aarch64_test", "x86_64_test") -load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS", "PROTOC_LINK_OPTS") +load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS") proto_library( name = "plugin_proto", @@ -123,7 +123,7 @@ cc_library( cc_binary( name = "protoc_nowkt", copts = COPTS, - linkopts = LINK_OPTS + PROTOC_LINK_OPTS, + linkopts = LINK_OPTS, visibility = [ "//src/google/protobuf:__pkg__", ], diff --git a/src/google/protobuf/io/tokenizer.cc b/src/google/protobuf/io/tokenizer.cc index 53229f8e67..d75dfabf19 100644 --- a/src/google/protobuf/io/tokenizer.cc +++ b/src/google/protobuf/io/tokenizer.cc @@ -942,7 +942,8 @@ bool Tokenizer::NextWithComments(std::string* prev_trailing_comments, // makes no sense to attach a comment to the following token. collector.Flush(); } - if (prev_line == line_ || trailing_comment_end_line == line_) { + if (result && + (prev_line == line_ || trailing_comment_end_line == line_)) { // When previous token and this one are on the same line, or // even if a multi-line trailing comment ends on the same line // as this token, it's unclear to what token the comment diff --git a/src/google/protobuf/io/tokenizer_unittest.cc b/src/google/protobuf/io/tokenizer_unittest.cc index 2654acba69..b956c18f39 100644 --- a/src/google/protobuf/io/tokenizer_unittest.cc +++ b/src/google/protobuf/io/tokenizer_unittest.cc @@ -643,6 +643,7 @@ inline std::ostream& operator<<(std::ostream& out, return out << absl::CEscape(test_case.input); } +// clang-format off DocCommentCase kDocCommentCases[] = { {"prev next", @@ -650,6 +651,18 @@ DocCommentCase kDocCommentCases[] = { {}, ""}, + {"prev // no next token\n", + + " no next token\n", + {}, + ""}, + + {"prev // no next token and no trailing newline", + + " no next token and no trailing newline", + {}, + ""}, + {"prev /* detached */ next", "", @@ -780,7 +793,7 @@ DocCommentCase kDocCommentCases[] = { prev /* a single block comment that spans multiple lines is detached if it ends - on the same line as next */ next" + on the same line as next */ next )pb", "", @@ -791,7 +804,7 @@ DocCommentCase kDocCommentCases[] = { ""}, {R"pb( - prev /* trailing */ /* leading */ next" + prev /* trailing */ /* leading */ next )pb", " trailing ", @@ -802,13 +815,26 @@ DocCommentCase kDocCommentCases[] = { prev /* multi-line trailing */ /* an oddly placed detached */ /* an oddly - placed leading */ next" + placed leading */ next )pb", " multi-line\ntrailing ", {" an oddly\nplaced detached "}, " an oddly\nplaced leading "}, + + {R"pb( + prev // trailing with newline + // detached + /* another detached */ + // leading but no next token to attach it to + )pb", + + " trailing with newline\n", + {" detached\n", " another detached ", + " leading but no next token to attach it to\n"}, + ""}, }; +// clang-format on TEST_2D(TokenizerTest, DocComments, kDocCommentCases, kBlockSizes) { // Set up the tokenizer. @@ -822,8 +848,8 @@ TEST_2D(TokenizerTest, DocComments, kDocCommentCases, kBlockSizes) { kDocCommentCases_case.input.size(), kBlockSizes_case); Tokenizer tokenizer2(&input2, &error_collector); - tokenizer.Next(); - tokenizer2.Next(); + EXPECT_TRUE(tokenizer.Next()); + EXPECT_TRUE(tokenizer2.Next()); EXPECT_EQ("prev", tokenizer.current().text); EXPECT_EQ("prev", tokenizer2.current().text); @@ -831,11 +857,13 @@ TEST_2D(TokenizerTest, DocComments, kDocCommentCases, kBlockSizes) { std::string prev_trailing_comments; std::vector detached_comments; std::string next_leading_comments; - tokenizer.NextWithComments(&prev_trailing_comments, &detached_comments, - &next_leading_comments); - tokenizer2.NextWithComments(NULL, NULL, NULL); - EXPECT_EQ("next", tokenizer.current().text); - EXPECT_EQ("next", tokenizer2.current().text); + bool has_next = tokenizer.NextWithComments( + &prev_trailing_comments, &detached_comments, &next_leading_comments); + EXPECT_EQ(has_next, tokenizer2.NextWithComments(nullptr, nullptr, nullptr)); + if (has_next) { + EXPECT_EQ("next", tokenizer.current().text); + EXPECT_EQ("next", tokenizer2.current().text); + } EXPECT_EQ(kDocCommentCases_case.prev_trailing_comments, prev_trailing_comments);