Merge pull request #12096 from deannagarcia/cherrypicsk

Cherrypick changes in preparation for 22.1

.bazelrc

@@ -17,13 +17,6 @@ build:msan --copt=-fsanitize-memory-use-after-dtor
 build:msan --action_env=MSAN_OPTIONS=poison_in_dtor=1
 build:msan --copt=-DMEMORY_SANITIZER=1
 
-# Use our instrumented LLVM libc++ in Kokoro.
-build:kokoro-msan --config=msan
-build:kokoro-msan --linkopt=-L/opt/libcxx_msan/lib
-build:kokoro-msan --linkopt=-Wl,-rpath,/opt/libcxx_msan/lib
-build:kokoro-msan --cxxopt=-stdlib=libc++ --linkopt=-stdlib=libc++
-
-
 build:tsan --config=san-common --copt=-fsanitize=thread --linkopt=-fsanitize=thread
 build:tsan --copt=-DTHREAD_SANITIZER=1
 

.github/workflows/test_cpp.yml

@@ -17,25 +17,25 @@ jobs:
           - { name: Optimized, flags: --config=opt }
           - { name: Debug, flags: --config=dbg }
           - { name: ASAN, flags: --config=asan }
-          - { name: MSAN, flags: --config=kokoro-msan }
+          - { name: MSAN, flags: --config=docker-msan }
           - { name: TSAN, flags: --config=tsan }
           - { name: UBSAN, flags: --config=ubsan }
           - { name: No-RTTI, flags: --cxxopt=-fno-rtti }
         include:
           # Set defaults
-          - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize@sha256:dbd2f15fb69734d72c3fd10cb819bbe2ce4890acf49e9a2f9403983fe48e8807
+          - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize@sha256:309dae3122031447d714414814d262e5f31cb93c0e248e9c02f9d8cdafd7e3b9
           - targets: //pkg/... //src/... @com_google_protobuf_examples//...
 
           # Override cases with custom images
           - config: { name: "TCMalloc" }
-            image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/tcmalloc@sha256:9d975616c3fd44d5a091aeb60ee94f37e22fb367d471d258fc18cb4a2387c943"
+            image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/tcmalloc@sha256:4df3b4749e787ba0a671ec0b783d0f1ba05f60be4c9e9fd72c875550a0cde1ea"
             targets: "//src/..."
           - config: { name: "aarch64" }
             targets: "//src/... //src/google/protobuf/compiler:protoc_aarch64_test"
-            image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-e863f8ec6b1dfe41f7dc573bac9c8072a0a68b1b"
+            image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-508417e5215994ade7585d28ba3aad681a25fa5d"
           - config: { name: "Bazel4" }
             targets: "//src/..."
-            image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:4.2.3-3b71de326b62f67bf754c4dc4016d6a2fa9dd664"
+            image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:4.2.3-6361b3a6e5c97e9951d03a4de28542fc45f1adab"
     name: Linux ${{ matrix.config.name }}
     runs-on: ubuntu-latest
     steps:

.github/workflows/test_csharp.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Run tests
         uses: ./.github/actions/bazel-docker
         with:
-          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/csharp:3.1.415-6.0.100-65526ea124d1034eac33e7c37cc6d65c5bef054f
+          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/csharp:3.1.415-6.0.100-508417e5215994ade7585d28ba3aad681a25fa5d
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: csharp_linux
           bazel: test //csharp/... --action_env=DOTNET_CLI_TELEMETRY_OPTOUT=1 --test_env=DOTNET_CLI_HOME=/home/bazel

.github/workflows/test_java.yml

@@ -28,7 +28,7 @@ jobs:
             targets: //java/... //java/internal:java_version
           - name: aarch64
             version: 'aarch64'
-            image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-e863f8ec6b1dfe41f7dc573bac9c8072a0a68b1b
+            image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-508417e5215994ade7585d28ba3aad681a25fa5d
             targets: //java/... //src/google/protobuf/compiler:protoc_aarch64_test
 
     name: Linux ${{ matrix.name }}

.github/workflows/test_php.yml

@@ -45,7 +45,7 @@ jobs:
       - name: Run tests
         uses: ./.github/actions/docker
         with:
-          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-6e95c0e221e4bd52e3b4dc1398c6336985196931
+          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-508417e5215994ade7585d28ba3aad681a25fa5d
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           command: ${{ matrix.command }}
 
@@ -80,6 +80,7 @@ jobs:
         id: cross-compile
         uses: ./.github/actions/cross-compile-protoc
         with:
+          image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-6361b3a6e5c97e9951d03a4de28542fc45f1adab
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           architecture: linux-i386
 
@@ -111,6 +112,7 @@ jobs:
         id: cross-compile
         uses: ./.github/actions/cross-compile-protoc
         with:
+          image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-6361b3a6e5c97e9951d03a4de28542fc45f1adab
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           architecture: linux-aarch64
 

.github/workflows/test_python.yml

@@ -27,7 +27,7 @@ jobs:
             targets: //python/... //python:aarch64_test
             # TODO(b/262628111) Enable this once conformance tests are fixed.
             flags: --define=use_fast_cpp_protos=true --test_tag_filters=-conformance
-            image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-e863f8ec6b1dfe41f7dc573bac9c8072a0a68b1b
+            image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-508417e5215994ade7585d28ba3aad681a25fa5d
 
     name: Linux ${{ matrix.type }} ${{ matrix.version }}
     runs-on: ubuntu-latest
@@ -39,7 +39,7 @@ jobs:
       - name: Run tests
         uses: ./.github/actions/bazel-docker
         with:
-          image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/python:{0}-65526ea124d1034eac33e7c37cc6d65c5bef054f', matrix.version) }}
+          image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/python:{0}-508417e5215994ade7585d28ba3aad681a25fa5d', matrix.version) }}
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: python_linux/${{ matrix.type }}_${{ matrix.version }}
           bazel: test ${{ matrix.targets }} ${{ matrix.flags }} --test_env=KOKORO_PYTHON_VERSION

.github/workflows/test_ruby.yml

@@ -34,7 +34,7 @@ jobs:
       - name: Run tests
         uses: ./.github/actions/bazel-docker
         with:
-          image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:{0}-{1}-75e79f791b96e056086f43ace729cf3ebf9a9f5d', matrix.ruby, matrix.bazel) }}
+          image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:{0}-{1}-508417e5215994ade7585d28ba3aad681a25fa5d', matrix.ruby, matrix.bazel) }}
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: ruby_linux/${{ matrix.ruby }}_${{ matrix.bazel }}
           bazel: test //ruby/... //ruby/tests:ruby_version --test_env=KOKORO_RUBY_VERSION
@@ -52,6 +52,7 @@ jobs:
         id: cross-compile
         uses: ./.github/actions/cross-compile-protoc
         with:
+          image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-6361b3a6e5c97e9951d03a4de28542fc45f1adab
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           architecture: linux-aarch64
 
@@ -125,7 +126,7 @@ jobs:
       - name: Run tests
         uses: ./.github/actions/bazel-docker
         with:
-          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:${{ matrix.ruby }}-${{ matrix.bazel }}-75e79f791b96e056086f43ace729cf3ebf9a9f5d
+          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:${{ matrix.ruby }}-${{ matrix.bazel }}-508417e5215994ade7585d28ba3aad681a25fa5d
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: ruby_install/${{ matrix.ruby }}_${{ matrix.bazel }}
           bash: >

BUILD.bazel

@@ -4,7 +4,7 @@ load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_proto_library")
 load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix")
 load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain", "proto_library")
 load("@rules_java//java:defs.bzl", "java_lite_proto_library", "java_proto_library")
-load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS", "PROTOC_LINK_OPTS")
+load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS")
 load(":protobuf.bzl", "internal_objc_proto_library", "internal_php_proto_library", "internal_py_proto_library", "internal_ruby_proto_library")
 
 licenses(["notice"])
@@ -168,7 +168,21 @@ alias(
 cc_binary(
     name = "protoc",
     copts = COPTS,
-    linkopts = LINK_OPTS + PROTOC_LINK_OPTS,
+    linkopts = LINK_OPTS,
+    visibility = ["//visibility:public"],
+    deps = ["//src/google/protobuf/compiler:protoc_lib"],
+)
+
+cc_binary(
+    name = "protoc_static",
+    copts = COPTS,
+    linkopts = LINK_OPTS,
+    features = select({
+      # This isn't possible on mac because there is no static library for lcrt0.o
+      "@platforms//os:osx": [],
+      # When cross-compiling we need to statically link all C++ libraries.
+      "//conditions:default": ["fully_static_link"],
+    }),
     visibility = ["//visibility:public"],
     deps = ["//src/google/protobuf/compiler:protoc_lib"],
 )

CMakeLists.txt

@@ -125,7 +125,7 @@ message(STATUS "${protobuf_VERSION_PRERELEASE}")
 
 # Package version
 set(protobuf_VERSION
-  "${protobuf_VERSION_MAJOR}.${protobuf_VERSION_MINOR}.${protobuf_VERSION_PATCH}")
+  "${protobuf_VERSION_MINOR}.${protobuf_VERSION_PATCH}")
 
 if(protobuf_VERSION_PRERELEASE)
   set(protobuf_VERSION "${protobuf_VERSION}.${protobuf_VERSION_PRERELEASE}")

build_defs/cpp_opts.bzl

@@ -41,10 +41,3 @@ LINK_OPTS = select({
         "-lm",
     ],
 })
-
-# When cross-compiling for Windows we need to statically link pthread and the C++ library.
-PROTOC_LINK_OPTS = select({
-    "//build_defs:config_win32": ["-static"],
-    "//build_defs:config_win64": ["-static"],
-    "//conditions:default": [],
-})

ci/common.bazelrc

@@ -19,10 +19,10 @@ build:msan --action_env=MSAN_OPTIONS=poison_in_dtor=1
 build:msan --copt=-DMEMORY_SANITIZER=1
 
 # Use our instrumented LLVM libc++ in Kokoro.
-build:kokoro-msan --config=msan
+build:docker-msan --config=msan
-build:kokoro-msan --linkopt=-L/opt/libcxx_msan/lib
+build:docker-msan --linkopt=-L/opt/libcxx_msan/lib --linkopt=-lc++abi
-build:kokoro-msan --linkopt=-Wl,-rpath,/opt/libcxx_msan/lib
+build:docker-msan --linkopt=-Wl,-rpath,/opt/libcxx_msan/lib
-build:kokoro-msan --cxxopt=-stdlib=libc++ --linkopt=-stdlib=libc++
+build:docker-msan --cxxopt=-stdlib=libc++ --linkopt=-stdlib=libc++
 
 
 build:tsan --config=san-common --copt=-fsanitize=thread --linkopt=-fsanitize=thread

pkg/BUILD.bazel

@@ -59,7 +59,7 @@ Please refer to our official github site for more installation instructions:
 
 pkg_files(
     name = "protoc_files",
-    srcs = ["//:protoc"],
+    srcs = ["//:protoc_static"],
     attributes = pkg_attributes(mode = "0555"),
     prefix = "bin/",
     visibility = ["//visibility:private"],

src/google/protobuf/compiler/BUILD.bazel

@@ -11,7 +11,7 @@ load(
 )
 load("@rules_proto//proto:defs.bzl", "proto_library")
 load("//build_defs:arch_tests.bzl", "aarch64_test", "x86_64_test")
-load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS", "PROTOC_LINK_OPTS")
+load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS")
 
 proto_library(
     name = "plugin_proto",
@@ -123,7 +123,7 @@ cc_library(
 cc_binary(
     name = "protoc_nowkt",
     copts = COPTS,
-    linkopts = LINK_OPTS + PROTOC_LINK_OPTS,
+    linkopts = LINK_OPTS,
     visibility = [
         "//src/google/protobuf:__pkg__",
     ],

src/google/protobuf/io/tokenizer.cc

@@ -942,7 +942,8 @@ bool Tokenizer::NextWithComments(std::string* prev_trailing_comments,
             // makes no sense to attach a comment to the following token.
             collector.Flush();
           }
-          if (prev_line == line_ || trailing_comment_end_line == line_) {
+          if (result &&
+              (prev_line == line_ || trailing_comment_end_line == line_)) {
             // When previous token and this one are on the same line, or
             // even if a multi-line trailing comment ends on the same line
             // as this token, it's unclear to what token the comment

src/google/protobuf/io/tokenizer_unittest.cc

@@ -643,6 +643,7 @@ inline std::ostream& operator<<(std::ostream& out,
   return out << absl::CEscape(test_case.input);
 }
 
+// clang-format off
 DocCommentCase kDocCommentCases[] = {
     {"prev next",
 
@@ -650,6 +651,18 @@ DocCommentCase kDocCommentCases[] = {
      {},
      ""},
 
+    {"prev // no next token\n",
+
+     " no next token\n",
+     {},
+     ""},
+
+    {"prev // no next token and no trailing newline",
+
+     " no next token and no trailing newline",
+     {},
+     ""},
+
     {"prev /* detached */ next",
 
      "",
@@ -780,7 +793,7 @@ DocCommentCase kDocCommentCases[] = {
      prev /* a single block comment
          that spans multiple lines
          is detached if it ends
-         on the same line as next */ next"
+         on the same line as next */ next
      )pb",
 
      "",
@@ -791,7 +804,7 @@ DocCommentCase kDocCommentCases[] = {
      ""},
 
     {R"pb(
-     prev /* trailing */ /* leading */ next"
+       prev /* trailing */ /* leading */ next
      )pb",
 
      " trailing ",
@@ -802,13 +815,26 @@ DocCommentCase kDocCommentCases[] = {
      prev /* multi-line
           trailing */ /* an oddly
                       placed detached */ /* an oddly
-                                         placed leading */ next"
+                                         placed leading */ next
      )pb",
 
      " multi-line\ntrailing ",
      {" an oddly\nplaced detached "},
      " an oddly\nplaced leading "},
+
+    {R"pb(
+       prev  // trailing with newline
+       // detached
+       /* another detached */
+       // leading but no next token to attach it to
+     )pb",
+
+     " trailing with newline\n",
+     {" detached\n", " another detached ",
+      " leading but no next token to attach it to\n"},
+     ""},
 };
+// clang-format on
 
 TEST_2D(TokenizerTest, DocComments, kDocCommentCases, kBlockSizes) {
   // Set up the tokenizer.
@@ -822,8 +848,8 @@ TEST_2D(TokenizerTest, DocComments, kDocCommentCases, kBlockSizes) {
                          kDocCommentCases_case.input.size(), kBlockSizes_case);
   Tokenizer tokenizer2(&input2, &error_collector);
 
-  tokenizer.Next();
+  EXPECT_TRUE(tokenizer.Next());
-  tokenizer2.Next();
+  EXPECT_TRUE(tokenizer2.Next());
 
   EXPECT_EQ("prev", tokenizer.current().text);
   EXPECT_EQ("prev", tokenizer2.current().text);
@@ -831,11 +857,13 @@ TEST_2D(TokenizerTest, DocComments, kDocCommentCases, kBlockSizes) {
   std::string prev_trailing_comments;
   std::vector<std::string> detached_comments;
   std::string next_leading_comments;
-  tokenizer.NextWithComments(&prev_trailing_comments, &detached_comments,
+  bool has_next = tokenizer.NextWithComments(
-                             &next_leading_comments);
+      &prev_trailing_comments, &detached_comments, &next_leading_comments);
-  tokenizer2.NextWithComments(NULL, NULL, NULL);
+  EXPECT_EQ(has_next, tokenizer2.NextWithComments(nullptr, nullptr, nullptr));
-  EXPECT_EQ("next", tokenizer.current().text);
+  if (has_next) {
-  EXPECT_EQ("next", tokenizer2.current().text);
+    EXPECT_EQ("next", tokenizer.current().text);
+    EXPECT_EQ("next", tokenizer2.current().text);
+  }
 
   EXPECT_EQ(kDocCommentCases_case.prev_trailing_comments,
             prev_trailing_comments);