diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 3d258fc0eb..e17f174d4e 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -10,7 +10,7 @@ jobs: name: Check for spelling errors runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - uses: codespell-project/actions-codespell@master with: check_filenames: true diff --git a/.github/workflows/generate_files.yml b/.github/workflows/generate_files.yml index bbeef6eaee..95df914fb1 100644 --- a/.github/workflows/generate_files.yml +++ b/.github/workflows/generate_files.yml @@ -22,7 +22,7 @@ jobs: fail-fast: false # Don't cancel all jobs if one fails. steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 with: # Note: this token has an expiration date, so if the workflow starts # failing then you may need to generate a fresh token. diff --git a/.github/workflows/objc_cocoapods.yml b/.github/workflows/objc_cocoapods.yml index 939e89096e..7ed6987420 100644 --- a/.github/workflows/objc_cocoapods.yml +++ b/.github/workflows/objc_cocoapods.yml @@ -31,7 +31,7 @@ jobs: PLATFORM: ["ios", "macos", "tvos"] CONFIGURATION: ["Debug", "Release"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - name: Pod lib lint run: | pod lib lint --verbose \ diff --git a/.github/workflows/ruby_install.yml b/.github/workflows/ruby_install.yml deleted file mode 100644 index a1823c77fe..0000000000 --- a/.github/workflows/ruby_install.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: Ruby Install Tests - -on: - push: - branches: - - main - - '[0-9]+.x' - pull_request: - branches: - - main - - '[0-9]+.x' - workflow_dispatch: - -jobs: - test_ruby_gems: - name: Test ruby gems - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - include: - - { ruby: 2.6, bazel: 6.0.0} - - { ruby: 2.7, bazel: 6.0.0} - - { ruby: 3.0, bazel: 6.0.0} - - { ruby: 3.1, bazel: 6.0.0} - - { ruby: 3.2, bazel: 6.0.0} - - { ruby: jruby-9.2, bazel: 6.0.0} - - { ruby: jruby-9.3, bazel: 6.0.0} - - { ruby: 2.6, bazel: 5.1.1} - - { ruby: jruby-9.2, bazel: 5.1.1} - - steps: - - uses: actions/checkout@v2 - - name: Install bazel - run: | - sudo apt-get install -qy wget - mkdir $HOME/bin - wget -O $HOME/bin/bazel https://github.com/bazelbuild/bazel/releases/download/${{ matrix.bazel }}/bazel-${{ matrix.bazel }}-linux-x86_64 - chmod a+x $HOME/bin/bazel - - name: Install git - run: | - sudo apt-get install -qy --no-install-recommends git - - name: Install ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: ${{ matrix.ruby }} - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: recursive - - name: Build cruby gem - run: $HOME/bin/bazel build ruby:release - if: ${{ !contains(matrix.ruby, 'jruby') }} - - name: Install cruby gem - run: gem install bazel-bin/ruby/google-protobuf-* - if: ${{ !contains(matrix.ruby, 'jruby') }} - - name: Build jruby gem - run: $HOME/bin/bazel build ruby:jruby_release - if: ${{ contains(matrix.ruby, 'jruby') }} - - name: Install jruby gem - run: gem install bazel-bin/ruby/google-protobuf-* - if: ${{ contains(matrix.ruby, 'jruby') }} - - name: Test installation - run: | - bazel run //:protoc -- --proto_path=$GITHUB_WORKSPACE/src --proto_path=$GITHUB_WORKSPACE/ruby/tests --proto_path=$GITHUB_WORKSPACE/ruby --ruby_out=$GITHUB_WORKSPACE/ruby tests/test_import_proto2.proto - bazel run //:protoc -- --proto_path=$GITHUB_WORKSPACE/src --proto_path=$GITHUB_WORKSPACE/ruby/tests --proto_path=$GITHUB_WORKSPACE/ruby --ruby_out=$GITHUB_WORKSPACE/ruby tests/basic_test.proto - ruby ruby/tests/basic.rb diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml new file mode 100644 index 0000000000..bcc83ddcb0 --- /dev/null +++ b/.github/workflows/test_cpp.yml @@ -0,0 +1,46 @@ +name: C++ Tests + +on: + workflow_call: + inputs: + safe-checkout: + required: true + description: "The SHA key for the commit we want to run over" + type: string + +jobs: + linux: + strategy: + fail-fast: false # Don't cancel all jobs if one fails. + matrix: + config: + - { name: Optimized, flags: --config=opt } + - { name: Debug, flags: --config=dbg } + - { name: ASAN, flags: --config=asan } + - { name: MSAN, flags: --config=kokoro-msan } + - { name: TSAN, flags: --config=tsan } + - { name: UBSAN, flags: --config=ubsan } + include: + # Set defaults + - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize@sha256:dbd2f15fb69734d72c3fd10cb819bbe2ce4890acf49e9a2f9403983fe48e8807 + - targets: //pkg/... //src/... @com_google_protobuf_examples//... + + # Override cases with custom images + - config: { name: "TCMalloc" } + image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/tcmalloc@sha256:9d975616c3fd44d5a091aeb60ee94f37e22fb367d471d258fc18cb4a2387c943" + targets: "//src/..." + - config: { name: "aarch64" } + targets: "//src/... //src/google/protobuf/compiler:protoc_aarch64_test" + image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-e863f8ec6b1dfe41f7dc573bac9c8072a0a68b1b" + - config: { name: "Bazel4" } + targets: "//src/..." + image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:4.2.3-3b71de326b62f67bf754c4dc4016d6a2fa9dd664" + name: mat${{ matrix.none }} + uses: ./.github/workflows/tool_docker.yml + with: + name: Linux ${{ matrix.config.name }} + safe-checkout: ${{ inputs.safe-checkout }} + image: ${{ matrix.image }} + bazel: test ${{ matrix.targets }} ${{ matrix.config.flags }} --distinct_host_configuration=false + bazel-cache: cpp_bazel/${{ matrix.config.name }} + secrets: inherit diff --git a/.github/workflows/php-ext.yml b/.github/workflows/test_php_ext.yml similarity index 74% rename from .github/workflows/php-ext.yml rename to .github/workflows/test_php_ext.yml index da7e6f8165..901512c13b 100644 --- a/.github/workflows/php-ext.yml +++ b/.github/workflows/test_php_ext.yml @@ -1,15 +1,19 @@ -name: PHP extension +name: PHP Extension Tests on: - - push - - pull_request + workflow_call: + inputs: + safe-checkout: + required: true + description: "The SHA key for the commit we want to run over" + type: string permissions: contents: read # to fetch code (actions/checkout) jobs: build-php: - name: Build PHP extension + name: Build runs-on: ubuntu-latest container: ${{ matrix.php-image }} strategy: @@ -17,6 +21,8 @@ jobs: php-image: - php:7.4-cli - php:8.1-cli + # TODO(b/266868629) Dockerize these instead of installing all the + # dependencies on each run. steps: - name: Install python3 run: | @@ -32,8 +38,9 @@ jobs: run: | apt-get install -qy --no-install-recommends git - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 with: + ref: ${{ inputs.safe-checkout }} submodules: recursive - name: Create package run: | diff --git a/.github/workflows/test_ruby_install.yml b/.github/workflows/test_ruby_install.yml new file mode 100644 index 0000000000..ae0dd68a28 --- /dev/null +++ b/.github/workflows/test_ruby_install.yml @@ -0,0 +1,44 @@ +name: Ruby Install Tests + +on: + workflow_call: + inputs: + safe-checkout: + required: true + description: "The SHA key for the commit we want to run over" + type: string + +jobs: + test_ruby_gems: + strategy: + fail-fast: false + matrix: + include: + - { name: Ruby 2.6, ruby: ruby-2.6.0, bazel: 5.1.1} + - { name: Ruby 2.7, ruby: ruby-2.7.0, bazel: 5.1.1} + - { name: Ruby 3.0, ruby: ruby-3.0.2, bazel: 5.1.1} + - { name: Ruby 3.1, ruby: ruby-3.1.0, bazel: 5.1.1} + - { name: Ruby 3.2, ruby: ruby-3.2.0, bazel: 5.1.1} + - { name: JRuby 9.2, ruby: jruby-9.2.20.1, bazel: 5.1.1} + - { name: JRuby 9.3, ruby: jruby-9.3.4.0, bazel: 5.1.1} + - { name: Ruby 2.6 (Bazel6), ruby: ruby-2.6.0, bazel: 6.0.0} + - { name: JRuby 9.2 (Bazel6), ruby: jruby-9.2.20.1, bazel: 6.0.0} + + name: mat${{ matrix.none }} + uses: ./.github/workflows/tool_docker.yml + with: + name: Linux ${{ matrix.name }} + safe-checkout: ${{ inputs.safe-checkout }} + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:${{ matrix.ruby }}-${{ matrix.bazel }}-75e79f791b96e056086f43ace729cf3ebf9a9f5d + bazel-cache: ruby_install/${{ matrix.ruby }}_${{ matrix.bazel }} + run-flags: --entrypoint "/bin/bash" + command: > + -l -c " + bazel --version && + ruby --version && + bazel build //ruby:release //:protoc $BAZEL_CACHE && + gem install bazel-bin/ruby/google-protobuf-* && + bazel-bin/protoc --proto_path=src --proto_path=ruby/tests --proto_path=ruby --ruby_out=ruby tests/test_import_proto2.proto && + bazel-bin/protoc --proto_path=src --proto_path=ruby/tests --proto_path=ruby --ruby_out=ruby tests/basic_test.proto && + ruby ruby/tests/basic.rb" + secrets: inherit diff --git a/.github/workflows/test_runner.yml b/.github/workflows/test_runner.yml new file mode 100644 index 0000000000..cf06217f19 --- /dev/null +++ b/.github/workflows/test_runner.yml @@ -0,0 +1,114 @@ +name: Tests + +# This file implements the protection strategy laid out in +# go/protobuf-gha-protected-resources. Pull requests from branches within this +# repository are considered safe and will immediately start running tests on +# every commit. Pull requests from forked repositories are unsafe, and leave +# us vulnerable to PWN requests and stolen resources. In these cases, we +# require a special "safe for tests" tag to be added to the pull request before +# we start testing. This will be immediately removed, so that further commits +# require their own stamp to test. + +on: + # continuous + schedule: + # Run daily at 10 AM UTC (2 AM PDT) + - cron: 0 10 * * * + + # postsubmit + push: + branches: + - main + - '[0-9]+.x' + # For testing purposes so we can stage this on the `gha` branch. + - gha + + # safe presubmit + pull_request: + branches: + - main + - '[0-9]+.x' + # For testing purposes so we can stage this on the `gha` branch. + - gha + + # unsafe presubmit + pull_request_target: + branches: + - main + - '[0-9]+.x' + # For testing purposes so we can stage this on the `gha` branch. + - gha + types: [labeled, opened, reopened, synchronize] + + # manual + workflow_dispatch: + +jobs: + check-tag: + name: Check for Safety + + # Avoid running tests twice on PR updates. If the PR is coming from our + # repository, it's safe and we can use `pull_request`. Otherwise, we should + # use `pull_request_target`. + if: | + github.event_name == 'push' || + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == 'protocolbuffers/protobuf') || + (github.event_name == 'pull_request_target' && + github.event.pull_request.head.repo.full_name != 'protocolbuffers/protobuf') + + runs-on: ubuntu-latest + outputs: + # Store the sha for checkout so we can easily use it later. For safe + # events, this will be blank and use the defaults. + checkout-sha: ${{ steps.safe-checkout.outputs.sha }} + steps: + - name: Check + # Trivially pass for safe PRs, and explicitly error for unsafe ones + # unless this is specifically an event for adding the safe label. + run: > + ${{ github.event_name != 'pull_request_target' || github.event.label.name == 'safe for tests' }} || + (echo "This pull request is from an unsafe fork and hasn't been approved to run tests!" && exit 1) + + - name: Cache safe commit + id: safe-checkout + run: > + ${{ github.event_name != 'pull_request_target' }} || + echo "sha=${{ github.event.pull_request.head.sha }}" >> $GITHUB_OUTPUT + + remove-tag: + name: Remove safety tag + needs: [check-tag] + if: github.event.action == 'labeled' + runs-on: ubuntu-latest + steps: + - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 # v1.3.0 + with: + labels: safe for tests + + # Note: this pattern of passing the head sha is vulnerable to PWN requests for + # pull_request_target events. We carefully limit those workflows to require a + # human stamp before continuing. + cpp-bazel: + name: C++ + needs: [check-tag] + uses: ./.github/workflows/test_cpp.yml + with: + safe-checkout: ${{ needs.check-tag.outputs.checkout-sha }} + secrets: inherit + + ruby-install: + name: Ruby Install + needs: [check-tag] + uses: ./.github/workflows/test_ruby_install.yml + with: + safe-checkout: ${{ needs.check-tag.outputs.checkout-sha }} + secrets: inherit + + php-ext: + name: PHP Extension + needs: [check-tag] + uses: ./.github/workflows/test_php_ext.yml + with: + safe-checkout: ${{ needs.check-tag.outputs.checkout-sha }} + secrets: inherit diff --git a/.github/workflows/tool_docker.yml b/.github/workflows/tool_docker.yml new file mode 100644 index 0000000000..1dfdd82f72 --- /dev/null +++ b/.github/workflows/tool_docker.yml @@ -0,0 +1,127 @@ +name: Run a Docker workflow + +on: + workflow_call: + inputs: + name: + required: True + description: "The name to display for the test" + type: string + image: + required: false + default: 'us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1' + description: "The docker image to use" + type: string + safe-checkout: + required: true + description: "The SHA key for the commit we want to run over" + type: string + run-flags: + required: false + description: "Additional flags to pass to docker run" + type: string + bazel-cache: + required: false + description: > + A unique path for the Bazel cache. This will trigger the generation + of a BAZEL_CACHE environment variant that provides the appropriate + flags for any bazel command. + type: string + + # WARNING: loading from cache appears to be slower than pull! + docker-cache: + required: false + description: "Enabled caching of pulled docker images." + default: false + type: boolean + + # Non-Bazel options + command: + required: false + description: "A raw docker command to run" + type: string + + # Bazel options + bazel: + required: false + description: "The Bazel command to run" + type: string + +jobs: + run: + name: ${{ inputs.name }} + timeout-minutes: 120 + runs-on: ubuntu-latest + steps: + - name: Checkout pending changes + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + with: + ref: ${{ inputs.safe-checkout }} + + # Authentication + - name: Setup QEMU for possible emulation + id: qemu-arm64 + uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1.2.0 + + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@ef5d53e30bbcd8d0836f4288f5e50ff3e086997d # v1.0.0 + with: + credentials_json: ${{ secrets.GAR_SERVICE_ACCOUNT }} + export_environment_variables: true + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@d51b5346f85640ec2aa2fa057354d2b82c2fcbce # v1.0.1 + - name: Use gcloud CLI + run: gcloud info + - name: Authenticate for GAR use + run: gcloud auth configure-docker -q us-docker.pkg.dev + + # Create the docker command + - name: Validate Docker command + if: ${{ inputs.command && inputs.bazel}} + run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1 + - name: Configure Bazel caching + # Skip bazel cache for local act runs due to issue with credential files + # and nested docker images + if: ${{ inputs.bazel-cache && !github.event.act_local_test }} + run: > + echo "BAZEL_CACHE= + --google_credentials=/workspace/$(basename $GOOGLE_APPLICATION_CREDENTIALS) + --remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/protobuf/gha/${{ inputs.bazel-cache }}" >> $GITHUB_ENV + - name: Configure Bazel cache updating + # External runs should never write to our caches. + if: ${{ inputs.bazel-cache && !inputs.safe-checkout && !github.event.act_local_test }} + run: echo "BAZEL_CACHE=$BAZEL_CACHE --remote_upload_local_results" >> $GITHUB_ENV + - name: Configure Bazel command + if: ${{ inputs.bazel }} + run: > + echo "DOCKER_COMMAND=${{ inputs.bazel }} + --keep_going --test_output=errors --test_timeout=600 + $BAZEL_CACHE" >> $GITHUB_ENV + + # Grab Docker image + - name: Check docker cache + if: ${{ inputs.docker-cache }} + id: check-docker-cache + uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 + with: + path: ci/docker/ + key: ${{ inputs.image }} + - name: Pull and store if cache miss + if: ${{ inputs.docker-cache && steps.check-docker-cache.outputs.cache-hit != 'true' }} + run: > + docker pull ${{ inputs.image }} && + mkdir -p ci/docker/$(dirname ${{ inputs.image }}) && + docker image save ${{ inputs.image }} --output ./ci/docker/${{ inputs.image }}.tar + - name: Use the cached image on cache hit + if: ${{ inputs.docker-cache && steps.check-docker-cache.outputs.cache-hit == 'true' }} + run: docker image load --input ./ci/docker/${{ inputs.image }}.tar + - name: Pull fresh docker image + if: ${{ !inputs.docker-cache }} + run: docker pull ${{ inputs.image }} + + - name: Run docker + run: > + docker run ${{ inputs.run-flags}} + -v${{ github.workspace }}:/workspace + ${{ inputs.image }} + ${{ inputs.command || '$DOCKER_COMMAND' }} diff --git a/.github/workflows/update_php_repo.yml b/.github/workflows/update_php_repo.yml index ed2199ffa3..acc933f9c3 100644 --- a/.github/workflows/update_php_repo.yml +++ b/.github/workflows/update_php_repo.yml @@ -15,12 +15,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout protobuf-php - uses: actions/checkout@v3 + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 with: repository: protocolbuffers/protobuf-php token: ${{ secrets.BOT_ACCESS_TOKEN }} - name: Clone protobuf - uses: actions/checkout@v3 + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 with: path: protobuf - name: Configure Git Bot diff --git a/ruby/BUILD.bazel b/ruby/BUILD.bazel index 533185473a..c581355c54 100755 --- a/ruby/BUILD.bazel +++ b/ruby/BUILD.bazel @@ -57,7 +57,7 @@ genrule( ) genrule( - name = "release", + name = "ruby_release", srcs = [ "@utf8_range//:utf8_range_srcs", "@utf8_range//:LICENSE", @@ -93,6 +93,15 @@ genrule( }), ) +filegroup( + name = "release", + srcs = select({ + "@rules_ruby//ruby/runtime:config_jruby": [":jruby_release"], + "//conditions:default": [":ruby_release"], + }), + tags = ["manual"], +) + ################################################################################ # Tests diff --git a/src/google/protobuf/BUILD.bazel b/src/google/protobuf/BUILD.bazel index cf72aafb08..2298456152 100644 --- a/src/google/protobuf/BUILD.bazel +++ b/src/google/protobuf/BUILD.bazel @@ -1026,6 +1026,7 @@ cc_test( "-Wno-deprecated-declarations", ], }), + timeout = "long", data = [":testdata"], deps = [ ":cc_test_protos", diff --git a/src/google/protobuf/io/zero_copy_stream_unittest.cc b/src/google/protobuf/io/zero_copy_stream_unittest.cc index e5ff07af35..1cacac08d6 100644 --- a/src/google/protobuf/io/zero_copy_stream_unittest.cc +++ b/src/google/protobuf/io/zero_copy_stream_unittest.cc @@ -733,7 +733,7 @@ TEST_F(IoTest, StringIo) { TEST_F(IoTest, LargeOutput) { // Filter out this test on 32-bit architectures and tsan builds. if(sizeof(void*) < 8) return; -#ifndef THREAD_SANITIZER +#if !defined(THREAD_SANITIZER) && !defined(MEMORY_SANITIZER) std::string str; StringOutputStream output(&str); void* unused_data; @@ -745,7 +745,7 @@ TEST_F(IoTest, LargeOutput) { // Further increases should be possible. output.Next(&unused_data, &size); EXPECT_GT(size, 0); -#endif // THREAD_SANITIZER +#endif // !THREAD_SANITIZER && !MEMORY_SANITIZER } TEST(DefaultReadCordTest, ReadSmallCord) { diff --git a/src/google/protobuf/json/BUILD.bazel b/src/google/protobuf/json/BUILD.bazel index f2194a987b..ea363acf1b 100644 --- a/src/google/protobuf/json/BUILD.bazel +++ b/src/google/protobuf/json/BUILD.bazel @@ -143,6 +143,7 @@ cc_test( name = "lexer_test", srcs = ["internal/lexer_test.cc"], copts = COPTS, + timeout = "long", deps = [ ":lexer", ":test_input_stream",