From ef8ebe6c19008801ae49ae37db4ec5963e952483 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Fri, 24 Feb 2023 11:18:04 -0800 Subject: [PATCH] Migrate our shared github actions to a separate repository. This will make PRs from forked repositories significantly less painful, since they'll agree on which version of each action to use. OTOH, we'll have a separate repo that needs to be maintained, and changes to it will need to be coordinated and versioned carefully. This will likely need to be done less often though now that our infrastructure is stable. PiperOrigin-RevId: 512117705 --- .github/actions/bash/action.yml | 27 ---- .github/actions/bazel-docker/action.yml | 75 ---------- .github/actions/bazel/action.yml | 134 ------------------ .github/actions/ccache/action.yml | 63 -------- .../actions/cross-compile-protoc/action.yml | 38 ----- .github/actions/docker/action.yml | 50 ------- .../actions/internal/bazel-setup/action.yml | 75 ---------- .../internal/ccache-setup-windows/action.yml | 68 --------- .../actions/internal/docker-run/action.yml | 62 -------- .../actions/internal/gcloud-auth/action.yml | 42 ------ .../repository-cache-restore/action.yml | 41 ------ .../internal/repository-cache-save/action.yml | 19 --- .../actions/internal/setup-runner/action.yml | 17 --- .github/workflows/staleness_check.yml | 2 +- .github/workflows/test_cpp.yml | 22 +-- .github/workflows/test_csharp.yml | 6 +- .github/workflows/test_java.yml | 4 +- .github/workflows/test_objectivec.yml | 8 +- .github/workflows/test_php.yml | 14 +- .github/workflows/test_python.yml | 4 +- .github/workflows/test_ruby.yml | 10 +- 21 files changed, 35 insertions(+), 746 deletions(-) delete mode 100644 .github/actions/bash/action.yml delete mode 100644 .github/actions/bazel-docker/action.yml delete mode 100644 .github/actions/bazel/action.yml delete mode 100644 .github/actions/ccache/action.yml delete mode 100644 .github/actions/cross-compile-protoc/action.yml delete mode 100644 .github/actions/docker/action.yml delete mode 100644 .github/actions/internal/bazel-setup/action.yml delete mode 100644 .github/actions/internal/ccache-setup-windows/action.yml delete mode 100644 .github/actions/internal/docker-run/action.yml delete mode 100644 .github/actions/internal/gcloud-auth/action.yml delete mode 100644 .github/actions/internal/repository-cache-restore/action.yml delete mode 100644 .github/actions/internal/repository-cache-save/action.yml delete mode 100644 .github/actions/internal/setup-runner/action.yml diff --git a/.github/actions/bash/action.yml b/.github/actions/bash/action.yml deleted file mode 100644 index d105a44f10..0000000000 --- a/.github/actions/bash/action.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: 'Non-Bazel Bash Run' -description: 'Run a bash script for Protobuf CI testing with a non-Bazel build system' -inputs: - credentials: - required: true - description: "The GCP credentials to use for reading the docker image" - type: string - command: - required: true - description: A command to run in the docker image - -runs: - using: 'composite' - steps: - - name: Setup Runner - uses: ./.github/actions/internal/setup-runner - - - name: Update stale files using Bazel - uses: ./.github/actions/bazel - with: - credentials: ${{ inputs.credentials }} - bazel-cache: regenerate-stale-files - bash: ./regenerate_stale_files.sh $BAZEL_FLAGS - - - name: Run - shell: bash - run: ${{ inputs.command }} diff --git a/.github/actions/bazel-docker/action.yml b/.github/actions/bazel-docker/action.yml deleted file mode 100644 index af1104e23d..0000000000 --- a/.github/actions/bazel-docker/action.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: 'Docker Bazel Run' -description: 'Run a Bazel-based docker image for Protobuf CI testing' -inputs: - credentials: - required: true - description: "The GCP credentials to use for reading the docker image" - type: string - image: - required: false - default: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1 - description: "The docker image to use" - type: string - bazel-cache: - required: true - description: > - A unique path for the Bazel cache. This will trigger the generation - of a BAZEL_CACHE environment variable inside the container that provides - the appropriate flags for any bazel command. - type: string - bazel: - required: false - description: "The Bazel command to run" - type: string - bash: - required: false - description: "A bash command to run. $BAZEL_FLAGS will be available to use for bazel runs." - type: string - -runs: - using: 'composite' - steps: - - name: Authenticate - id: auth - uses: ./.github/actions/internal/gcloud-auth - with: - credentials: ${{ inputs.credentials }} - - - name: Setup Runner - uses: ./.github/actions/internal/setup-runner - - - name: Setup Bazel - id: bazel - uses: ./.github/actions/internal/bazel-setup - with: - credentials-file: /workspace/$(basename ${{ steps.auth.outputs.credentials-file }}) - bazel-cache: ${{ inputs.bazel-cache }} - - - name: Hook up repository Cache - shell: bash - run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --repository_cache='/workspace/${{ env.REPOSITORY_CACHE_PATH }}'" >> $GITHUB_ENV - - - name: Validate inputs - if: ${{ (inputs.bash && inputs.bazel) || (!inputs.bash && !inputs.bazel) }} - shell: bash - run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1 - - - name: Run Bash Docker - uses: ./.github/actions/internal/docker-run - if: ${{ inputs.bash }} - with: - image: ${{ inputs.image }} - run-flags: --entrypoint "/bin/bash" - command: -l -c "${{ inputs.bash }}" - - - name: Run Bazel Docker - uses: ./.github/actions/internal/docker-run - if: ${{ !inputs.bash }} - with: - image: ${{ inputs.image }} - command: ${{ inputs.bazel }} ${{ env.BAZEL_FLAGS }} - - - name: Save Bazel repository cache - # Only allow repository cache updates during post-submits. - if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' }} - uses: ./.github/actions/internal/repository-cache-save diff --git a/.github/actions/bazel/action.yml b/.github/actions/bazel/action.yml deleted file mode 100644 index 5fd3e9a901..0000000000 --- a/.github/actions/bazel/action.yml +++ /dev/null @@ -1,134 +0,0 @@ -name: 'Docker Bazel Run' -description: 'Run a Bazel-based docker image for Protobuf CI testing' -inputs: - credentials: - required: true - description: The GCP credentials to use for reading the docker image - type: string - bazel-cache: - required: true - description: > - A unique path for the Bazel cache. This will trigger the generation - of a BAZEL_CACHE environment variable inside the container that provides - the appropriate flags for any bazel command. - type: string - version: - required: false - description: A pinned Bazel version to use - default: '5.1.1' - type: string - bazel: - required: false - description: The Bazel command to run - type: string - bash: - required: false - description: > - A bash command to run. $BAZEL_FLAGS and $BAZEL_STARTUP_FLAGS will be - available to use for bazel runs. - type: string - -runs: - using: 'composite' - steps: - - name: Authenticate - id: auth - uses: ./.github/actions/internal/gcloud-auth - with: - credentials: ${{ inputs.credentials }} - - - name: Setup Runner - uses: ./.github/actions/internal/setup-runner - - - name: Setup Bazel - id: bazel - uses: ./.github/actions/internal/bazel-setup - with: - credentials-file: ${{ steps.auth.outputs.credentials-file }} - bazel-cache: ${{ inputs.bazel-cache }} - - - name: Get Linux bazelisk path - if: runner.os == 'Linux' - shell: bash - run: echo "BAZELISK_PATH=~/.cache/bazelisk" >> $GITHUB_ENV - - - name: Get MacOS bazelisk path - if: runner.os == 'macOS' - shell: bash - run: echo "BAZELISK_PATH=~/Library/Caches/bazelisk" >> $GITHUB_ENV - - - name: Get Windows bazelisk path - if: runner.os == 'Windows' - shell: bash - run: echo "BAZELISK_PATH=$LOCALAPPDATA\bazelisk" >> $GITHUB_ENV - - - name: Cache Bazelisk - if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' }} - uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 - with: - path: ${{ env.BAZELISK_PATH }} - key: bazel-${{ runner.os }}-${{ inputs.version }} - - - name: Restore Bazelisk - if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} - uses: actions/cache/restore@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 - with: - path: ${{ env.BAZELISK_PATH }} - key: bazel-${{ runner.os }}-${{ inputs.version }} - - - name: Hook up repository Cache - shell: bash - run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --repository_cache=$(pwd)/${{ env.REPOSITORY_CACHE_PATH }}" >> $GITHUB_ENV - - - name: Validate inputs - if: ${{ (inputs.bash && inputs.bazel) || (!inputs.bash && !inputs.bazel) }} - shell: bash - run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1 - - - name: Pin Bazel version - shell: bash - run: echo "USE_BAZEL_VERSION=${{ inputs.version }}" >> $GITHUB_ENV - - - name: Output Bazel version - shell: bash - run: bazelisk version - - # Bazel has multiple Xcode calls with hardcoded timeouts. Many of these - # end up timing out on our github runners, causing flakes on every mac - # build that invoked Bazel. To work around this, we manually inoke these - # calls before running Bazel to make sure they end up in Xcode's cache for - # quicker runs later. All of these calls are obtained from xcrun calls in - # https://github.com/bazelbuild/bazel/blob/e8a69f5d5acaeb6af760631490ecbf73e8a04eeb/tools/cpp/osx_cc_configure.bzl. - # See https://github.com/bazelbuild/bazel/issues/17437 for more details. - # TODO(b/269503614) Remove this once Bazel provides an official solution. - - name: Warm up Xcode - if: ${{ runner.os == 'macOS' }} - shell: bash - run: | - mkdir -p mac_bazel_workaround - bazelisk ${{ steps.bazel.outputs.bazel-startup-flags }} build @bazel_tools//tools/osx:xcode_locator.m $BAZEL_FLAGS - XCODE_LOCATOR_FLAGS="--sdk macosx clang -mmacosx-version-min=10.9 -fobjc-arc -framework CoreServices -framework Foundation" - SINGLE_ARCH_COMPILE_FLAGS="--sdk macosx clang -mmacosx-version-min=10.9 -std=c++11 -lc++ -O3" - COMPILE_FLAGS="$SINGLE_ARCH_COMPILE_FLAGS -arch arm64 -arch x86_64 -Wl,-no_adhoc_codesign -Wl,-no_uuid -O3" - time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $XCODE_LOCATOR_FLAGS -o mac_bazel_workaround/xcode-locator-bin $(bazel info output_base)/external/bazel_tools/tools/osx/xcode_locator.m - time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $SINGLE_ARCH_COMPILE_FLAGS -o mac_bazel_workaround/libtool_check_unique $(bazel info output_base)/external/bazel_tools/tools/objc/libtool_check_unique.cc - time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $COMPILE_FLAGS -o mac_bazel_workaround/libtool_check_unique $(bazel info output_base)/external/bazel_tools/tools/objc/libtool_check_unique.cc - time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $SINGLE_ARCH_COMPILE_FLAGS -o mac_bazel_workaround/wrapped_clang $(bazel info output_base)/external/bazel_tools/tools/osx/crosstool/wrapped_clang.cc - time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $COMPILE_FLAGS -o mac_bazel_workaround/wrapped_clang $(bazel info output_base)/external/bazel_tools/tools/osx/crosstool/wrapped_clang.cc - - - name: Run Bash - if: ${{ inputs.bash }} - run: ${{ inputs.bash }} - shell: bash - - - name: Run Bazel - if: ${{ !inputs.bash }} - run: >- - bazelisk ${{ steps.bazel.outputs.bazel-startup-flags }} - ${{ inputs.bazel }} $BAZEL_FLAGS - shell: bash - - - name: Save Bazel repository cache - # Only allow repository cache updates during post-submits. - if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target'}} - uses: ./.github/actions/internal/repository-cache-save diff --git a/.github/actions/ccache/action.yml b/.github/actions/ccache/action.yml deleted file mode 100644 index 2d70550364..0000000000 --- a/.github/actions/ccache/action.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: 'CCache Setup' -description: 'Run a Bazel-based docker image for Protobuf CI testing' -inputs: - cache-prefix: - required: true - description: A unique prefix to prevent cache pollution - type: string - support-modules: - required: false - description: Whether or not we need to support modules. This can result in extra cache misses. - -runs: - using: 'composite' - steps: - - name: Configure ccache environment variables - shell: bash - run: | - echo "CCACHE_BASEDIR=${{ github.workspace }}" >> $GITHUB_ENV - echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV - echo "CCACHE_COMPRESS=true" >> $GITHUB_ENV - echo "CCACHE_COMPRESSLEVEL=5" >> $GITHUB_ENV - echo "CCACHE_MAXSIZE=100M" >> $GITHUB_ENV - echo "CCACHE_SLOPPINESS=clang_index_store,include_file_ctime,include_file_mtime,file_macro,time_macros" >> $GITHUB_ENV - echo "CCACHE_DIRECT=true" >> $GITHUB_ENV - echo "CCACHE_CMAKE_FLAGS=-Dprotobuf_ALLOW_CCACHE=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" >> $GITHUB_ENV - - - name: Setup ccache on Windows - if: ${{ runner.os == 'Windows' }} - uses: ./.github/actions/internal/ccache-setup-windows - - name: Setup ccache on Mac - if: ${{ runner.os == 'macOS' }} - shell: bash - run: brew install ccache - - - name: Setup fixed path ccache caching - uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 - with: - path: | - .ccache/** - !.ccache/lock - !.ccache/tmp - # Always push to a cache key unique to this commit. - key: ${{ format('ccache-{0}-{1}-{2}', inputs.cache-prefix, github.ref_name, github.sha) }} - # Select a cache to restore from with the follow order of preference: - # 1) The exact same commit we're running over - # 2) The latest cache from the current ref branch - # 3) The latest push to the base ref of a pull request - restore-keys: | - ${{ format('ccache-{0}-{1}-{2}', inputs.cache-prefix, github.ref_name, github.sha) }} - ${{ format('ccache-{0}-{1}', inputs.cache-prefix, github.ref_name) }} - ${{ format('ccache-{0}-{1}', inputs.cache-prefix, github.base_ref) }} - - - name: Enable module support - if: ${{ inputs.support-modules }} - shell: bash - run: | - echo "CCACHE_SLOPPINESS=$CCACHE_SLOPPINESS,modules" >> $GITHUB_ENV - echo "CCACHE_DEPEND=true" >> $GITHUB_ENV - - - name: Zero out ccache - if: ${{ runner.os != 'Linux' }} - shell: bash - run: ccache -z diff --git a/.github/actions/cross-compile-protoc/action.yml b/.github/actions/cross-compile-protoc/action.yml deleted file mode 100644 index 23660af705..0000000000 --- a/.github/actions/cross-compile-protoc/action.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: 'Cross-compile protoc' -description: 'Produces a cross-compiled protoc binary for a target architecture' -inputs: - credentials: - required: true - description: The GCP credentials to use for reading the docker image - type: string - architecture: - required: true - description: The target architecture to build for - type: string -outputs: - protoc: - description: "Cross-compiled protoc location. Also output to $PROTOC" - value: ${{ steps.output.outputs.protoc }} - -runs: - using: 'composite' - steps: - - name: Cross compile protoc for ${{ inputs.architecture }} - uses: ./.github/actions/bazel-docker - with: - credentials: ${{ inputs.credentials }} - bazel-cache: xcompile-protoc/${{ inputs.architecture }} - bash: | - bazel build //:protoc --config=${{ inputs.architecture }} $BAZEL_FLAGS - cp bazel-bin/protoc . - - - name: Set protoc environment variable - shell: bash - run: echo "PROTOC=protoc-${{ inputs.architecture }}" >> $GITHUB_ENV - - - name: Extract binary - id: output - shell: bash - run: | - mv protoc $PROTOC - echo "protoc=$PROTOC" >> $GITHUB_OUTPUT diff --git a/.github/actions/docker/action.yml b/.github/actions/docker/action.yml deleted file mode 100644 index 54de3f908e..0000000000 --- a/.github/actions/docker/action.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: 'Docker Non-Bazel Run' -description: 'Run a docker image for Protobuf CI testing with a non-Bazel build system' -inputs: - credentials: - required: true - description: "The GCP credentials to use for reading the docker image" - type: string - command: - required: true - description: A command to run in the docker image - image: - required: false - default: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1 - description: "The docker image to use" - type: string - platform: - required: false - description: "The platform to use for the image" - type: string - skip-staleness-check: - required: false - description: "Skip staleness checks" - type: boolean - -runs: - using: 'composite' - steps: - - name: Setup Runner - uses: ./.github/actions/internal/setup-runner - - - name: Update stale files using Bazel - if: ${{ !inputs.skip-staleness-check }} - uses: ./.github/actions/bazel-docker - with: - image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1 - credentials: ${{ inputs.credentials }} - bazel-cache: regenerate-stale-files - bash: ./regenerate_stale_files.sh $BAZEL_FLAGS - - - name: Generate docker flags - if: inputs.platform - shell: bash - run: echo "DOCKER_RUN_FLAGS=--platform ${{inputs.platform}}" >> $GITHUB_ENV - - - name: Run Docker - uses: ./.github/actions/internal/docker-run - with: - image: ${{ inputs.image }} - command: ${{ inputs.command }} - run-flags: ${{ env.DOCKER_RUN_FLAGS }} diff --git a/.github/actions/internal/bazel-setup/action.yml b/.github/actions/internal/bazel-setup/action.yml deleted file mode 100644 index 0ef5d5a226..0000000000 --- a/.github/actions/internal/bazel-setup/action.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: Setup Bazel -description: Setup a Bazel environment for Protobuf CI testing -inputs: - credentials-file: - required: true - description: The GCP credentials file to use for caching - type: string - bazel-cache: - required: true - description: A unique path for the Bazel cache. - type: string - -outputs: - bazel-flags: - description: Bazel flags that should be sent to all Bazel invocations - value: ${{ steps.output.outputs.bazel-flags }} - bazel-startup-flags: - description: Bazel startup flags that should be sent to all Bazel invocations - value: ${{ steps.output.outputs.bazel-startup-flags }} - -runs: - using: 'composite' - steps: - - name: Initialize BAZEL environment variable - shell: bash - run: echo "BAZEL=bazelisk" >> $GITHUB_ENV - - - name: Initialize Windows startup flags - if: runner.os == 'Windows' - shell: bash - run: echo "BAZEL_STARTUP_FLAGS=--output_user_root=C:/tmp --windows_enable_symlinks" >> $GITHUB_ENV - - - name: Initialize Bazel flags - shell: bash - run: echo "BAZEL_FLAGS=--keep_going --test_output=errors --test_timeout=600" >> $GITHUB_ENV - - - name: Initialize Windows-specific Bazel flags - if: runner.os == 'Windows' - shell: bash - run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --enable_runfiles" >> $GITHUB_ENV - - - name: Initialize MacOS-specific Bazel flags - if: runner.os == 'macOS' - shell: bash - run: | - echo "BAZEL_FLAGS=$BAZEL_FLAGS --xcode_version_config=//.github:host_xcodes" >> $GITHUB_ENV - echo "DEVELOPER_DIR=${{ env.DEVELOPER_DIR || '/Applications/Xcode_14.1.app/Contents/Developer' }}" >> $GITHUB_ENV - - - name: Configure Bazel caching - # Skip bazel cache for local act runs due to issue with credential files - # and nested docker images - if: ${{ inputs.bazel-cache && !github.event.act_local_test }} - shell: bash - run: >- - echo "BAZEL_FLAGS=$BAZEL_FLAGS - --google_credentials=${{ inputs.credentials-file }} - --remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/protobuf/gha/${{ inputs.bazel-cache }}" >> $GITHUB_ENV - - - name: Configure Bazel cache writing - # External runs should never write to our caches. - if: ${{ github.event_name != 'pull_request_target' && inputs.bazel-cache && !github.event.act_local_test }} - shell: bash - run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --remote_upload_local_results" >> $GITHUB_ENV - - - name: Output Bazel flags - id: output - shell: bash - run: | - echo "bazel-flags=$BAZEL_FLAGS" >> $GITHUB_OUTPUT - echo "bazel-startup-flags=$BAZEL_STARTUP_FLAGS" >> $GITHUB_OUTPUT - - - name: Restore Bazel repository cache - uses: ./.github/actions/internal/repository-cache-restore - with: - bazel-cache: ${{ inputs.bazel-cache }} diff --git a/.github/actions/internal/ccache-setup-windows/action.yml b/.github/actions/internal/ccache-setup-windows/action.yml deleted file mode 100644 index 7b9f9925b3..0000000000 --- a/.github/actions/internal/ccache-setup-windows/action.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: 'CCache Setup' -description: 'Setup ccache for us in Windows CI' -inputs: - ccache-version: - required: false - default: '4.7.4' - description: A pinned version of ccache - type: string - -runs: - using: 'composite' - steps: - - name: Setup MSVC - uses: ilammy/msvc-dev-cmd@cec98b9d092141f74527d0afa6feb2af698cfe89 # v1.12.1 - with: - arch: x64 - vsversion: '2019' - - - name: Setup ccache path - shell: bash - run: | - echo "CCACHE_EXE_PATH=$LOCALAPPDATA\ccache-${{ inputs.ccache-version }}-windows-x86_64" >> $GITHUB_ENV - echo "$LOCALAPPDATA\ccache-${{ inputs.ccache-version }}-windows-x86_64" >> $GITHUB_PATH - - - name: Add ccache to Powershell path - shell: pwsh - run: echo "%LOCALAPPDATA%\ccache-${{ inputs.ccache-version }}-windows-x86_64" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - - name: Setup caching of ccache download - if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' }} - id: ccache-cache - uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 - with: - path: ${{ env.CCACHE_EXE_PATH }} - key: ccache-exe-${{ inputs.ccache-version }} - - - name: Restore ccache download - if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} - id: ccache-restore - uses: actions/cache/restore@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 - with: - path: ${{ env.CCACHE_EXE_PATH }} - key: ccache-exe-${{ inputs.ccache-version }} - - - name: Download ccache - shell: bash - if: ${{ steps.ccache-cache.outputs.cache-hit != 'true' && steps.ccache-restore.outputs.cache-hit != 'true'}} - run: | - cd $LOCALAPPDATA - curl -kLSs "https://github.com/ccache/ccache/releases/download/v${{ inputs.ccache-version }}/ccache-${{ inputs.ccache-version }}-windows-x86_64.zip" -o ccache.zip - unzip ccache.zip - rm ccache.zip - ccache --version - - - name: Configure ccache environment variables - shell: pwsh - run: | - Write-Host $Env:GITHUB_REF - $cllocation = (Get-Command cl.exe).Path - echo "CCACHE_COMPILER=$cllocation" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append - echo "CCACHE_COMPILERTYPE=msvc" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append - - - name: Configure Windows-specific ccache environment variables - shell: bash - # Windows caches are about 2x larger than other platforms. - run: | - echo "CCACHE_COMPRESSLEVEL=10" >> $GITHUB_ENV - echo "CCACHE_MAXSIZE=200M" >> $GITHUB_ENV diff --git a/.github/actions/internal/docker-run/action.yml b/.github/actions/internal/docker-run/action.yml deleted file mode 100644 index 1cc14894d8..0000000000 --- a/.github/actions/internal/docker-run/action.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: 'Run Docker' -description: 'Run a docker image for Protobuf CI testing' -inputs: - image: - required: true - description: "The docker image to use" - type: string - command: - required: true - description: "A raw docker command to run" - type: string - run-flags: - required: false - description: "Additional flags to pass to docker run" - type: string - - # WARNING: loading from cache appears to be slower than pull! - docker-cache: - required: false - description: "Enabled caching of pulled docker images." - -runs: - using: 'composite' - steps: - - name: Authenticate for GAR use - shell: bash - run: gcloud auth configure-docker -q us-docker.pkg.dev - - - name: Setup QEMU for possible emulation - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2.1.0 - - - name: Check docker cache - if: ${{ inputs.docker-cache }} - id: check-docker-cache - uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 - with: - path: ci/docker/ - key: ${{ inputs.image }} - - - name: Pull and store if cache miss - shell: bash - if: ${{ inputs.docker-cache && steps.check-docker-cache.outputs.cache-hit != 'true' }} - run: > - time docker pull -q ${{ inputs.image }} && - mkdir -p ci/docker/$(dirname ${{ inputs.image }}) && - time docker image save ${{ inputs.image }} --output ./ci/docker/${{ inputs.image }}.tar - - - name: Use the cached image on cache hit - shell: bash - if: ${{ inputs.docker-cache && steps.check-docker-cache.outputs.cache-hit == 'true' }} - run: time docker image load --input ./ci/docker/${{ inputs.image }}.tar - - - name: Pull fresh docker image - shell: bash - if: ${{ !inputs.docker-cache }} - run: time docker pull -q ${{ inputs.image }} - - - name: Run docker - shell: bash - run: > - time docker run ${{ inputs.run-flags}} -v${{ github.workspace }}:/workspace - ${{ inputs.image }} ${{ inputs.command }} diff --git a/.github/actions/internal/gcloud-auth/action.yml b/.github/actions/internal/gcloud-auth/action.yml deleted file mode 100644 index 189619ddba..0000000000 --- a/.github/actions/internal/gcloud-auth/action.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: 'Authenticate for GCP' -description: 'Authenticate a workflow for Protobuf CI testing' -inputs: - credentials: - required: true - description: "The GCP credentials to use for GCP" - type: string - -outputs: - credentials-file: - description: "Credentials file generated for GCP" - value: ${{ steps.output.outputs.credentials-file }} - -runs: - using: 'composite' - steps: - - name: Authenticate to Google Cloud - id: auth - uses: google-github-actions/auth@ef5d53e30bbcd8d0836f4288f5e50ff3e086997d # v1.0.0 - with: - credentials_json: ${{ inputs.credentials }} - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@d51b5346f85640ec2aa2fa057354d2b82c2fcbce # v1.0.1 - - name: Use gcloud CLI - shell: bash - run: gcloud info - - - name: Store credentials path - shell: bash - run: echo "CREDENTIALS_FILE=${{ steps.auth.outputs.credentials_file_path }}" >> $GITHUB_ENV - - - name: Fix credentials path (Windows) - if: ${{ runner.os == 'Windows' }} - # Bash commands in windows don't like the backslash in the file path. - # Assume we're running in the root directory and grab the base name. - shell: bash - run: echo "CREDENTIALS_FILE="$(basename ${CREDENTIALS_FILE//\\//}) >> $GITHUB_ENV - - - name: Output credentials file - id: output - shell: bash - run: echo "credentials-file=${{ env.CREDENTIALS_FILE }}" >> $GITHUB_OUTPUT diff --git a/.github/actions/internal/repository-cache-restore/action.yml b/.github/actions/internal/repository-cache-restore/action.yml deleted file mode 100644 index a5b4a0a924..0000000000 --- a/.github/actions/internal/repository-cache-restore/action.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Restore Repository Cache -description: Restore the Bazel repository cache from our github action cache -inputs: - bazel-cache: - required: true - description: A unique path for the Bazel cache. - type: string - -# By design, these actions will restore the latest cache for this branch/os, -# and only save a new version if something has changed. Initially this will -# cause a lot of churn, since each test has a slightly different set of -# repositories to download. Over time though, since we don't upload no-op -# changes, this should converge to a stable set of 3 caches per branch. Every -# run will update the current cache with a new test's repositories, until there -# are no unique ones left. -# -# This saves asymptotic space, since each one of these can get up to ~500 MB -# and Github prunes the cache after 10 GB. -runs: - using: 'composite' - steps: - - name: Setup Bazel repository cache variables - shell: bash - run: | - REPOSITORY_CACHE_BASE=repository-cache-${{ github.base_ref || github.ref_name }}-${{ runner.os }} - echo "REPOSITORY_CACHE_BASE=$REPOSITORY_CACHE_BASE" >> $GITHUB_ENV - echo "REPOSITORY_CACHE_NAME=$REPOSITORY_CACHE_BASE-${{ inputs.bazel-cache}}-${{ github.sha }}" >> $GITHUB_ENV - echo "REPOSITORY_CACHE_PATH=.repository-cache" >> $GITHUB_ENV - - - name: Restore Bazel repository cache - id: restore-cache - uses: actions/cache/restore@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 - with: - path: ${{ github.workspace }}/${{ env.REPOSITORY_CACHE_PATH }} - key: ${{ env.REPOSITORY_CACHE_NAME }} - restore-keys: ${{ env.REPOSITORY_CACHE_BASE }} - - - name: Initialize BAZEL environment variable - if: ${{ steps.restore-cache.cache-hit }} - shell: bash - run: echo "REPOSITORY_CACHE_HASH=${{ hashFiles(format('{0}/**', env.REPOSITORY_CACHE_PATH)) }}" >> $GITHUB_ENV diff --git a/.github/actions/internal/repository-cache-save/action.yml b/.github/actions/internal/repository-cache-save/action.yml deleted file mode 100644 index 1324b2bf21..0000000000 --- a/.github/actions/internal/repository-cache-save/action.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Restore Repository Cache -description: Restore the Bazel repository cache from our github action cache - -# Note: this action will only work if repository-cache-restore has already -# been called. All bazel actions should specify the repository_cache parameter -# using REPOSITORY_CACHE_PATH. -# -# We intentionally upload to REPOSITORY_CACHE_BASE to prevent a flood of new -# caches on any change. Only 1 job per os in each test run will be allowed to -# update the cache because they're all trying to write to the same location. -runs: - using: 'composite' - steps: - - name: Save modified Bazel repository cache - if: ${{ env.REPOSITORY_CACHE_HASH != hashFiles(format('{0}/**', env.REPOSITORY_CACHE_PATH)) }} - uses: actions/cache/save@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 - with: - path: ${{ github.workspace }}/${{ env.REPOSITORY_CACHE_PATH }} - key: ${{ env.REPOSITORY_CACHE_BASE }}-${{ github.sha }} diff --git a/.github/actions/internal/setup-runner/action.yml b/.github/actions/internal/setup-runner/action.yml deleted file mode 100644 index 64afdc3c7c..0000000000 --- a/.github/actions/internal/setup-runner/action.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Setup CI Runner -# TODO(b/267357823) Consider moving this to it's own repository so we can -# include the call to actions/checkout. -description: Setup any platform-specific adjustments we need to make for CI -runs: - using: 'composite' - steps: - - name: Fix Windows line breaks - if: runner.os == 'Windows' - shell: bash - run: find . -type f -print0 | xargs -0 d2u 2>/dev/null - - - name: Install bazelrc files - shell: bash - run: | - cp ci/*.bazelrc . - cp -f ${{ runner.os }}.bazelrc .bazelrc diff --git a/.github/workflows/staleness_check.yml b/.github/workflows/staleness_check.yml index 63c4cf39ce..9b9c4e3fbe 100644 --- a/.github/workflows/staleness_check.yml +++ b/.github/workflows/staleness_check.yml @@ -24,7 +24,7 @@ jobs: ref: ${{ matrix.branch}} - name: Run all staleness tests - uses: ./.github/actions/bazel + uses: protocolbuffers/protobuf-ci/bazel@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: staleness_check/${{ matrix.branch}}_${{ matrix.os.value }} diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index f846d80a8d..1a63692a59 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -44,7 +44,7 @@ jobs: with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: ./.github/actions/bazel-docker + uses: protocolbuffers/protobuf-ci/bazel-docker@v1 with: image: ${{ matrix.image }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -84,12 +84,12 @@ jobs: ref: ${{ inputs.safe-checkout }} - name: Setup ccache - uses: ./.github/actions/ccache + uses: protocolbuffers/protobuf-ci/ccache@v1 with: cache-prefix: linux-cmake-${{ matrix.name }} - name: Run tests - uses: ./.github/actions/docker + uses: protocolbuffers/protobuf-ci/docker@v1 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:e8282f42b1c1c0e6227d746e861954fd6fad0bd2e7424e6ceb23c249289a4016 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -106,12 +106,12 @@ jobs: ref: ${{ inputs.safe-checkout }} - name: Setup ccache - uses: ./.github/actions/ccache + uses: protocolbuffers/protobuf-ci/ccache@v1 with: cache-prefix: linux-cmake-install - name: Run tests - uses: ./.github/actions/docker + uses: protocolbuffers/protobuf-ci/docker@v1 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:e8282f42b1c1c0e6227d746e861954fd6fad0bd2e7424e6ceb23c249289a4016 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -135,12 +135,12 @@ jobs: ref: ${{ inputs.safe-checkout }} - name: Setup ccache - uses: ./.github/actions/ccache + uses: protocolbuffers/protobuf-ci/ccache@v1 with: cache-prefix: linux-cmake-32-bit - name: Run tests - uses: ./.github/actions/docker + uses: protocolbuffers/protobuf-ci/docker@v1 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:f99f051daa8b12f4ebad5927f389bc71372f771ab080290ab451cbaf1648f9ea platform: linux/386 @@ -173,7 +173,7 @@ jobs: with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: ./.github/actions/bazel + uses: protocolbuffers/protobuf-ci/bazel@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel: test ${{ matrix.bazel }} @@ -215,14 +215,14 @@ jobs: ref: ${{ inputs.safe-checkout }} - name: Setup ccache - uses: ./.github/actions/ccache + uses: protocolbuffers/protobuf-ci/ccache@v1 with: cache-prefix: ${{ matrix.name }} # Install phase. - name: Configure CMake for install if: matrix.install-flags - uses: ./.github/actions/bash + uses: protocolbuffers/protobuf-ci/bash@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: cmake . ${{ matrix.install-flags }} ${{ env.CCACHE_CMAKE_FLAGS }} @@ -244,7 +244,7 @@ jobs: run: cmake --build . --target clean && rm CMakeCache.txt - name: Configure CMake - uses: ./.github/actions/bash + uses: protocolbuffers/protobuf-ci/bash@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: cmake . ${{ matrix.flags }} ${{ env.CCACHE_CMAKE_FLAGS }} diff --git a/.github/workflows/test_csharp.yml b/.github/workflows/test_csharp.yml index 022d4f1f6c..d85867f3e9 100644 --- a/.github/workflows/test_csharp.yml +++ b/.github/workflows/test_csharp.yml @@ -18,7 +18,7 @@ jobs: with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: ./.github/actions/bazel-docker + uses: protocolbuffers/protobuf-ci/bazel-docker@v1 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/csharp:3.1.415-6.0.100-65526ea124d1034eac33e7c37cc6d65c5bef054f credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -57,7 +57,7 @@ jobs: - name: Build protobuf C# tests under x86_64 docker image # Tests are built "dotnet publish" because we want all the dependencies to the copied to the destination directory # (we want to avoid references to ~/.nuget that won't be available in the subsequent docker run) - uses: ./.github/actions/docker + uses: protocolbuffers/protobuf-ci/docker@v1 with: image: mcr.microsoft.com/dotnet/sdk:6.0.100-bullseye-slim credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -74,7 +74,7 @@ jobs: # running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user # otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity, # we just run map the user's home to a throwaway temporary directory - uses: ./.github/actions/docker + uses: protocolbuffers/protobuf-ci/docker@v1 with: image: mcr.microsoft.com/dotnet/sdk:6.0.100-bullseye-slim-arm64v8 skip-staleness-check: true diff --git a/.github/workflows/test_java.yml b/.github/workflows/test_java.yml index 1ad65537a1..bb6ea697e9 100644 --- a/.github/workflows/test_java.yml +++ b/.github/workflows/test_java.yml @@ -35,7 +35,7 @@ jobs: with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: ./.github/actions/bazel-docker + uses: protocolbuffers/protobuf-ci/bazel-docker@v1 with: image: ${{ matrix.image }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -51,7 +51,7 @@ jobs: with: ref: ${{ inputs.safe-checkout }} - name: Build protoc - uses: ./.github/actions/bazel + uses: protocolbuffers/protobuf-ci/bazel@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: java_linux_linkage diff --git a/.github/workflows/test_objectivec.yml b/.github/workflows/test_objectivec.yml index 4093b180b7..019bd4c95e 100644 --- a/.github/workflows/test_objectivec.yml +++ b/.github/workflows/test_objectivec.yml @@ -36,13 +36,13 @@ jobs: ref: ${{ inputs.safe-checkout }} - name: Setup ccache - uses: ./.github/actions/ccache + uses: protocolbuffers/protobuf-ci/ccache@v1 with: cache-prefix: objectivec_${{ matrix.platform }}_${{ matrix.xc_config }} support-modules: true - name: Run tests - uses: ./.github/actions/bash + uses: protocolbuffers/protobuf-ci/bash@v1 env: CC: ${{ github.workspace }}/ci/clang_wrapper CXX: ${{ github.workspace }}/ci/clang_wrapper++ @@ -78,7 +78,7 @@ jobs: with: ref: ${{ inputs.safe-checkout }} - name: Pod lib lint - uses: ./.github/actions/bash + uses: protocolbuffers/protobuf-ci/bash@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: | @@ -107,7 +107,7 @@ jobs: with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: ./.github/actions/bazel + uses: protocolbuffers/protobuf-ci/bazel@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel: test ${{ matrix.config.flags }} ${{ matrix.bazel_targets }} diff --git a/.github/workflows/test_php.yml b/.github/workflows/test_php.yml index 2d93d8f2b2..4b9a6d02ef 100644 --- a/.github/workflows/test_php.yml +++ b/.github/workflows/test_php.yml @@ -43,7 +43,7 @@ jobs: submodules: recursive ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: ./.github/actions/docker + uses: protocolbuffers/protobuf-ci/docker@v1 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-6e95c0e221e4bd52e3b4dc1398c6336985196931 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -78,13 +78,13 @@ jobs: - name: Cross compile protoc for i386 id: cross-compile - uses: ./.github/actions/cross-compile-protoc + uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} architecture: linux-i386 - name: Run tests - uses: ./.github/actions/docker + uses: protocolbuffers/protobuf-ci/docker@v1 with: image: ${{ env.image }} skip-staleness-check: true @@ -109,13 +109,13 @@ jobs: - name: Cross compile protoc for aarch64 id: cross-compile - uses: ./.github/actions/cross-compile-protoc + uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} architecture: linux-aarch64 - name: Run tests - uses: ./.github/actions/docker + uses: protocolbuffers/protobuf-ci/docker@v1 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php-aarch64:0cc100b6e03d14c1e8f71ae794dc162ed122fe31@sha256:77b70feba68dced1f0fd21b52a08d3d2e0c5c797bfe68435a0038ce87ecfd310 platform: linux/arm64 @@ -157,7 +157,7 @@ jobs: run: php --version | grep ${{ matrix.version }} || (echo "Invalid PHP version - $(php --version)" && exit 1) - name: Run tests - uses: ./.github/actions/bazel + uses: protocolbuffers/protobuf-ci/bazel@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: php_macos/${{ matrix.version }} @@ -171,7 +171,7 @@ jobs: popd - name: Run conformance tests - uses: ./.github/actions/bazel + uses: protocolbuffers/protobuf-ci/bazel@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: php_macos/${{ matrix.version }} diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml index 04fa54acc0..13e5575b24 100644 --- a/.github/workflows/test_python.yml +++ b/.github/workflows/test_python.yml @@ -37,7 +37,7 @@ jobs: with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: ./.github/actions/bazel-docker + uses: protocolbuffers/protobuf-ci/bazel-docker@v1 with: image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/python:{0}-65526ea124d1034eac33e7c37cc6d65c5bef054f', matrix.version) }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -82,7 +82,7 @@ jobs: source venv/bin/activate - name: Run tests - uses: ./.github/actions/bazel + uses: protocolbuffers/protobuf-ci/bazel@v1 env: KOKORO_PYTHON_VERSION: ${{ matrix.version }} with: diff --git a/.github/workflows/test_ruby.yml b/.github/workflows/test_ruby.yml index f5e3663fd2..0f9392f526 100644 --- a/.github/workflows/test_ruby.yml +++ b/.github/workflows/test_ruby.yml @@ -32,7 +32,7 @@ jobs: with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: ./.github/actions/bazel-docker + uses: protocolbuffers/protobuf-ci/bazel-docker@v1 with: image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:{0}-{1}-75e79f791b96e056086f43ace729cf3ebf9a9f5d', matrix.ruby, matrix.bazel) }} credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} @@ -50,13 +50,13 @@ jobs: - name: Cross compile protoc for aarch64 id: cross-compile - uses: ./.github/actions/cross-compile-protoc + uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} architecture: linux-aarch64 - name: Run tests - uses: ./.github/actions/docker + uses: protocolbuffers/protobuf-ci/docker@v1 with: image: arm64v8/ruby:2.7.3-buster skip-staleness-check: true @@ -94,7 +94,7 @@ jobs: run: ruby --version | grep ${{ matrix.version }} || (echo "Invalid Ruby version - $(ruby --version)" && exit 1) - name: Run tests - uses: ./.github/actions/bazel + uses: protocolbuffers/protobuf-ci/bazel@v1 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: ruby_macos/${{ matrix.version }} @@ -122,7 +122,7 @@ jobs: with: ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: ./.github/actions/bazel-docker + uses: protocolbuffers/protobuf-ci/bazel-docker@v1 with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:${{ matrix.ruby }}-${{ matrix.bazel }}-75e79f791b96e056086f43ace729cf3ebf9a9f5d credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}