Migrate our shared github actions to a separate repository.

This will make PRs from forked repositories significantly less painful, since they'll agree on which version of each action to use.  OTOH, we'll have a separate repo that needs to be maintained, and changes to it will need to be coordinated and versioned carefully.  This will likely need to be done less often though now that our infrastructure is stable.

PiperOrigin-RevId: 512117705
pull/11894/head
Mike Kruskal 2 years ago committed by Copybara-Service
parent 9ef627938a
commit ef8ebe6c19
  1. 27
      .github/actions/bash/action.yml
  2. 75
      .github/actions/bazel-docker/action.yml
  3. 134
      .github/actions/bazel/action.yml
  4. 63
      .github/actions/ccache/action.yml
  5. 38
      .github/actions/cross-compile-protoc/action.yml
  6. 50
      .github/actions/docker/action.yml
  7. 75
      .github/actions/internal/bazel-setup/action.yml
  8. 68
      .github/actions/internal/ccache-setup-windows/action.yml
  9. 62
      .github/actions/internal/docker-run/action.yml
  10. 42
      .github/actions/internal/gcloud-auth/action.yml
  11. 41
      .github/actions/internal/repository-cache-restore/action.yml
  12. 19
      .github/actions/internal/repository-cache-save/action.yml
  13. 17
      .github/actions/internal/setup-runner/action.yml
  14. 2
      .github/workflows/staleness_check.yml
  15. 22
      .github/workflows/test_cpp.yml
  16. 6
      .github/workflows/test_csharp.yml
  17. 4
      .github/workflows/test_java.yml
  18. 8
      .github/workflows/test_objectivec.yml
  19. 14
      .github/workflows/test_php.yml
  20. 4
      .github/workflows/test_python.yml
  21. 10
      .github/workflows/test_ruby.yml

@ -1,27 +0,0 @@
name: 'Non-Bazel Bash Run'
description: 'Run a bash script for Protobuf CI testing with a non-Bazel build system'
inputs:
credentials:
required: true
description: "The GCP credentials to use for reading the docker image"
type: string
command:
required: true
description: A command to run in the docker image
runs:
using: 'composite'
steps:
- name: Setup Runner
uses: ./.github/actions/internal/setup-runner
- name: Update stale files using Bazel
uses: ./.github/actions/bazel
with:
credentials: ${{ inputs.credentials }}
bazel-cache: regenerate-stale-files
bash: ./regenerate_stale_files.sh $BAZEL_FLAGS
- name: Run
shell: bash
run: ${{ inputs.command }}

@ -1,75 +0,0 @@
name: 'Docker Bazel Run'
description: 'Run a Bazel-based docker image for Protobuf CI testing'
inputs:
credentials:
required: true
description: "The GCP credentials to use for reading the docker image"
type: string
image:
required: false
default: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1
description: "The docker image to use"
type: string
bazel-cache:
required: true
description: >
A unique path for the Bazel cache. This will trigger the generation
of a BAZEL_CACHE environment variable inside the container that provides
the appropriate flags for any bazel command.
type: string
bazel:
required: false
description: "The Bazel command to run"
type: string
bash:
required: false
description: "A bash command to run. $BAZEL_FLAGS will be available to use for bazel runs."
type: string
runs:
using: 'composite'
steps:
- name: Authenticate
id: auth
uses: ./.github/actions/internal/gcloud-auth
with:
credentials: ${{ inputs.credentials }}
- name: Setup Runner
uses: ./.github/actions/internal/setup-runner
- name: Setup Bazel
id: bazel
uses: ./.github/actions/internal/bazel-setup
with:
credentials-file: /workspace/$(basename ${{ steps.auth.outputs.credentials-file }})
bazel-cache: ${{ inputs.bazel-cache }}
- name: Hook up repository Cache
shell: bash
run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --repository_cache='/workspace/${{ env.REPOSITORY_CACHE_PATH }}'" >> $GITHUB_ENV
- name: Validate inputs
if: ${{ (inputs.bash && inputs.bazel) || (!inputs.bash && !inputs.bazel) }}
shell: bash
run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1
- name: Run Bash Docker
uses: ./.github/actions/internal/docker-run
if: ${{ inputs.bash }}
with:
image: ${{ inputs.image }}
run-flags: --entrypoint "/bin/bash"
command: -l -c "${{ inputs.bash }}"
- name: Run Bazel Docker
uses: ./.github/actions/internal/docker-run
if: ${{ !inputs.bash }}
with:
image: ${{ inputs.image }}
command: ${{ inputs.bazel }} ${{ env.BAZEL_FLAGS }}
- name: Save Bazel repository cache
# Only allow repository cache updates during post-submits.
if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' }}
uses: ./.github/actions/internal/repository-cache-save

@ -1,134 +0,0 @@
name: 'Docker Bazel Run'
description: 'Run a Bazel-based docker image for Protobuf CI testing'
inputs:
credentials:
required: true
description: The GCP credentials to use for reading the docker image
type: string
bazel-cache:
required: true
description: >
A unique path for the Bazel cache. This will trigger the generation
of a BAZEL_CACHE environment variable inside the container that provides
the appropriate flags for any bazel command.
type: string
version:
required: false
description: A pinned Bazel version to use
default: '5.1.1'
type: string
bazel:
required: false
description: The Bazel command to run
type: string
bash:
required: false
description: >
A bash command to run. $BAZEL_FLAGS and $BAZEL_STARTUP_FLAGS will be
available to use for bazel runs.
type: string
runs:
using: 'composite'
steps:
- name: Authenticate
id: auth
uses: ./.github/actions/internal/gcloud-auth
with:
credentials: ${{ inputs.credentials }}
- name: Setup Runner
uses: ./.github/actions/internal/setup-runner
- name: Setup Bazel
id: bazel
uses: ./.github/actions/internal/bazel-setup
with:
credentials-file: ${{ steps.auth.outputs.credentials-file }}
bazel-cache: ${{ inputs.bazel-cache }}
- name: Get Linux bazelisk path
if: runner.os == 'Linux'
shell: bash
run: echo "BAZELISK_PATH=~/.cache/bazelisk" >> $GITHUB_ENV
- name: Get MacOS bazelisk path
if: runner.os == 'macOS'
shell: bash
run: echo "BAZELISK_PATH=~/Library/Caches/bazelisk" >> $GITHUB_ENV
- name: Get Windows bazelisk path
if: runner.os == 'Windows'
shell: bash
run: echo "BAZELISK_PATH=$LOCALAPPDATA\bazelisk" >> $GITHUB_ENV
- name: Cache Bazelisk
if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' }}
uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4
with:
path: ${{ env.BAZELISK_PATH }}
key: bazel-${{ runner.os }}-${{ inputs.version }}
- name: Restore Bazelisk
if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }}
uses: actions/cache/restore@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4
with:
path: ${{ env.BAZELISK_PATH }}
key: bazel-${{ runner.os }}-${{ inputs.version }}
- name: Hook up repository Cache
shell: bash
run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --repository_cache=$(pwd)/${{ env.REPOSITORY_CACHE_PATH }}" >> $GITHUB_ENV
- name: Validate inputs
if: ${{ (inputs.bash && inputs.bazel) || (!inputs.bash && !inputs.bazel) }}
shell: bash
run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1
- name: Pin Bazel version
shell: bash
run: echo "USE_BAZEL_VERSION=${{ inputs.version }}" >> $GITHUB_ENV
- name: Output Bazel version
shell: bash
run: bazelisk version
# Bazel has multiple Xcode calls with hardcoded timeouts. Many of these
# end up timing out on our github runners, causing flakes on every mac
# build that invoked Bazel. To work around this, we manually inoke these
# calls before running Bazel to make sure they end up in Xcode's cache for
# quicker runs later. All of these calls are obtained from xcrun calls in
# https://github.com/bazelbuild/bazel/blob/e8a69f5d5acaeb6af760631490ecbf73e8a04eeb/tools/cpp/osx_cc_configure.bzl.
# See https://github.com/bazelbuild/bazel/issues/17437 for more details.
# TODO(b/269503614) Remove this once Bazel provides an official solution.
- name: Warm up Xcode
if: ${{ runner.os == 'macOS' }}
shell: bash
run: |
mkdir -p mac_bazel_workaround
bazelisk ${{ steps.bazel.outputs.bazel-startup-flags }} build @bazel_tools//tools/osx:xcode_locator.m $BAZEL_FLAGS
XCODE_LOCATOR_FLAGS="--sdk macosx clang -mmacosx-version-min=10.9 -fobjc-arc -framework CoreServices -framework Foundation"
SINGLE_ARCH_COMPILE_FLAGS="--sdk macosx clang -mmacosx-version-min=10.9 -std=c++11 -lc++ -O3"
COMPILE_FLAGS="$SINGLE_ARCH_COMPILE_FLAGS -arch arm64 -arch x86_64 -Wl,-no_adhoc_codesign -Wl,-no_uuid -O3"
time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $XCODE_LOCATOR_FLAGS -o mac_bazel_workaround/xcode-locator-bin $(bazel info output_base)/external/bazel_tools/tools/osx/xcode_locator.m
time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $SINGLE_ARCH_COMPILE_FLAGS -o mac_bazel_workaround/libtool_check_unique $(bazel info output_base)/external/bazel_tools/tools/objc/libtool_check_unique.cc
time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $COMPILE_FLAGS -o mac_bazel_workaround/libtool_check_unique $(bazel info output_base)/external/bazel_tools/tools/objc/libtool_check_unique.cc
time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $SINGLE_ARCH_COMPILE_FLAGS -o mac_bazel_workaround/wrapped_clang $(bazel info output_base)/external/bazel_tools/tools/osx/crosstool/wrapped_clang.cc
time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $COMPILE_FLAGS -o mac_bazel_workaround/wrapped_clang $(bazel info output_base)/external/bazel_tools/tools/osx/crosstool/wrapped_clang.cc
- name: Run Bash
if: ${{ inputs.bash }}
run: ${{ inputs.bash }}
shell: bash
- name: Run Bazel
if: ${{ !inputs.bash }}
run: >-
bazelisk ${{ steps.bazel.outputs.bazel-startup-flags }}
${{ inputs.bazel }} $BAZEL_FLAGS
shell: bash
- name: Save Bazel repository cache
# Only allow repository cache updates during post-submits.
if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target'}}
uses: ./.github/actions/internal/repository-cache-save

@ -1,63 +0,0 @@
name: 'CCache Setup'
description: 'Run a Bazel-based docker image for Protobuf CI testing'
inputs:
cache-prefix:
required: true
description: A unique prefix to prevent cache pollution
type: string
support-modules:
required: false
description: Whether or not we need to support modules. This can result in extra cache misses.
runs:
using: 'composite'
steps:
- name: Configure ccache environment variables
shell: bash
run: |
echo "CCACHE_BASEDIR=${{ github.workspace }}" >> $GITHUB_ENV
echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV
echo "CCACHE_COMPRESS=true" >> $GITHUB_ENV
echo "CCACHE_COMPRESSLEVEL=5" >> $GITHUB_ENV
echo "CCACHE_MAXSIZE=100M" >> $GITHUB_ENV
echo "CCACHE_SLOPPINESS=clang_index_store,include_file_ctime,include_file_mtime,file_macro,time_macros" >> $GITHUB_ENV
echo "CCACHE_DIRECT=true" >> $GITHUB_ENV
echo "CCACHE_CMAKE_FLAGS=-Dprotobuf_ALLOW_CCACHE=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" >> $GITHUB_ENV
- name: Setup ccache on Windows
if: ${{ runner.os == 'Windows' }}
uses: ./.github/actions/internal/ccache-setup-windows
- name: Setup ccache on Mac
if: ${{ runner.os == 'macOS' }}
shell: bash
run: brew install ccache
- name: Setup fixed path ccache caching
uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4
with:
path: |
.ccache/**
!.ccache/lock
!.ccache/tmp
# Always push to a cache key unique to this commit.
key: ${{ format('ccache-{0}-{1}-{2}', inputs.cache-prefix, github.ref_name, github.sha) }}
# Select a cache to restore from with the follow order of preference:
# 1) The exact same commit we're running over
# 2) The latest cache from the current ref branch
# 3) The latest push to the base ref of a pull request
restore-keys: |
${{ format('ccache-{0}-{1}-{2}', inputs.cache-prefix, github.ref_name, github.sha) }}
${{ format('ccache-{0}-{1}', inputs.cache-prefix, github.ref_name) }}
${{ format('ccache-{0}-{1}', inputs.cache-prefix, github.base_ref) }}
- name: Enable module support
if: ${{ inputs.support-modules }}
shell: bash
run: |
echo "CCACHE_SLOPPINESS=$CCACHE_SLOPPINESS,modules" >> $GITHUB_ENV
echo "CCACHE_DEPEND=true" >> $GITHUB_ENV
- name: Zero out ccache
if: ${{ runner.os != 'Linux' }}
shell: bash
run: ccache -z

@ -1,38 +0,0 @@
name: 'Cross-compile protoc'
description: 'Produces a cross-compiled protoc binary for a target architecture'
inputs:
credentials:
required: true
description: The GCP credentials to use for reading the docker image
type: string
architecture:
required: true
description: The target architecture to build for
type: string
outputs:
protoc:
description: "Cross-compiled protoc location. Also output to $PROTOC"
value: ${{ steps.output.outputs.protoc }}
runs:
using: 'composite'
steps:
- name: Cross compile protoc for ${{ inputs.architecture }}
uses: ./.github/actions/bazel-docker
with:
credentials: ${{ inputs.credentials }}
bazel-cache: xcompile-protoc/${{ inputs.architecture }}
bash: |
bazel build //:protoc --config=${{ inputs.architecture }} $BAZEL_FLAGS
cp bazel-bin/protoc .
- name: Set protoc environment variable
shell: bash
run: echo "PROTOC=protoc-${{ inputs.architecture }}" >> $GITHUB_ENV
- name: Extract binary
id: output
shell: bash
run: |
mv protoc $PROTOC
echo "protoc=$PROTOC" >> $GITHUB_OUTPUT

@ -1,50 +0,0 @@
name: 'Docker Non-Bazel Run'
description: 'Run a docker image for Protobuf CI testing with a non-Bazel build system'
inputs:
credentials:
required: true
description: "The GCP credentials to use for reading the docker image"
type: string
command:
required: true
description: A command to run in the docker image
image:
required: false
default: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1
description: "The docker image to use"
type: string
platform:
required: false
description: "The platform to use for the image"
type: string
skip-staleness-check:
required: false
description: "Skip staleness checks"
type: boolean
runs:
using: 'composite'
steps:
- name: Setup Runner
uses: ./.github/actions/internal/setup-runner
- name: Update stale files using Bazel
if: ${{ !inputs.skip-staleness-check }}
uses: ./.github/actions/bazel-docker
with:
image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1
credentials: ${{ inputs.credentials }}
bazel-cache: regenerate-stale-files
bash: ./regenerate_stale_files.sh $BAZEL_FLAGS
- name: Generate docker flags
if: inputs.platform
shell: bash
run: echo "DOCKER_RUN_FLAGS=--platform ${{inputs.platform}}" >> $GITHUB_ENV
- name: Run Docker
uses: ./.github/actions/internal/docker-run
with:
image: ${{ inputs.image }}
command: ${{ inputs.command }}
run-flags: ${{ env.DOCKER_RUN_FLAGS }}

@ -1,75 +0,0 @@
name: Setup Bazel
description: Setup a Bazel environment for Protobuf CI testing
inputs:
credentials-file:
required: true
description: The GCP credentials file to use for caching
type: string
bazel-cache:
required: true
description: A unique path for the Bazel cache.
type: string
outputs:
bazel-flags:
description: Bazel flags that should be sent to all Bazel invocations
value: ${{ steps.output.outputs.bazel-flags }}
bazel-startup-flags:
description: Bazel startup flags that should be sent to all Bazel invocations
value: ${{ steps.output.outputs.bazel-startup-flags }}
runs:
using: 'composite'
steps:
- name: Initialize BAZEL environment variable
shell: bash
run: echo "BAZEL=bazelisk" >> $GITHUB_ENV
- name: Initialize Windows startup flags
if: runner.os == 'Windows'
shell: bash
run: echo "BAZEL_STARTUP_FLAGS=--output_user_root=C:/tmp --windows_enable_symlinks" >> $GITHUB_ENV
- name: Initialize Bazel flags
shell: bash
run: echo "BAZEL_FLAGS=--keep_going --test_output=errors --test_timeout=600" >> $GITHUB_ENV
- name: Initialize Windows-specific Bazel flags
if: runner.os == 'Windows'
shell: bash
run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --enable_runfiles" >> $GITHUB_ENV
- name: Initialize MacOS-specific Bazel flags
if: runner.os == 'macOS'
shell: bash
run: |
echo "BAZEL_FLAGS=$BAZEL_FLAGS --xcode_version_config=//.github:host_xcodes" >> $GITHUB_ENV
echo "DEVELOPER_DIR=${{ env.DEVELOPER_DIR || '/Applications/Xcode_14.1.app/Contents/Developer' }}" >> $GITHUB_ENV
- name: Configure Bazel caching
# Skip bazel cache for local act runs due to issue with credential files
# and nested docker images
if: ${{ inputs.bazel-cache && !github.event.act_local_test }}
shell: bash
run: >-
echo "BAZEL_FLAGS=$BAZEL_FLAGS
--google_credentials=${{ inputs.credentials-file }}
--remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/protobuf/gha/${{ inputs.bazel-cache }}" >> $GITHUB_ENV
- name: Configure Bazel cache writing
# External runs should never write to our caches.
if: ${{ github.event_name != 'pull_request_target' && inputs.bazel-cache && !github.event.act_local_test }}
shell: bash
run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --remote_upload_local_results" >> $GITHUB_ENV
- name: Output Bazel flags
id: output
shell: bash
run: |
echo "bazel-flags=$BAZEL_FLAGS" >> $GITHUB_OUTPUT
echo "bazel-startup-flags=$BAZEL_STARTUP_FLAGS" >> $GITHUB_OUTPUT
- name: Restore Bazel repository cache
uses: ./.github/actions/internal/repository-cache-restore
with:
bazel-cache: ${{ inputs.bazel-cache }}

@ -1,68 +0,0 @@
name: 'CCache Setup'
description: 'Setup ccache for us in Windows CI'
inputs:
ccache-version:
required: false
default: '4.7.4'
description: A pinned version of ccache
type: string
runs:
using: 'composite'
steps:
- name: Setup MSVC
uses: ilammy/msvc-dev-cmd@cec98b9d092141f74527d0afa6feb2af698cfe89 # v1.12.1
with:
arch: x64
vsversion: '2019'
- name: Setup ccache path
shell: bash
run: |
echo "CCACHE_EXE_PATH=$LOCALAPPDATA\ccache-${{ inputs.ccache-version }}-windows-x86_64" >> $GITHUB_ENV
echo "$LOCALAPPDATA\ccache-${{ inputs.ccache-version }}-windows-x86_64" >> $GITHUB_PATH
- name: Add ccache to Powershell path
shell: pwsh
run: echo "%LOCALAPPDATA%\ccache-${{ inputs.ccache-version }}-windows-x86_64" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
- name: Setup caching of ccache download
if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' }}
id: ccache-cache
uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4
with:
path: ${{ env.CCACHE_EXE_PATH }}
key: ccache-exe-${{ inputs.ccache-version }}
- name: Restore ccache download
if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }}
id: ccache-restore
uses: actions/cache/restore@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4
with:
path: ${{ env.CCACHE_EXE_PATH }}
key: ccache-exe-${{ inputs.ccache-version }}
- name: Download ccache
shell: bash
if: ${{ steps.ccache-cache.outputs.cache-hit != 'true' && steps.ccache-restore.outputs.cache-hit != 'true'}}
run: |
cd $LOCALAPPDATA
curl -kLSs "https://github.com/ccache/ccache/releases/download/v${{ inputs.ccache-version }}/ccache-${{ inputs.ccache-version }}-windows-x86_64.zip" -o ccache.zip
unzip ccache.zip
rm ccache.zip
ccache --version
- name: Configure ccache environment variables
shell: pwsh
run: |
Write-Host $Env:GITHUB_REF
$cllocation = (Get-Command cl.exe).Path
echo "CCACHE_COMPILER=$cllocation" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
echo "CCACHE_COMPILERTYPE=msvc" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
- name: Configure Windows-specific ccache environment variables
shell: bash
# Windows caches are about 2x larger than other platforms.
run: |
echo "CCACHE_COMPRESSLEVEL=10" >> $GITHUB_ENV
echo "CCACHE_MAXSIZE=200M" >> $GITHUB_ENV

@ -1,62 +0,0 @@
name: 'Run Docker'
description: 'Run a docker image for Protobuf CI testing'
inputs:
image:
required: true
description: "The docker image to use"
type: string
command:
required: true
description: "A raw docker command to run"
type: string
run-flags:
required: false
description: "Additional flags to pass to docker run"
type: string
# WARNING: loading from cache appears to be slower than pull!
docker-cache:
required: false
description: "Enabled caching of pulled docker images."
runs:
using: 'composite'
steps:
- name: Authenticate for GAR use
shell: bash
run: gcloud auth configure-docker -q us-docker.pkg.dev
- name: Setup QEMU for possible emulation
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2.1.0
- name: Check docker cache
if: ${{ inputs.docker-cache }}
id: check-docker-cache
uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4
with:
path: ci/docker/
key: ${{ inputs.image }}
- name: Pull and store if cache miss
shell: bash
if: ${{ inputs.docker-cache && steps.check-docker-cache.outputs.cache-hit != 'true' }}
run: >
time docker pull -q ${{ inputs.image }} &&
mkdir -p ci/docker/$(dirname ${{ inputs.image }}) &&
time docker image save ${{ inputs.image }} --output ./ci/docker/${{ inputs.image }}.tar
- name: Use the cached image on cache hit
shell: bash
if: ${{ inputs.docker-cache && steps.check-docker-cache.outputs.cache-hit == 'true' }}
run: time docker image load --input ./ci/docker/${{ inputs.image }}.tar
- name: Pull fresh docker image
shell: bash
if: ${{ !inputs.docker-cache }}
run: time docker pull -q ${{ inputs.image }}
- name: Run docker
shell: bash
run: >
time docker run ${{ inputs.run-flags}} -v${{ github.workspace }}:/workspace
${{ inputs.image }} ${{ inputs.command }}

@ -1,42 +0,0 @@
name: 'Authenticate for GCP'
description: 'Authenticate a workflow for Protobuf CI testing'
inputs:
credentials:
required: true
description: "The GCP credentials to use for GCP"
type: string
outputs:
credentials-file:
description: "Credentials file generated for GCP"
value: ${{ steps.output.outputs.credentials-file }}
runs:
using: 'composite'
steps:
- name: Authenticate to Google Cloud
id: auth
uses: google-github-actions/auth@ef5d53e30bbcd8d0836f4288f5e50ff3e086997d # v1.0.0
with:
credentials_json: ${{ inputs.credentials }}
- name: Set up Cloud SDK
uses: google-github-actions/setup-gcloud@d51b5346f85640ec2aa2fa057354d2b82c2fcbce # v1.0.1
- name: Use gcloud CLI
shell: bash
run: gcloud info
- name: Store credentials path
shell: bash
run: echo "CREDENTIALS_FILE=${{ steps.auth.outputs.credentials_file_path }}" >> $GITHUB_ENV
- name: Fix credentials path (Windows)
if: ${{ runner.os == 'Windows' }}
# Bash commands in windows don't like the backslash in the file path.
# Assume we're running in the root directory and grab the base name.
shell: bash
run: echo "CREDENTIALS_FILE="$(basename ${CREDENTIALS_FILE//\\//}) >> $GITHUB_ENV
- name: Output credentials file
id: output
shell: bash
run: echo "credentials-file=${{ env.CREDENTIALS_FILE }}" >> $GITHUB_OUTPUT

@ -1,41 +0,0 @@
name: Restore Repository Cache
description: Restore the Bazel repository cache from our github action cache
inputs:
bazel-cache:
required: true
description: A unique path for the Bazel cache.
type: string
# By design, these actions will restore the latest cache for this branch/os,
# and only save a new version if something has changed. Initially this will
# cause a lot of churn, since each test has a slightly different set of
# repositories to download. Over time though, since we don't upload no-op
# changes, this should converge to a stable set of 3 caches per branch. Every
# run will update the current cache with a new test's repositories, until there
# are no unique ones left.
#
# This saves asymptotic space, since each one of these can get up to ~500 MB
# and Github prunes the cache after 10 GB.
runs:
using: 'composite'
steps:
- name: Setup Bazel repository cache variables
shell: bash
run: |
REPOSITORY_CACHE_BASE=repository-cache-${{ github.base_ref || github.ref_name }}-${{ runner.os }}
echo "REPOSITORY_CACHE_BASE=$REPOSITORY_CACHE_BASE" >> $GITHUB_ENV
echo "REPOSITORY_CACHE_NAME=$REPOSITORY_CACHE_BASE-${{ inputs.bazel-cache}}-${{ github.sha }}" >> $GITHUB_ENV
echo "REPOSITORY_CACHE_PATH=.repository-cache" >> $GITHUB_ENV
- name: Restore Bazel repository cache
id: restore-cache
uses: actions/cache/restore@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4
with:
path: ${{ github.workspace }}/${{ env.REPOSITORY_CACHE_PATH }}
key: ${{ env.REPOSITORY_CACHE_NAME }}
restore-keys: ${{ env.REPOSITORY_CACHE_BASE }}
- name: Initialize BAZEL environment variable
if: ${{ steps.restore-cache.cache-hit }}
shell: bash
run: echo "REPOSITORY_CACHE_HASH=${{ hashFiles(format('{0}/**', env.REPOSITORY_CACHE_PATH)) }}" >> $GITHUB_ENV

@ -1,19 +0,0 @@
name: Restore Repository Cache
description: Restore the Bazel repository cache from our github action cache
# Note: this action will only work if repository-cache-restore has already
# been called. All bazel actions should specify the repository_cache parameter
# using REPOSITORY_CACHE_PATH.
#
# We intentionally upload to REPOSITORY_CACHE_BASE to prevent a flood of new
# caches on any change. Only 1 job per os in each test run will be allowed to
# update the cache because they're all trying to write to the same location.
runs:
using: 'composite'
steps:
- name: Save modified Bazel repository cache
if: ${{ env.REPOSITORY_CACHE_HASH != hashFiles(format('{0}/**', env.REPOSITORY_CACHE_PATH)) }}
uses: actions/cache/save@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4
with:
path: ${{ github.workspace }}/${{ env.REPOSITORY_CACHE_PATH }}
key: ${{ env.REPOSITORY_CACHE_BASE }}-${{ github.sha }}

@ -1,17 +0,0 @@
name: Setup CI Runner
# TODO(b/267357823) Consider moving this to it's own repository so we can
# include the call to actions/checkout.
description: Setup any platform-specific adjustments we need to make for CI
runs:
using: 'composite'
steps:
- name: Fix Windows line breaks
if: runner.os == 'Windows'
shell: bash
run: find . -type f -print0 | xargs -0 d2u 2>/dev/null
- name: Install bazelrc files
shell: bash
run: |
cp ci/*.bazelrc .
cp -f ${{ runner.os }}.bazelrc .bazelrc

@ -24,7 +24,7 @@ jobs:
ref: ${{ matrix.branch}}
- name: Run all staleness tests
uses: ./.github/actions/bazel
uses: protocolbuffers/protobuf-ci/bazel@v1
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: staleness_check/${{ matrix.branch}}_${{ matrix.os.value }}

@ -44,7 +44,7 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: ./.github/actions/bazel-docker
uses: protocolbuffers/protobuf-ci/bazel-docker@v1
with:
image: ${{ matrix.image }}
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
@ -84,12 +84,12 @@ jobs:
ref: ${{ inputs.safe-checkout }}
- name: Setup ccache
uses: ./.github/actions/ccache
uses: protocolbuffers/protobuf-ci/ccache@v1
with:
cache-prefix: linux-cmake-${{ matrix.name }}
- name: Run tests
uses: ./.github/actions/docker
uses: protocolbuffers/protobuf-ci/docker@v1
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:e8282f42b1c1c0e6227d746e861954fd6fad0bd2e7424e6ceb23c249289a4016
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
@ -106,12 +106,12 @@ jobs:
ref: ${{ inputs.safe-checkout }}
- name: Setup ccache
uses: ./.github/actions/ccache
uses: protocolbuffers/protobuf-ci/ccache@v1
with:
cache-prefix: linux-cmake-install
- name: Run tests
uses: ./.github/actions/docker
uses: protocolbuffers/protobuf-ci/docker@v1
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:e8282f42b1c1c0e6227d746e861954fd6fad0bd2e7424e6ceb23c249289a4016
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
@ -135,12 +135,12 @@ jobs:
ref: ${{ inputs.safe-checkout }}
- name: Setup ccache
uses: ./.github/actions/ccache
uses: protocolbuffers/protobuf-ci/ccache@v1
with:
cache-prefix: linux-cmake-32-bit
- name: Run tests
uses: ./.github/actions/docker
uses: protocolbuffers/protobuf-ci/docker@v1
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:f99f051daa8b12f4ebad5927f389bc71372f771ab080290ab451cbaf1648f9ea
platform: linux/386
@ -173,7 +173,7 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: ./.github/actions/bazel
uses: protocolbuffers/protobuf-ci/bazel@v1
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel: test ${{ matrix.bazel }}
@ -215,14 +215,14 @@ jobs:
ref: ${{ inputs.safe-checkout }}
- name: Setup ccache
uses: ./.github/actions/ccache
uses: protocolbuffers/protobuf-ci/ccache@v1
with:
cache-prefix: ${{ matrix.name }}
# Install phase.
- name: Configure CMake for install
if: matrix.install-flags
uses: ./.github/actions/bash
uses: protocolbuffers/protobuf-ci/bash@v1
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
command: cmake . ${{ matrix.install-flags }} ${{ env.CCACHE_CMAKE_FLAGS }}
@ -244,7 +244,7 @@ jobs:
run: cmake --build . --target clean && rm CMakeCache.txt
- name: Configure CMake
uses: ./.github/actions/bash
uses: protocolbuffers/protobuf-ci/bash@v1
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
command: cmake . ${{ matrix.flags }} ${{ env.CCACHE_CMAKE_FLAGS }}

@ -18,7 +18,7 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: ./.github/actions/bazel-docker
uses: protocolbuffers/protobuf-ci/bazel-docker@v1
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/csharp:3.1.415-6.0.100-65526ea124d1034eac33e7c37cc6d65c5bef054f
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
@ -57,7 +57,7 @@ jobs:
- name: Build protobuf C# tests under x86_64 docker image
# Tests are built "dotnet publish" because we want all the dependencies to the copied to the destination directory
# (we want to avoid references to ~/.nuget that won't be available in the subsequent docker run)
uses: ./.github/actions/docker
uses: protocolbuffers/protobuf-ci/docker@v1
with:
image: mcr.microsoft.com/dotnet/sdk:6.0.100-bullseye-slim
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
@ -74,7 +74,7 @@ jobs:
# running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user
# otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity,
# we just run map the user's home to a throwaway temporary directory
uses: ./.github/actions/docker
uses: protocolbuffers/protobuf-ci/docker@v1
with:
image: mcr.microsoft.com/dotnet/sdk:6.0.100-bullseye-slim-arm64v8
skip-staleness-check: true

@ -35,7 +35,7 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: ./.github/actions/bazel-docker
uses: protocolbuffers/protobuf-ci/bazel-docker@v1
with:
image: ${{ matrix.image }}
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
@ -51,7 +51,7 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Build protoc
uses: ./.github/actions/bazel
uses: protocolbuffers/protobuf-ci/bazel@v1
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: java_linux_linkage

@ -36,13 +36,13 @@ jobs:
ref: ${{ inputs.safe-checkout }}
- name: Setup ccache
uses: ./.github/actions/ccache
uses: protocolbuffers/protobuf-ci/ccache@v1
with:
cache-prefix: objectivec_${{ matrix.platform }}_${{ matrix.xc_config }}
support-modules: true
- name: Run tests
uses: ./.github/actions/bash
uses: protocolbuffers/protobuf-ci/bash@v1
env:
CC: ${{ github.workspace }}/ci/clang_wrapper
CXX: ${{ github.workspace }}/ci/clang_wrapper++
@ -78,7 +78,7 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Pod lib lint
uses: ./.github/actions/bash
uses: protocolbuffers/protobuf-ci/bash@v1
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
command: |
@ -107,7 +107,7 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: ./.github/actions/bazel
uses: protocolbuffers/protobuf-ci/bazel@v1
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel: test ${{ matrix.config.flags }} ${{ matrix.bazel_targets }}

@ -43,7 +43,7 @@ jobs:
submodules: recursive
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: ./.github/actions/docker
uses: protocolbuffers/protobuf-ci/docker@v1
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-6e95c0e221e4bd52e3b4dc1398c6336985196931
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
@ -78,13 +78,13 @@ jobs:
- name: Cross compile protoc for i386
id: cross-compile
uses: ./.github/actions/cross-compile-protoc
uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v1
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
architecture: linux-i386
- name: Run tests
uses: ./.github/actions/docker
uses: protocolbuffers/protobuf-ci/docker@v1
with:
image: ${{ env.image }}
skip-staleness-check: true
@ -109,13 +109,13 @@ jobs:
- name: Cross compile protoc for aarch64
id: cross-compile
uses: ./.github/actions/cross-compile-protoc
uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v1
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
architecture: linux-aarch64
- name: Run tests
uses: ./.github/actions/docker
uses: protocolbuffers/protobuf-ci/docker@v1
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php-aarch64:0cc100b6e03d14c1e8f71ae794dc162ed122fe31@sha256:77b70feba68dced1f0fd21b52a08d3d2e0c5c797bfe68435a0038ce87ecfd310
platform: linux/arm64
@ -157,7 +157,7 @@ jobs:
run: php --version | grep ${{ matrix.version }} || (echo "Invalid PHP version - $(php --version)" && exit 1)
- name: Run tests
uses: ./.github/actions/bazel
uses: protocolbuffers/protobuf-ci/bazel@v1
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: php_macos/${{ matrix.version }}
@ -171,7 +171,7 @@ jobs:
popd
- name: Run conformance tests
uses: ./.github/actions/bazel
uses: protocolbuffers/protobuf-ci/bazel@v1
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: php_macos/${{ matrix.version }}

@ -37,7 +37,7 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: ./.github/actions/bazel-docker
uses: protocolbuffers/protobuf-ci/bazel-docker@v1
with:
image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/python:{0}-65526ea124d1034eac33e7c37cc6d65c5bef054f', matrix.version) }}
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
@ -82,7 +82,7 @@ jobs:
source venv/bin/activate
- name: Run tests
uses: ./.github/actions/bazel
uses: protocolbuffers/protobuf-ci/bazel@v1
env:
KOKORO_PYTHON_VERSION: ${{ matrix.version }}
with:

@ -32,7 +32,7 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: ./.github/actions/bazel-docker
uses: protocolbuffers/protobuf-ci/bazel-docker@v1
with:
image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:{0}-{1}-75e79f791b96e056086f43ace729cf3ebf9a9f5d', matrix.ruby, matrix.bazel) }}
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
@ -50,13 +50,13 @@ jobs:
- name: Cross compile protoc for aarch64
id: cross-compile
uses: ./.github/actions/cross-compile-protoc
uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v1
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
architecture: linux-aarch64
- name: Run tests
uses: ./.github/actions/docker
uses: protocolbuffers/protobuf-ci/docker@v1
with:
image: arm64v8/ruby:2.7.3-buster
skip-staleness-check: true
@ -94,7 +94,7 @@ jobs:
run: ruby --version | grep ${{ matrix.version }} || (echo "Invalid Ruby version - $(ruby --version)" && exit 1)
- name: Run tests
uses: ./.github/actions/bazel
uses: protocolbuffers/protobuf-ci/bazel@v1
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: ruby_macos/${{ matrix.version }}
@ -122,7 +122,7 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: ./.github/actions/bazel-docker
uses: protocolbuffers/protobuf-ci/bazel-docker@v1
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:${{ matrix.ruby }}-${{ matrix.bazel }}-75e79f791b96e056086f43ace729cf3ebf9a9f5d
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}

Loading…
Cancel
Save