Migrate to composite GHA actions (#11718)
This migrates from reusable workflows to composite GHA actions. This has the following advantages: 1) We can split them into smaller, easier to reason about chunks 2) We can reuse them more freely between each other and workflows 3) They don't complicate the job names 4) In theory they'll be easier to test in isolation. While composite actions can't be unit-tested, we can easily break them up into nodejs or docker actions that can be. As a proof of concept, some of our non-Bazel tests are migrated to GHA as well (CMake + Composer) Closes #11718 COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/11718 from protocolbuffers:gha-actions 5403307bc00616e94816aa460813939d8f37a1bd PiperOrigin-RevId: 506375417pull/11745/head
Mike Kruskal
2 years ago
committed by
Copybara-Service
parent
037f06d96d
commit
2b4e4260f0
14 changed files with 350 additions and 162 deletions
@ -0,0 +1,76 @@ |
||||
name: 'Docker Bazel Run' |
||||
description: 'Run a Bazel-based docker image for Protobuf CI testing' |
||||
inputs: |
||||
credentials: |
||||
required: true |
||||
description: "The GCP credentials to use for reading the docker image" |
||||
type: string |
||||
image: |
||||
required: false |
||||
default: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1 |
||||
description: "The docker image to use" |
||||
type: string |
||||
bazel-cache: |
||||
required: false |
||||
description: > |
||||
A unique path for the Bazel cache. This will trigger the generation |
||||
of a BAZEL_CACHE environment variable inside the container that provides |
||||
the appropriate flags for any bazel command. |
||||
type: string |
||||
bazel: |
||||
required: false |
||||
description: "The Bazel command to run" |
||||
type: string |
||||
bash: |
||||
required: false |
||||
description: "A bash command to run. $BAZEL_FLAGS will be available to use for bazel runs." |
||||
type: string |
||||
|
||||
runs: |
||||
using: 'composite' |
||||
steps: |
||||
- name: Authenticate |
||||
id: auth |
||||
uses: ./.github/actions/internal/docker-auth |
||||
with: |
||||
credentials: ${{ inputs.credentials }} |
||||
|
||||
- name: Validate inputs |
||||
if: ${{ inputs.bash && inputs.bazel}} |
||||
shell: bash |
||||
run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1 |
||||
|
||||
- name: Initialize Bazel flags |
||||
shell: bash |
||||
run: echo "BAZEL_FLAGS=--keep_going --test_output=errors --test_timeout=600" >> $GITHUB_ENV |
||||
|
||||
- name: Configure Bazel caching |
||||
shell: bash |
||||
# Skip bazel cache for local act runs due to issue with credential files |
||||
# and nested docker images |
||||
if: ${{ inputs.bazel-cache && !github.event.act_local_test }} |
||||
run: > |
||||
echo "BAZEL_FLAGS=$BAZEL_FLAGS |
||||
--google_credentials=/workspace/$(basename ${{ steps.auth.outputs.credentials-file }}) |
||||
--remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/protobuf/gha/${{ inputs.bazel-cache }}" >> $GITHUB_ENV |
||||
|
||||
- name: Configure Bazel cache writing |
||||
shell: bash |
||||
# External PRs should never write to our caches. |
||||
if: ${{ github.event_name != 'pull_request_target' && inputs.bazel-cache && !github.event.act_local_test }} |
||||
run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --remote_upload_local_results" >> $GITHUB_ENV |
||||
|
||||
- name: Run Bash Docker |
||||
uses: ./.github/actions/internal/docker-run |
||||
if: ${{ inputs.bash }} |
||||
with: |
||||
image: ${{ inputs.image }} |
||||
run-flags: --entrypoint "/bin/bash" |
||||
command: -l -c "${{ inputs.bash }}" |
||||
|
||||
- name: Run Bazel Docker |
||||
uses: ./.github/actions/internal/docker-run |
||||
if: ${{ !inputs.bash }} |
||||
with: |
||||
image: ${{ inputs.image }} |
||||
command: ${{ inputs.bazel }} $BAZEL_FLAGS |
||||
@ -0,0 +1,29 @@ |
||||
name: 'Authenticate Docker' |
||||
description: 'Authenticate a workflow for Protobuf CI testing' |
||||
inputs: |
||||
credentials: |
||||
required: true |
||||
description: "The GCP credentials to use for reading the docker image" |
||||
type: string |
||||
|
||||
outputs: |
||||
credentials-file: |
||||
description: "Credentials file generated for GCP" |
||||
value: ${{ steps.auth.outputs.credentials_file_path }} |
||||
|
||||
runs: |
||||
using: 'composite' |
||||
steps: |
||||
- name: Authenticate to Google Cloud |
||||
id: auth |
||||
uses: google-github-actions/auth@ef5d53e30bbcd8d0836f4288f5e50ff3e086997d # v1.0.0 |
||||
with: |
||||
credentials_json: ${{ inputs.credentials }} |
||||
- name: Set up Cloud SDK |
||||
uses: google-github-actions/setup-gcloud@d51b5346f85640ec2aa2fa057354d2b82c2fcbce # v1.0.1 |
||||
- name: Use gcloud CLI |
||||
shell: bash |
||||
run: gcloud info |
||||
- name: Authenticate for GAR use |
||||
shell: bash |
||||
run: gcloud auth configure-docker -q us-docker.pkg.dev |
||||
@ -0,0 +1,58 @@ |
||||
name: 'Run Docker' |
||||
description: 'Run a docker image for Protobuf CI testing' |
||||
inputs: |
||||
image: |
||||
required: true |
||||
description: "The docker image to use" |
||||
type: string |
||||
command: |
||||
required: true |
||||
description: "A raw docker command to run" |
||||
type: string |
||||
run-flags: |
||||
required: false |
||||
description: "Additional flags to pass to docker run" |
||||
type: string |
||||
|
||||
# WARNING: loading from cache appears to be slower than pull! |
||||
docker-cache: |
||||
required: false |
||||
description: "Enabled caching of pulled docker images." |
||||
|
||||
runs: |
||||
using: 'composite' |
||||
steps: |
||||
- name: Setup QEMU for possible emulation |
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2.1.0 |
||||
|
||||
- name: Check docker cache |
||||
if: ${{ inputs.docker-cache }} |
||||
id: check-docker-cache |
||||
uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 |
||||
with: |
||||
path: ci/docker/ |
||||
key: ${{ inputs.image }} |
||||
|
||||
- name: Pull and store if cache miss |
||||
shell: bash |
||||
if: ${{ inputs.docker-cache && steps.check-docker-cache.outputs.cache-hit != 'true' }} |
||||
run: > |
||||
time docker pull -q ${{ inputs.image }} && |
||||
mkdir -p ci/docker/$(dirname ${{ inputs.image }}) && |
||||
time docker image save ${{ inputs.image }} --output ./ci/docker/${{ inputs.image }}.tar |
||||
|
||||
- name: Use the cached image on cache hit |
||||
shell: bash |
||||
if: ${{ inputs.docker-cache && steps.check-docker-cache.outputs.cache-hit == 'true' }} |
||||
run: time docker image load --input ./ci/docker/${{ inputs.image }}.tar |
||||
|
||||
- name: Pull fresh docker image |
||||
shell: bash |
||||
if: ${{ !inputs.docker-cache }} |
||||
run: time docker pull -q ${{ inputs.image }} |
||||
|
||||
- name: Run docker |
||||
shell: bash |
||||
run: > |
||||
time docker run ${{ inputs.run-flags}} -v${{ github.workspace }}:/workspace |
||||
${{ inputs.image }} ${{ inputs.command }} |
||||
@ -0,0 +1,32 @@ |
||||
name: 'Docker Non-Bazel Run' |
||||
description: 'Run a docker image for Protobuf CI testing with a non-Bazel build system' |
||||
inputs: |
||||
credentials: |
||||
required: true |
||||
description: "The GCP credentials to use for reading the docker image" |
||||
type: string |
||||
command: |
||||
required: true |
||||
description: A command to run in the docker image |
||||
image: |
||||
required: false |
||||
default: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1 |
||||
description: "The docker image to use" |
||||
type: string |
||||
|
||||
runs: |
||||
using: 'composite' |
||||
steps: |
||||
- name: Update stale files using Bazel |
||||
uses: ./.github/actions/bazel-docker |
||||
with: |
||||
image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1 |
||||
credentials: ${{ inputs.credentials }} |
||||
bazel-cache: regenerate-stale-files |
||||
bash: ./regenerate_stale_files.sh $BAZEL_FLAGS |
||||
|
||||
- name: Run Docker |
||||
uses: ./.github/actions/internal/docker-run |
||||
with: |
||||
image: ${{ inputs.image }} |
||||
command: ${{ inputs.command }} |
||||
@ -0,0 +1,50 @@ |
||||
name: PHP Tests |
||||
|
||||
on: |
||||
workflow_call: |
||||
inputs: |
||||
safe-checkout: |
||||
required: true |
||||
description: "The SHA key for the commit we want to run over" |
||||
type: string |
||||
|
||||
jobs: |
||||
linux: |
||||
strategy: |
||||
fail-fast: false # Don't cancel all jobs if one fails. |
||||
matrix: |
||||
include: |
||||
- name: 7.3 Debug |
||||
version: 7.3.28-dbg |
||||
command: composer test \&\& composer test_c |
||||
- name: 7.4 Debug |
||||
version: 7.4.18-dbg |
||||
command: composer test \&\& composer test_c |
||||
- name: 8.0 Optimized |
||||
version: 8.0.5 |
||||
command: composer test \&\& composer test_c |
||||
- name: 8.0 Debug |
||||
version: 8.0.5-dbg |
||||
command: composer test \&\& composer test_c |
||||
- name: 8.0 Memory Leak |
||||
version: 8.0.5-dbg |
||||
# Run specialized memory leak & multirequest tests. |
||||
command: composer test_c \&\& tests/multirequest.sh \&\& tests/memory_leak_test.sh |
||||
- name: 8.0 Valgrind |
||||
version: 8.0.5-dbg |
||||
command: composer test_valgrind |
||||
|
||||
name: Linux PHP ${{ matrix.name}} |
||||
runs-on: ubuntu-latest |
||||
steps: |
||||
- name: Checkout pending changes |
||||
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 |
||||
with: |
||||
submodules: recursive |
||||
ref: ${{ inputs.safe-checkout }} |
||||
- name: Run tests |
||||
uses: ./.github/actions/non-bazel-docker |
||||
with: |
||||
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-6e95c0e221e4bd52e3b4dc1398c6336985196931 |
||||
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} |
||||
command: ${{ matrix.command }} |
||||
@ -1,127 +0,0 @@ |
||||
name: Run a Docker workflow |
||||
|
||||
on: |
||||
workflow_call: |
||||
inputs: |
||||
name: |
||||
required: True |
||||
description: "The name to display for the test" |
||||
type: string |
||||
image: |
||||
required: false |
||||
default: 'us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1' |
||||
description: "The docker image to use" |
||||
type: string |
||||
safe-checkout: |
||||
required: true |
||||
description: "The SHA key for the commit we want to run over" |
||||
type: string |
||||
run-flags: |
||||
required: false |
||||
description: "Additional flags to pass to docker run" |
||||
type: string |
||||
bazel-cache: |
||||
required: false |
||||
description: > |
||||
A unique path for the Bazel cache. This will trigger the generation |
||||
of a BAZEL_CACHE environment variant that provides the appropriate |
||||
flags for any bazel command. |
||||
type: string |
||||
|
||||
# WARNING: loading from cache appears to be slower than pull! |
||||
docker-cache: |
||||
required: false |
||||
description: "Enabled caching of pulled docker images." |
||||
default: false |
||||
type: boolean |
||||
|
||||
# Non-Bazel options |
||||
command: |
||||
required: false |
||||
description: "A raw docker command to run" |
||||
type: string |
||||
|
||||
# Bazel options |
||||
bazel: |
||||
required: false |
||||
description: "The Bazel command to run" |
||||
type: string |
||||
|
||||
jobs: |
||||
run: |
||||
name: ${{ inputs.name }} |
||||
timeout-minutes: 120 |
||||
runs-on: ubuntu-latest |
||||
steps: |
||||
- name: Checkout pending changes |
||||
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 |
||||
with: |
||||
ref: ${{ inputs.safe-checkout }} |
||||
|
||||
# Authentication |
||||
- name: Setup QEMU for possible emulation |
||||
id: qemu-arm64 |
||||
uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1.2.0 |
||||
|
||||
- name: Authenticate to Google Cloud |
||||
uses: google-github-actions/auth@ef5d53e30bbcd8d0836f4288f5e50ff3e086997d # v1.0.0 |
||||
with: |
||||
credentials_json: ${{ secrets.GAR_SERVICE_ACCOUNT }} |
||||
export_environment_variables: true |
||||
- name: Set up Cloud SDK |
||||
uses: google-github-actions/setup-gcloud@d51b5346f85640ec2aa2fa057354d2b82c2fcbce # v1.0.1 |
||||
- name: Use gcloud CLI |
||||
run: gcloud info |
||||
- name: Authenticate for GAR use |
||||
run: gcloud auth configure-docker -q us-docker.pkg.dev |
||||
|
||||
# Create the docker command |
||||
- name: Validate Docker command |
||||
if: ${{ inputs.command && inputs.bazel}} |
||||
run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1 |
||||
- name: Configure Bazel caching |
||||
# Skip bazel cache for local act runs due to issue with credential files |
||||
# and nested docker images |
||||
if: ${{ inputs.bazel-cache && !github.event.act_local_test }} |
||||
run: > |
||||
echo "BAZEL_CACHE= |
||||
--google_credentials=/workspace/$(basename $GOOGLE_APPLICATION_CREDENTIALS) |
||||
--remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/protobuf/gha/${{ inputs.bazel-cache }}" >> $GITHUB_ENV |
||||
- name: Configure Bazel cache updating |
||||
# External runs should never write to our caches. |
||||
if: ${{ inputs.bazel-cache && !inputs.safe-checkout && !github.event.act_local_test }} |
||||
run: echo "BAZEL_CACHE=$BAZEL_CACHE --remote_upload_local_results" >> $GITHUB_ENV |
||||
- name: Configure Bazel command |
||||
if: ${{ inputs.bazel }} |
||||
run: > |
||||
echo "DOCKER_COMMAND=${{ inputs.bazel }} |
||||
--keep_going --test_output=errors --test_timeout=600 |
||||
$BAZEL_CACHE" >> $GITHUB_ENV |
||||
|
||||
# Grab Docker image |
||||
- name: Check docker cache |
||||
if: ${{ inputs.docker-cache }} |
||||
id: check-docker-cache |
||||
uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 |
||||
with: |
||||
path: ci/docker/ |
||||
key: ${{ inputs.image }} |
||||
- name: Pull and store if cache miss |
||||
if: ${{ inputs.docker-cache && steps.check-docker-cache.outputs.cache-hit != 'true' }} |
||||
run: > |
||||
docker pull ${{ inputs.image }} && |
||||
mkdir -p ci/docker/$(dirname ${{ inputs.image }}) && |
||||
docker image save ${{ inputs.image }} --output ./ci/docker/${{ inputs.image }}.tar |
||||
- name: Use the cached image on cache hit |
||||
if: ${{ inputs.docker-cache && steps.check-docker-cache.outputs.cache-hit == 'true' }} |
||||
run: docker image load --input ./ci/docker/${{ inputs.image }}.tar |
||||
- name: Pull fresh docker image |
||||
if: ${{ !inputs.docker-cache }} |
||||
run: docker pull ${{ inputs.image }} |
||||
|
||||
- name: Run docker |
||||
run: > |
||||
docker run ${{ inputs.run-flags}} |
||||
-v${{ github.workspace }}:/workspace |
||||
${{ inputs.image }} |
||||
${{ inputs.command || '$DOCKER_COMMAND' }} |
||||
Loading…
Reference in new issue