This will make PRs from forked repositories significantly less painful, since they'll agree on which version of each action to use. OTOH, we'll have a separate repo that needs to be maintained, and changes to it will need to be coordinated and versioned carefully. This will likely need to be done less often though now that our infrastructure is stable. PiperOrigin-RevId: 512117705pull/11894/head
Mike Kruskal
2 years ago
committed by
Copybara-Service
parent
9ef627938a
commit
ef8ebe6c19
21 changed files with 35 additions and 746 deletions
@ -1,27 +0,0 @@ |
||||
name: 'Non-Bazel Bash Run' |
||||
description: 'Run a bash script for Protobuf CI testing with a non-Bazel build system' |
||||
inputs: |
||||
credentials: |
||||
required: true |
||||
description: "The GCP credentials to use for reading the docker image" |
||||
type: string |
||||
command: |
||||
required: true |
||||
description: A command to run in the docker image |
||||
|
||||
runs: |
||||
using: 'composite' |
||||
steps: |
||||
- name: Setup Runner |
||||
uses: ./.github/actions/internal/setup-runner |
||||
|
||||
- name: Update stale files using Bazel |
||||
uses: ./.github/actions/bazel |
||||
with: |
||||
credentials: ${{ inputs.credentials }} |
||||
bazel-cache: regenerate-stale-files |
||||
bash: ./regenerate_stale_files.sh $BAZEL_FLAGS |
||||
|
||||
- name: Run |
||||
shell: bash |
||||
run: ${{ inputs.command }} |
||||
@ -1,75 +0,0 @@ |
||||
name: 'Docker Bazel Run' |
||||
description: 'Run a Bazel-based docker image for Protobuf CI testing' |
||||
inputs: |
||||
credentials: |
||||
required: true |
||||
description: "The GCP credentials to use for reading the docker image" |
||||
type: string |
||||
image: |
||||
required: false |
||||
default: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1 |
||||
description: "The docker image to use" |
||||
type: string |
||||
bazel-cache: |
||||
required: true |
||||
description: > |
||||
A unique path for the Bazel cache. This will trigger the generation |
||||
of a BAZEL_CACHE environment variable inside the container that provides |
||||
the appropriate flags for any bazel command. |
||||
type: string |
||||
bazel: |
||||
required: false |
||||
description: "The Bazel command to run" |
||||
type: string |
||||
bash: |
||||
required: false |
||||
description: "A bash command to run. $BAZEL_FLAGS will be available to use for bazel runs." |
||||
type: string |
||||
|
||||
runs: |
||||
using: 'composite' |
||||
steps: |
||||
- name: Authenticate |
||||
id: auth |
||||
uses: ./.github/actions/internal/gcloud-auth |
||||
with: |
||||
credentials: ${{ inputs.credentials }} |
||||
|
||||
- name: Setup Runner |
||||
uses: ./.github/actions/internal/setup-runner |
||||
|
||||
- name: Setup Bazel |
||||
id: bazel |
||||
uses: ./.github/actions/internal/bazel-setup |
||||
with: |
||||
credentials-file: /workspace/$(basename ${{ steps.auth.outputs.credentials-file }}) |
||||
bazel-cache: ${{ inputs.bazel-cache }} |
||||
|
||||
- name: Hook up repository Cache |
||||
shell: bash |
||||
run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --repository_cache='/workspace/${{ env.REPOSITORY_CACHE_PATH }}'" >> $GITHUB_ENV |
||||
|
||||
- name: Validate inputs |
||||
if: ${{ (inputs.bash && inputs.bazel) || (!inputs.bash && !inputs.bazel) }} |
||||
shell: bash |
||||
run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1 |
||||
|
||||
- name: Run Bash Docker |
||||
uses: ./.github/actions/internal/docker-run |
||||
if: ${{ inputs.bash }} |
||||
with: |
||||
image: ${{ inputs.image }} |
||||
run-flags: --entrypoint "/bin/bash" |
||||
command: -l -c "${{ inputs.bash }}" |
||||
|
||||
- name: Run Bazel Docker |
||||
uses: ./.github/actions/internal/docker-run |
||||
if: ${{ !inputs.bash }} |
||||
with: |
||||
image: ${{ inputs.image }} |
||||
command: ${{ inputs.bazel }} ${{ env.BAZEL_FLAGS }} |
||||
|
||||
- name: Save Bazel repository cache |
||||
# Only allow repository cache updates during post-submits. |
||||
if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' }} |
||||
uses: ./.github/actions/internal/repository-cache-save |
||||
@ -1,134 +0,0 @@ |
||||
name: 'Docker Bazel Run' |
||||
description: 'Run a Bazel-based docker image for Protobuf CI testing' |
||||
inputs: |
||||
credentials: |
||||
required: true |
||||
description: The GCP credentials to use for reading the docker image |
||||
type: string |
||||
bazel-cache: |
||||
required: true |
||||
description: > |
||||
A unique path for the Bazel cache. This will trigger the generation |
||||
of a BAZEL_CACHE environment variable inside the container that provides |
||||
the appropriate flags for any bazel command. |
||||
type: string |
||||
version: |
||||
required: false |
||||
description: A pinned Bazel version to use |
||||
default: '5.1.1' |
||||
type: string |
||||
bazel: |
||||
required: false |
||||
description: The Bazel command to run |
||||
type: string |
||||
bash: |
||||
required: false |
||||
description: > |
||||
A bash command to run. $BAZEL_FLAGS and $BAZEL_STARTUP_FLAGS will be |
||||
available to use for bazel runs. |
||||
type: string |
||||
|
||||
runs: |
||||
using: 'composite' |
||||
steps: |
||||
- name: Authenticate |
||||
id: auth |
||||
uses: ./.github/actions/internal/gcloud-auth |
||||
with: |
||||
credentials: ${{ inputs.credentials }} |
||||
|
||||
- name: Setup Runner |
||||
uses: ./.github/actions/internal/setup-runner |
||||
|
||||
- name: Setup Bazel |
||||
id: bazel |
||||
uses: ./.github/actions/internal/bazel-setup |
||||
with: |
||||
credentials-file: ${{ steps.auth.outputs.credentials-file }} |
||||
bazel-cache: ${{ inputs.bazel-cache }} |
||||
|
||||
- name: Get Linux bazelisk path |
||||
if: runner.os == 'Linux' |
||||
shell: bash |
||||
run: echo "BAZELISK_PATH=~/.cache/bazelisk" >> $GITHUB_ENV |
||||
|
||||
- name: Get MacOS bazelisk path |
||||
if: runner.os == 'macOS' |
||||
shell: bash |
||||
run: echo "BAZELISK_PATH=~/Library/Caches/bazelisk" >> $GITHUB_ENV |
||||
|
||||
- name: Get Windows bazelisk path |
||||
if: runner.os == 'Windows' |
||||
shell: bash |
||||
run: echo "BAZELISK_PATH=$LOCALAPPDATA\bazelisk" >> $GITHUB_ENV |
||||
|
||||
- name: Cache Bazelisk |
||||
if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' }} |
||||
uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 |
||||
with: |
||||
path: ${{ env.BAZELISK_PATH }} |
||||
key: bazel-${{ runner.os }}-${{ inputs.version }} |
||||
|
||||
- name: Restore Bazelisk |
||||
if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} |
||||
uses: actions/cache/restore@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 |
||||
with: |
||||
path: ${{ env.BAZELISK_PATH }} |
||||
key: bazel-${{ runner.os }}-${{ inputs.version }} |
||||
|
||||
- name: Hook up repository Cache |
||||
shell: bash |
||||
run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --repository_cache=$(pwd)/${{ env.REPOSITORY_CACHE_PATH }}" >> $GITHUB_ENV |
||||
|
||||
- name: Validate inputs |
||||
if: ${{ (inputs.bash && inputs.bazel) || (!inputs.bash && !inputs.bazel) }} |
||||
shell: bash |
||||
run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1 |
||||
|
||||
- name: Pin Bazel version |
||||
shell: bash |
||||
run: echo "USE_BAZEL_VERSION=${{ inputs.version }}" >> $GITHUB_ENV |
||||
|
||||
- name: Output Bazel version |
||||
shell: bash |
||||
run: bazelisk version |
||||
|
||||
# Bazel has multiple Xcode calls with hardcoded timeouts. Many of these |
||||
# end up timing out on our github runners, causing flakes on every mac |
||||
# build that invoked Bazel. To work around this, we manually inoke these |
||||
# calls before running Bazel to make sure they end up in Xcode's cache for |
||||
# quicker runs later. All of these calls are obtained from xcrun calls in |
||||
# https://github.com/bazelbuild/bazel/blob/e8a69f5d5acaeb6af760631490ecbf73e8a04eeb/tools/cpp/osx_cc_configure.bzl. |
||||
# See https://github.com/bazelbuild/bazel/issues/17437 for more details. |
||||
# TODO(b/269503614) Remove this once Bazel provides an official solution. |
||||
- name: Warm up Xcode |
||||
if: ${{ runner.os == 'macOS' }} |
||||
shell: bash |
||||
run: | |
||||
mkdir -p mac_bazel_workaround |
||||
bazelisk ${{ steps.bazel.outputs.bazel-startup-flags }} build @bazel_tools//tools/osx:xcode_locator.m $BAZEL_FLAGS |
||||
XCODE_LOCATOR_FLAGS="--sdk macosx clang -mmacosx-version-min=10.9 -fobjc-arc -framework CoreServices -framework Foundation" |
||||
SINGLE_ARCH_COMPILE_FLAGS="--sdk macosx clang -mmacosx-version-min=10.9 -std=c++11 -lc++ -O3" |
||||
COMPILE_FLAGS="$SINGLE_ARCH_COMPILE_FLAGS -arch arm64 -arch x86_64 -Wl,-no_adhoc_codesign -Wl,-no_uuid -O3" |
||||
time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $XCODE_LOCATOR_FLAGS -o mac_bazel_workaround/xcode-locator-bin $(bazel info output_base)/external/bazel_tools/tools/osx/xcode_locator.m |
||||
time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $SINGLE_ARCH_COMPILE_FLAGS -o mac_bazel_workaround/libtool_check_unique $(bazel info output_base)/external/bazel_tools/tools/objc/libtool_check_unique.cc |
||||
time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $COMPILE_FLAGS -o mac_bazel_workaround/libtool_check_unique $(bazel info output_base)/external/bazel_tools/tools/objc/libtool_check_unique.cc |
||||
time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $SINGLE_ARCH_COMPILE_FLAGS -o mac_bazel_workaround/wrapped_clang $(bazel info output_base)/external/bazel_tools/tools/osx/crosstool/wrapped_clang.cc |
||||
time env -i DEVELOPER_DIR=$DEVELOPER_DIR xcrun $COMPILE_FLAGS -o mac_bazel_workaround/wrapped_clang $(bazel info output_base)/external/bazel_tools/tools/osx/crosstool/wrapped_clang.cc |
||||
|
||||
- name: Run Bash |
||||
if: ${{ inputs.bash }} |
||||
run: ${{ inputs.bash }} |
||||
shell: bash |
||||
|
||||
- name: Run Bazel |
||||
if: ${{ !inputs.bash }} |
||||
run: >- |
||||
bazelisk ${{ steps.bazel.outputs.bazel-startup-flags }} |
||||
${{ inputs.bazel }} $BAZEL_FLAGS |
||||
shell: bash |
||||
|
||||
- name: Save Bazel repository cache |
||||
# Only allow repository cache updates during post-submits. |
||||
if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target'}} |
||||
uses: ./.github/actions/internal/repository-cache-save |
||||
@ -1,63 +0,0 @@ |
||||
name: 'CCache Setup' |
||||
description: 'Run a Bazel-based docker image for Protobuf CI testing' |
||||
inputs: |
||||
cache-prefix: |
||||
required: true |
||||
description: A unique prefix to prevent cache pollution |
||||
type: string |
||||
support-modules: |
||||
required: false |
||||
description: Whether or not we need to support modules. This can result in extra cache misses. |
||||
|
||||
runs: |
||||
using: 'composite' |
||||
steps: |
||||
- name: Configure ccache environment variables |
||||
shell: bash |
||||
run: | |
||||
echo "CCACHE_BASEDIR=${{ github.workspace }}" >> $GITHUB_ENV |
||||
echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV |
||||
echo "CCACHE_COMPRESS=true" >> $GITHUB_ENV |
||||
echo "CCACHE_COMPRESSLEVEL=5" >> $GITHUB_ENV |
||||
echo "CCACHE_MAXSIZE=100M" >> $GITHUB_ENV |
||||
echo "CCACHE_SLOPPINESS=clang_index_store,include_file_ctime,include_file_mtime,file_macro,time_macros" >> $GITHUB_ENV |
||||
echo "CCACHE_DIRECT=true" >> $GITHUB_ENV |
||||
echo "CCACHE_CMAKE_FLAGS=-Dprotobuf_ALLOW_CCACHE=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" >> $GITHUB_ENV |
||||
|
||||
- name: Setup ccache on Windows |
||||
if: ${{ runner.os == 'Windows' }} |
||||
uses: ./.github/actions/internal/ccache-setup-windows |
||||
- name: Setup ccache on Mac |
||||
if: ${{ runner.os == 'macOS' }} |
||||
shell: bash |
||||
run: brew install ccache |
||||
|
||||
- name: Setup fixed path ccache caching |
||||
uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 |
||||
with: |
||||
path: | |
||||
.ccache/** |
||||
!.ccache/lock |
||||
!.ccache/tmp |
||||
# Always push to a cache key unique to this commit. |
||||
key: ${{ format('ccache-{0}-{1}-{2}', inputs.cache-prefix, github.ref_name, github.sha) }} |
||||
# Select a cache to restore from with the follow order of preference: |
||||
# 1) The exact same commit we're running over |
||||
# 2) The latest cache from the current ref branch |
||||
# 3) The latest push to the base ref of a pull request |
||||
restore-keys: | |
||||
${{ format('ccache-{0}-{1}-{2}', inputs.cache-prefix, github.ref_name, github.sha) }} |
||||
${{ format('ccache-{0}-{1}', inputs.cache-prefix, github.ref_name) }} |
||||
${{ format('ccache-{0}-{1}', inputs.cache-prefix, github.base_ref) }} |
||||
|
||||
- name: Enable module support |
||||
if: ${{ inputs.support-modules }} |
||||
shell: bash |
||||
run: | |
||||
echo "CCACHE_SLOPPINESS=$CCACHE_SLOPPINESS,modules" >> $GITHUB_ENV |
||||
echo "CCACHE_DEPEND=true" >> $GITHUB_ENV |
||||
|
||||
- name: Zero out ccache |
||||
if: ${{ runner.os != 'Linux' }} |
||||
shell: bash |
||||
run: ccache -z |
||||
@ -1,38 +0,0 @@ |
||||
name: 'Cross-compile protoc' |
||||
description: 'Produces a cross-compiled protoc binary for a target architecture' |
||||
inputs: |
||||
credentials: |
||||
required: true |
||||
description: The GCP credentials to use for reading the docker image |
||||
type: string |
||||
architecture: |
||||
required: true |
||||
description: The target architecture to build for |
||||
type: string |
||||
outputs: |
||||
protoc: |
||||
description: "Cross-compiled protoc location. Also output to $PROTOC" |
||||
value: ${{ steps.output.outputs.protoc }} |
||||
|
||||
runs: |
||||
using: 'composite' |
||||
steps: |
||||
- name: Cross compile protoc for ${{ inputs.architecture }} |
||||
uses: ./.github/actions/bazel-docker |
||||
with: |
||||
credentials: ${{ inputs.credentials }} |
||||
bazel-cache: xcompile-protoc/${{ inputs.architecture }} |
||||
bash: | |
||||
bazel build //:protoc --config=${{ inputs.architecture }} $BAZEL_FLAGS |
||||
cp bazel-bin/protoc . |
||||
|
||||
- name: Set protoc environment variable |
||||
shell: bash |
||||
run: echo "PROTOC=protoc-${{ inputs.architecture }}" >> $GITHUB_ENV |
||||
|
||||
- name: Extract binary |
||||
id: output |
||||
shell: bash |
||||
run: | |
||||
mv protoc $PROTOC |
||||
echo "protoc=$PROTOC" >> $GITHUB_OUTPUT |
||||
@ -1,50 +0,0 @@ |
||||
name: 'Docker Non-Bazel Run' |
||||
description: 'Run a docker image for Protobuf CI testing with a non-Bazel build system' |
||||
inputs: |
||||
credentials: |
||||
required: true |
||||
description: "The GCP credentials to use for reading the docker image" |
||||
type: string |
||||
command: |
||||
required: true |
||||
description: A command to run in the docker image |
||||
image: |
||||
required: false |
||||
default: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1 |
||||
description: "The docker image to use" |
||||
type: string |
||||
platform: |
||||
required: false |
||||
description: "The platform to use for the image" |
||||
type: string |
||||
skip-staleness-check: |
||||
required: false |
||||
description: "Skip staleness checks" |
||||
type: boolean |
||||
|
||||
runs: |
||||
using: 'composite' |
||||
steps: |
||||
- name: Setup Runner |
||||
uses: ./.github/actions/internal/setup-runner |
||||
|
||||
- name: Update stale files using Bazel |
||||
if: ${{ !inputs.skip-staleness-check }} |
||||
uses: ./.github/actions/bazel-docker |
||||
with: |
||||
image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1 |
||||
credentials: ${{ inputs.credentials }} |
||||
bazel-cache: regenerate-stale-files |
||||
bash: ./regenerate_stale_files.sh $BAZEL_FLAGS |
||||
|
||||
- name: Generate docker flags |
||||
if: inputs.platform |
||||
shell: bash |
||||
run: echo "DOCKER_RUN_FLAGS=--platform ${{inputs.platform}}" >> $GITHUB_ENV |
||||
|
||||
- name: Run Docker |
||||
uses: ./.github/actions/internal/docker-run |
||||
with: |
||||
image: ${{ inputs.image }} |
||||
command: ${{ inputs.command }} |
||||
run-flags: ${{ env.DOCKER_RUN_FLAGS }} |
||||
@ -1,75 +0,0 @@ |
||||
name: Setup Bazel |
||||
description: Setup a Bazel environment for Protobuf CI testing |
||||
inputs: |
||||
credentials-file: |
||||
required: true |
||||
description: The GCP credentials file to use for caching |
||||
type: string |
||||
bazel-cache: |
||||
required: true |
||||
description: A unique path for the Bazel cache. |
||||
type: string |
||||
|
||||
outputs: |
||||
bazel-flags: |
||||
description: Bazel flags that should be sent to all Bazel invocations |
||||
value: ${{ steps.output.outputs.bazel-flags }} |
||||
bazel-startup-flags: |
||||
description: Bazel startup flags that should be sent to all Bazel invocations |
||||
value: ${{ steps.output.outputs.bazel-startup-flags }} |
||||
|
||||
runs: |
||||
using: 'composite' |
||||
steps: |
||||
- name: Initialize BAZEL environment variable |
||||
shell: bash |
||||
run: echo "BAZEL=bazelisk" >> $GITHUB_ENV |
||||
|
||||
- name: Initialize Windows startup flags |
||||
if: runner.os == 'Windows' |
||||
shell: bash |
||||
run: echo "BAZEL_STARTUP_FLAGS=--output_user_root=C:/tmp --windows_enable_symlinks" >> $GITHUB_ENV |
||||
|
||||
- name: Initialize Bazel flags |
||||
shell: bash |
||||
run: echo "BAZEL_FLAGS=--keep_going --test_output=errors --test_timeout=600" >> $GITHUB_ENV |
||||
|
||||
- name: Initialize Windows-specific Bazel flags |
||||
if: runner.os == 'Windows' |
||||
shell: bash |
||||
run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --enable_runfiles" >> $GITHUB_ENV |
||||
|
||||
- name: Initialize MacOS-specific Bazel flags |
||||
if: runner.os == 'macOS' |
||||
shell: bash |
||||
run: | |
||||
echo "BAZEL_FLAGS=$BAZEL_FLAGS --xcode_version_config=//.github:host_xcodes" >> $GITHUB_ENV |
||||
echo "DEVELOPER_DIR=${{ env.DEVELOPER_DIR || '/Applications/Xcode_14.1.app/Contents/Developer' }}" >> $GITHUB_ENV |
||||
|
||||
- name: Configure Bazel caching |
||||
# Skip bazel cache for local act runs due to issue with credential files |
||||
# and nested docker images |
||||
if: ${{ inputs.bazel-cache && !github.event.act_local_test }} |
||||
shell: bash |
||||
run: >- |
||||
echo "BAZEL_FLAGS=$BAZEL_FLAGS |
||||
--google_credentials=${{ inputs.credentials-file }} |
||||
--remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/protobuf/gha/${{ inputs.bazel-cache }}" >> $GITHUB_ENV |
||||
|
||||
- name: Configure Bazel cache writing |
||||
# External runs should never write to our caches. |
||||
if: ${{ github.event_name != 'pull_request_target' && inputs.bazel-cache && !github.event.act_local_test }} |
||||
shell: bash |
||||
run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --remote_upload_local_results" >> $GITHUB_ENV |
||||
|
||||
- name: Output Bazel flags |
||||
id: output |
||||
shell: bash |
||||
run: | |
||||
echo "bazel-flags=$BAZEL_FLAGS" >> $GITHUB_OUTPUT |
||||
echo "bazel-startup-flags=$BAZEL_STARTUP_FLAGS" >> $GITHUB_OUTPUT |
||||
|
||||
- name: Restore Bazel repository cache |
||||
uses: ./.github/actions/internal/repository-cache-restore |
||||
with: |
||||
bazel-cache: ${{ inputs.bazel-cache }} |
||||
@ -1,68 +0,0 @@ |
||||
name: 'CCache Setup' |
||||
description: 'Setup ccache for us in Windows CI' |
||||
inputs: |
||||
ccache-version: |
||||
required: false |
||||
default: '4.7.4' |
||||
description: A pinned version of ccache |
||||
type: string |
||||
|
||||
runs: |
||||
using: 'composite' |
||||
steps: |
||||
- name: Setup MSVC |
||||
uses: ilammy/msvc-dev-cmd@cec98b9d092141f74527d0afa6feb2af698cfe89 # v1.12.1 |
||||
with: |
||||
arch: x64 |
||||
vsversion: '2019' |
||||
|
||||
- name: Setup ccache path |
||||
shell: bash |
||||
run: | |
||||
echo "CCACHE_EXE_PATH=$LOCALAPPDATA\ccache-${{ inputs.ccache-version }}-windows-x86_64" >> $GITHUB_ENV |
||||
echo "$LOCALAPPDATA\ccache-${{ inputs.ccache-version }}-windows-x86_64" >> $GITHUB_PATH |
||||
|
||||
- name: Add ccache to Powershell path |
||||
shell: pwsh |
||||
run: echo "%LOCALAPPDATA%\ccache-${{ inputs.ccache-version }}-windows-x86_64" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append |
||||
|
||||
- name: Setup caching of ccache download |
||||
if: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' }} |
||||
id: ccache-cache |
||||
uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 |
||||
with: |
||||
path: ${{ env.CCACHE_EXE_PATH }} |
||||
key: ccache-exe-${{ inputs.ccache-version }} |
||||
|
||||
- name: Restore ccache download |
||||
if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} |
||||
id: ccache-restore |
||||
uses: actions/cache/restore@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 |
||||
with: |
||||
path: ${{ env.CCACHE_EXE_PATH }} |
||||
key: ccache-exe-${{ inputs.ccache-version }} |
||||
|
||||
- name: Download ccache |
||||
shell: bash |
||||
if: ${{ steps.ccache-cache.outputs.cache-hit != 'true' && steps.ccache-restore.outputs.cache-hit != 'true'}} |
||||
run: | |
||||
cd $LOCALAPPDATA |
||||
curl -kLSs "https://github.com/ccache/ccache/releases/download/v${{ inputs.ccache-version }}/ccache-${{ inputs.ccache-version }}-windows-x86_64.zip" -o ccache.zip |
||||
unzip ccache.zip |
||||
rm ccache.zip |
||||
ccache --version |
||||
|
||||
- name: Configure ccache environment variables |
||||
shell: pwsh |
||||
run: | |
||||
Write-Host $Env:GITHUB_REF |
||||
$cllocation = (Get-Command cl.exe).Path |
||||
echo "CCACHE_COMPILER=$cllocation" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append |
||||
echo "CCACHE_COMPILERTYPE=msvc" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append |
||||
|
||||
- name: Configure Windows-specific ccache environment variables |
||||
shell: bash |
||||
# Windows caches are about 2x larger than other platforms. |
||||
run: | |
||||
echo "CCACHE_COMPRESSLEVEL=10" >> $GITHUB_ENV |
||||
echo "CCACHE_MAXSIZE=200M" >> $GITHUB_ENV |
||||
@ -1,62 +0,0 @@ |
||||
name: 'Run Docker' |
||||
description: 'Run a docker image for Protobuf CI testing' |
||||
inputs: |
||||
image: |
||||
required: true |
||||
description: "The docker image to use" |
||||
type: string |
||||
command: |
||||
required: true |
||||
description: "A raw docker command to run" |
||||
type: string |
||||
run-flags: |
||||
required: false |
||||
description: "Additional flags to pass to docker run" |
||||
type: string |
||||
|
||||
# WARNING: loading from cache appears to be slower than pull! |
||||
docker-cache: |
||||
required: false |
||||
description: "Enabled caching of pulled docker images." |
||||
|
||||
runs: |
||||
using: 'composite' |
||||
steps: |
||||
- name: Authenticate for GAR use |
||||
shell: bash |
||||
run: gcloud auth configure-docker -q us-docker.pkg.dev |
||||
|
||||
- name: Setup QEMU for possible emulation |
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2.1.0 |
||||
|
||||
- name: Check docker cache |
||||
if: ${{ inputs.docker-cache }} |
||||
id: check-docker-cache |
||||
uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 |
||||
with: |
||||
path: ci/docker/ |
||||
key: ${{ inputs.image }} |
||||
|
||||
- name: Pull and store if cache miss |
||||
shell: bash |
||||
if: ${{ inputs.docker-cache && steps.check-docker-cache.outputs.cache-hit != 'true' }} |
||||
run: > |
||||
time docker pull -q ${{ inputs.image }} && |
||||
mkdir -p ci/docker/$(dirname ${{ inputs.image }}) && |
||||
time docker image save ${{ inputs.image }} --output ./ci/docker/${{ inputs.image }}.tar |
||||
|
||||
- name: Use the cached image on cache hit |
||||
shell: bash |
||||
if: ${{ inputs.docker-cache && steps.check-docker-cache.outputs.cache-hit == 'true' }} |
||||
run: time docker image load --input ./ci/docker/${{ inputs.image }}.tar |
||||
|
||||
- name: Pull fresh docker image |
||||
shell: bash |
||||
if: ${{ !inputs.docker-cache }} |
||||
run: time docker pull -q ${{ inputs.image }} |
||||
|
||||
- name: Run docker |
||||
shell: bash |
||||
run: > |
||||
time docker run ${{ inputs.run-flags}} -v${{ github.workspace }}:/workspace |
||||
${{ inputs.image }} ${{ inputs.command }} |
||||
@ -1,42 +0,0 @@ |
||||
name: 'Authenticate for GCP' |
||||
description: 'Authenticate a workflow for Protobuf CI testing' |
||||
inputs: |
||||
credentials: |
||||
required: true |
||||
description: "The GCP credentials to use for GCP" |
||||
type: string |
||||
|
||||
outputs: |
||||
credentials-file: |
||||
description: "Credentials file generated for GCP" |
||||
value: ${{ steps.output.outputs.credentials-file }} |
||||
|
||||
runs: |
||||
using: 'composite' |
||||
steps: |
||||
- name: Authenticate to Google Cloud |
||||
id: auth |
||||
uses: google-github-actions/auth@ef5d53e30bbcd8d0836f4288f5e50ff3e086997d # v1.0.0 |
||||
with: |
||||
credentials_json: ${{ inputs.credentials }} |
||||
- name: Set up Cloud SDK |
||||
uses: google-github-actions/setup-gcloud@d51b5346f85640ec2aa2fa057354d2b82c2fcbce # v1.0.1 |
||||
- name: Use gcloud CLI |
||||
shell: bash |
||||
run: gcloud info |
||||
|
||||
- name: Store credentials path |
||||
shell: bash |
||||
run: echo "CREDENTIALS_FILE=${{ steps.auth.outputs.credentials_file_path }}" >> $GITHUB_ENV |
||||
|
||||
- name: Fix credentials path (Windows) |
||||
if: ${{ runner.os == 'Windows' }} |
||||
# Bash commands in windows don't like the backslash in the file path. |
||||
# Assume we're running in the root directory and grab the base name. |
||||
shell: bash |
||||
run: echo "CREDENTIALS_FILE="$(basename ${CREDENTIALS_FILE//\\//}) >> $GITHUB_ENV |
||||
|
||||
- name: Output credentials file |
||||
id: output |
||||
shell: bash |
||||
run: echo "credentials-file=${{ env.CREDENTIALS_FILE }}" >> $GITHUB_OUTPUT |
||||
@ -1,41 +0,0 @@ |
||||
name: Restore Repository Cache |
||||
description: Restore the Bazel repository cache from our github action cache |
||||
inputs: |
||||
bazel-cache: |
||||
required: true |
||||
description: A unique path for the Bazel cache. |
||||
type: string |
||||
|
||||
# By design, these actions will restore the latest cache for this branch/os, |
||||
# and only save a new version if something has changed. Initially this will |
||||
# cause a lot of churn, since each test has a slightly different set of |
||||
# repositories to download. Over time though, since we don't upload no-op |
||||
# changes, this should converge to a stable set of 3 caches per branch. Every |
||||
# run will update the current cache with a new test's repositories, until there |
||||
# are no unique ones left. |
||||
# |
||||
# This saves asymptotic space, since each one of these can get up to ~500 MB |
||||
# and Github prunes the cache after 10 GB. |
||||
runs: |
||||
using: 'composite' |
||||
steps: |
||||
- name: Setup Bazel repository cache variables |
||||
shell: bash |
||||
run: | |
||||
REPOSITORY_CACHE_BASE=repository-cache-${{ github.base_ref || github.ref_name }}-${{ runner.os }} |
||||
echo "REPOSITORY_CACHE_BASE=$REPOSITORY_CACHE_BASE" >> $GITHUB_ENV |
||||
echo "REPOSITORY_CACHE_NAME=$REPOSITORY_CACHE_BASE-${{ inputs.bazel-cache}}-${{ github.sha }}" >> $GITHUB_ENV |
||||
echo "REPOSITORY_CACHE_PATH=.repository-cache" >> $GITHUB_ENV |
||||
|
||||
- name: Restore Bazel repository cache |
||||
id: restore-cache |
||||
uses: actions/cache/restore@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 |
||||
with: |
||||
path: ${{ github.workspace }}/${{ env.REPOSITORY_CACHE_PATH }} |
||||
key: ${{ env.REPOSITORY_CACHE_NAME }} |
||||
restore-keys: ${{ env.REPOSITORY_CACHE_BASE }} |
||||
|
||||
- name: Initialize BAZEL environment variable |
||||
if: ${{ steps.restore-cache.cache-hit }} |
||||
shell: bash |
||||
run: echo "REPOSITORY_CACHE_HASH=${{ hashFiles(format('{0}/**', env.REPOSITORY_CACHE_PATH)) }}" >> $GITHUB_ENV |
||||
@ -1,19 +0,0 @@ |
||||
name: Restore Repository Cache |
||||
description: Restore the Bazel repository cache from our github action cache |
||||
|
||||
# Note: this action will only work if repository-cache-restore has already |
||||
# been called. All bazel actions should specify the repository_cache parameter |
||||
# using REPOSITORY_CACHE_PATH. |
||||
# |
||||
# We intentionally upload to REPOSITORY_CACHE_BASE to prevent a flood of new |
||||
# caches on any change. Only 1 job per os in each test run will be allowed to |
||||
# update the cache because they're all trying to write to the same location. |
||||
runs: |
||||
using: 'composite' |
||||
steps: |
||||
- name: Save modified Bazel repository cache |
||||
if: ${{ env.REPOSITORY_CACHE_HASH != hashFiles(format('{0}/**', env.REPOSITORY_CACHE_PATH)) }} |
||||
uses: actions/cache/save@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 |
||||
with: |
||||
path: ${{ github.workspace }}/${{ env.REPOSITORY_CACHE_PATH }} |
||||
key: ${{ env.REPOSITORY_CACHE_BASE }}-${{ github.sha }} |
||||
@ -1,17 +0,0 @@ |
||||
name: Setup CI Runner |
||||
# TODO(b/267357823) Consider moving this to it's own repository so we can |
||||
# include the call to actions/checkout. |
||||
description: Setup any platform-specific adjustments we need to make for CI |
||||
runs: |
||||
using: 'composite' |
||||
steps: |
||||
- name: Fix Windows line breaks |
||||
if: runner.os == 'Windows' |
||||
shell: bash |
||||
run: find . -type f -print0 | xargs -0 d2u 2>/dev/null |
||||
|
||||
- name: Install bazelrc files |
||||
shell: bash |
||||
run: | |
||||
cp ci/*.bazelrc . |
||||
cp -f ${{ runner.os }}.bazelrc .bazelrc |
||||
Loading…
Reference in new issue