From 5cb1b41a808dd76109398b21d5cd240629679da0 Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Thu, 26 Jan 2023 12:03:17 -0800 Subject: [PATCH] Enabled caching for Python wheel builds. To avoid duplication, I factored out the bazel caching step into a separate action. PiperOrigin-RevId: 504906052 --- .github/actions/setup-bazel-cache/action.yml | 21 ++++++++++++++++ .github/workflows/bazel_tests.yml | 26 ++++++-------------- .github/workflows/python_tests.yml | 5 +++- 3 files changed, 33 insertions(+), 19 deletions(-) create mode 100644 .github/actions/setup-bazel-cache/action.yml diff --git a/.github/actions/setup-bazel-cache/action.yml b/.github/actions/setup-bazel-cache/action.yml new file mode 100644 index 0000000000..ac9ea83329 --- /dev/null +++ b/.github/actions/setup-bazel-cache/action.yml @@ -0,0 +1,21 @@ +name: "Setup Bazel Caching" +description: "Sets up Bazel caching" +inputs: + cache_url: + description: "URL of the Bazel cache to read/write" + required: false + default: https://storage.googleapis.com/protobuf-bazel-cache/upb + read_only: + description: "If true, we can read from the cache but not write it." + required: false + default: ${{ github.event.pull_request.head.repo.full_name != 'protocolbuffers/upb' }} +outputs: + cache_args: + description: "Caching related arguments to pass to 'bazel build" + value: --remote_cache=${{ inputs.cache_url }} ${{ steps.set_auth_arg.outputs.auth_arg }} +runs: + using: "composite" + steps: + - id: set_auth_arg + run: echo auth_arg=${{ inputs.read_only == 'true' && '--remote_upload_local_results=false' || '--google_default_credentials' }} >> $GITHUB_OUTPUT + shell: bash diff --git a/.github/workflows/bazel_tests.yml b/.github/workflows/bazel_tests.yml index e221dddc89..024427abbe 100644 --- a/.github/workflows/bazel_tests.yml +++ b/.github/workflows/bazel_tests.yml @@ -15,8 +15,6 @@ jobs: ubuntu: runs-on: ${{ matrix.os }} - env: - BAZEL_CACHE: --remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/upb strategy: fail-fast: false # Don't cancel all jobs if one fails. @@ -58,12 +56,9 @@ jobs: run: | ${{ matrix.CC }} --version ${{ matrix.BAZEL }} --version - - name: Set up Bazel read/write caching - run: echo "BAZEL_CACHE_AUTH=--google_default_credentials" >> $GITHUB_ENV - if: ${{ github.event.pull_request.head.repo.full_name == 'protocolbuffers/upb' }} - - name: Set up Bazel read-only caching - run: echo "BAZEL_CACHE_AUTH=--remote_upload_local_results=false" >> $GITHUB_ENV - if: ${{ github.event.pull_request.head.repo.full_name != 'protocolbuffers/upb' }} + - id: bazel-cache + name: Set up Bazel caching + uses: ./.github/actions/setup-bazel-cache - name: Setup Python venv run: rm -rf /tmp/venv && python3 -m venv /tmp/venv && source /tmp/venv/bin/activate && python3 --version - name: Install dependencies @@ -72,13 +67,11 @@ jobs: - name: Install numpy run: source /tmp/venv/bin/activate && pip3 install numpy - name: Run tests - run: cd ${{ github.workspace }} && source /tmp/venv/bin/activate && CC=${{ matrix.CC }} ${{ matrix.BAZEL }} test --test_output=errors $BAZEL_CACHE $BAZEL_CACHE_AUTH ... ${{ matrix.flags }} + run: cd ${{ github.workspace }} && source /tmp/venv/bin/activate && CC=${{ matrix.CC }} ${{ matrix.BAZEL }} test --test_output=errors ${{ steps.bazel-cache.outputs.cache_args }} ... ${{ matrix.flags }} # Temporarily disabled until Copybara fixes its cap of 30 jobs per PR. # no-python: # runs-on: ubuntu-20-large - # env: - # BAZEL_CACHE: --remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/upb # strategy: # fail-fast: false # Don't cancel all jobs if one fails. @@ -93,13 +86,10 @@ jobs: # credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }} # export_environment_variables: true # if: ${{ github.event.pull_request.head.repo.full_name == 'protocolbuffers/upb' }} - # - name: Set up Bazel read/write caching - # run: echo "BAZEL_CACHE_AUTH=--google_default_credentials" >> $GITHUB_ENV - # if: ${{ github.event.pull_request.head.repo.full_name == 'protocolbuffers/upb' }} - # - name: Set up Bazel read-only caching - # run: echo "BAZEL_CACHE_AUTH=--remote_upload_local_results=false" >> $GITHUB_ENV - # if: ${{ github.event.pull_request.head.repo.full_name != 'protocolbuffers/upb' }} + # - id: bazel-cache + # name: Set up Bazel caching + # uses: ./.github/actions/setup-bazel-cache # - name: Uninstall python # run: which python3 && sudo mv `which python3` /tmp && ! which python3 # - name: Run tests - # run: cd ${{ github.workspace }} && bazel test --test_output=errors $BAZEL_CACHE $BAZEL_CACHE_AUTH //python/... -- -//python/dist:source_wheel + # run: cd ${{ github.workspace }} && bazel test --test_output=errors ${{ steps.bazel-cache.outputs.cache_args }} //python/... -- -//python/dist:source_wheel diff --git a/.github/workflows/python_tests.yml b/.github/workflows/python_tests.yml index 723bb96554..ed1606e376 100644 --- a/.github/workflows/python_tests.yml +++ b/.github/workflows/python_tests.yml @@ -32,8 +32,11 @@ jobs: run: gcloud auth configure-docker -q us-docker.pkg.dev - name: Pull Docker Image run: docker pull $DOCKER_IMAGE + - id: bazel-cache + name: Set up Bazel caching + uses: ./.github/actions/setup-bazel-cache - name: Build Wheels - run: cd ${{ github.workspace }} && docker run -e GOOGLE_APPLICATION_CREDENTIALS=/workspace/$(basename $GOOGLE_APPLICATION_CREDENTIALS) -v$PWD:/workspace $DOCKER_IMAGE build $BAZEL_CACHE --crosstool_top=@com_google_protobuf//toolchain:clang_suite --symlink_prefix=/ -c dbg python/dist python/dist:test_wheel python/dist:source_wheel + run: cd ${{ github.workspace }} && docker run -e GOOGLE_APPLICATION_CREDENTIALS=/workspace/$(basename $GOOGLE_APPLICATION_CREDENTIALS) -v$PWD:/workspace $DOCKER_IMAGE build $BAZEL_CACHE --crosstool_top=@com_google_protobuf//toolchain:clang_suite --symlink_prefix=/ -c dbg python/dist ${{ steps.bazel-cache.outputs.cache_args }} python/dist:test_wheel python/dist:source_wheel - name: Move Wheels run: mkdir wheels && find _build/out \( -name 'protobuf*.whl' -o -name 'protobuf*.tar.gz' \) -exec mv '{}' wheels ';' - uses: actions/upload-artifact@v3