From 9df7b0aa65bbdd518a40c30cdf0225cac0bb3ade Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Wed, 1 Nov 2023 12:17:02 -0700 Subject: [PATCH 01/11] Migrate tests to windows 2022 by default. For the windows 2019 test, we need to upgrade python for the latest gcloud to work. PiperOrigin-RevId: 578600130 --- .github/workflows/test_cpp.yml | 35 +++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 5b0856604d..5677238b79 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -310,7 +310,7 @@ jobs: os: macos-12 bazel: //src/... - name: Windows - os: windows-2019 + os: windows-2022 bazel: //src/... @com_google_protobuf_examples//... --test_tag_filters=-conformance --build_tag_filters=-conformance name: ${{ matrix.name }} Bazel runs-on: ${{ matrix.os }} @@ -335,36 +335,34 @@ jobs: os: macos-12 flags: -DCMAKE_CXX_STANDARD=14 - name: Windows CMake + os: windows-2022 + flags: >- + -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF + -Dprotobuf_BUILD_SHARED_LIBS=OFF + -Dprotobuf_BUILD_EXAMPLES=ON + vsversion: '2022' + - name: Windows CMake 2019 os: windows-2019 flags: >- -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dprotobuf_BUILD_EXAMPLES=ON vsversion: '2019' - # TODO(b/285566773) Re-enable this test. - # This is broken due to a github runner update. - # See https://github.com/actions/runner-images/issues/7662 for more details - #- name: Windows CMake 2022 - # os: windows-2022 - # flags: >- - # -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF - # -Dprotobuf_BUILD_SHARED_LIBS=OFF - # -Dprotobuf_BUILD_EXAMPLES=ON - # vsversion: '2022' + python-version: '3.8' - name: Windows CMake Shared - os: windows-2019 + os: windows-2022 flags: >- -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_BUILD_SHARED_LIBS=ON - vsversion: '2019' + vsversion: '2022' - name: Windows CMake Install - os: windows-2019 + os: windows-2022 install-flags: -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_BUILD_TESTS=OFF flags: >- -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_REMOVE_INSTALLED_HEADERS=ON -Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF - vsversion: '2019' + vsversion: '2022' name: ${{ matrix.name }} runs-on: ${{ matrix.os }} steps: @@ -374,6 +372,13 @@ jobs: ref: ${{ inputs.safe-checkout }} submodules: recursive + - name: Install Python + if: ${{ matrix.python-version }} + uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - name: Setup ccache uses: protocolbuffers/protobuf-ci/ccache@v1 with: From cd7a77948704ee431093e5137011331c184f8d17 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Wed, 1 Nov 2023 16:18:47 -0700 Subject: [PATCH 02/11] Fix windows 2019 build. setup-gcloud doesn't honor the environment variables set by setup-python, so we need to manually set CLOUDSDK_PYTHON before running it. PiperOrigin-RevId: 578674246 --- .github/workflows/test_cpp.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 5677238b79..254e441cd3 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -348,6 +348,7 @@ jobs: -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dprotobuf_BUILD_EXAMPLES=ON vsversion: '2019' + # windows-2019 has python3.7 installed, which is incompatible with the latest gcloud python-version: '3.8' - name: Windows CMake Shared os: windows-2022 @@ -372,12 +373,17 @@ jobs: ref: ${{ inputs.safe-checkout }} submodules: recursive + # Workaround for incompatibility between gcloud and windows-2019 runners. - name: Install Python if: ${{ matrix.python-version }} uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 with: python-version: ${{ matrix.python-version }} cache: pip + - name: Use custom python for gcloud + if: ${{ matrix.python-version }} + run: echo "CLOUDSDK_PYTHON=${Python3_ROOT_DIR}\\python3" >> $GITHUB_ENV + shell: bash - name: Setup ccache uses: protocolbuffers/protobuf-ci/ccache@v1 From f9612c2973d31295e04a401401a2e1bdf089038f Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Wed, 1 Nov 2023 22:17:29 -0700 Subject: [PATCH 03/11] Enable caching of pip dependencies PiperOrigin-RevId: 578740011 --- .github/workflows/test_python.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml index 3547635277..5daf2d365e 100644 --- a/.github/workflows/test_python.yml +++ b/.github/workflows/test_python.yml @@ -73,6 +73,7 @@ jobs: with: python-version: ${{ matrix.version }} cache: pip + cache-dependency-path: 'python/requirements.txt' - name: Validate version run: python3 --version | grep ${{ matrix.version }} || (echo "Invalid Python version - $(python3 --version)" && exit 1) From 88c3493c961f6e48079def21444c0a1753d207c9 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 2 Nov 2023 11:45:18 -0700 Subject: [PATCH 04/11] Remove hardcoded requirements.txt --- .github/workflows/test_python.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml index 5daf2d365e..3547635277 100644 --- a/.github/workflows/test_python.yml +++ b/.github/workflows/test_python.yml @@ -73,7 +73,6 @@ jobs: with: python-version: ${{ matrix.version }} cache: pip - cache-dependency-path: 'python/requirements.txt' - name: Validate version run: python3 --version | grep ${{ matrix.version }} || (echo "Invalid Python version - $(python3 --version)" && exit 1) From 984c430464b2edbfb1bd4c90d7cf2b18fbb35511 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Sat, 21 Oct 2023 09:37:33 -0700 Subject: [PATCH 05/11] Update `cc_file_list_aspect` to handle targets with missing `hdrs/textual_hdrs` Our stale file regeneration logic broke with Bazel 6.4.0, and I suspect it was caused by this change: https://github.com/bazelbuild/bazel/pull/19534 Our logic assumed that any target with a `CcInfo` provider must have `hdrs` and `textual_hdrs` attributes, but it seems that this is no longer true for `cc_proto_library` starting with Bazel 6.4.0. The fix is just to use `getattr` and treat the item as an empty list if it's missing. PiperOrigin-RevId: 575473886 --- pkg/cc_dist_library.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cc_dist_library.bzl b/pkg/cc_dist_library.bzl index 42553ca3f8..9fc2c414e6 100644 --- a/pkg/cc_dist_library.bzl +++ b/pkg/cc_dist_library.bzl @@ -170,12 +170,12 @@ def _cc_file_list_aspect_impl(target, ctx): return [CcFileList( hdrs = _get_transitive_sources( - _flatten_target_files(rule_attr.hdrs).to_list(), + _flatten_target_files(getattr(rule_attr, "hdrs", [])).to_list(), "hdrs", rule_attr.deps, ), textual_hdrs = _get_transitive_sources( - _flatten_target_files(rule_attr.textual_hdrs).to_list(), + _flatten_target_files(getattr(rule_attr, "textual_hdrs", [])).to_list(), "textual_hdrs", rule_attr.deps, ), From dd53f9f3627ed088fb3c88e30b22091a50044983 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 2 Nov 2023 12:34:11 -0700 Subject: [PATCH 06/11] Bump rules_python --- protobuf_deps.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index 9a5001e7a9..35fe0f42c7 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -101,8 +101,8 @@ def protobuf_deps(): _github_archive( name = "rules_python", repo = "https://github.com/bazelbuild/rules_python", - commit = "912a5051f51581784fd64094f6bdabf93f6d698f", # 0.14.0 - sha256 = "a3e4b4ade7c4a52e757b16a16e94d0b2640333062180cba577d81fac087a501d", + commit = "02b521fce3c7b36b05813aa986d72777cc3ee328", # 0.24.0 + sha256 = "f9e4f6acf82449324d56669bda4bdb28b48688ad2990d8b39fa5b93ed39c9ad1", ) if not native.existing_rule("rules_ruby"): From 63a3a849d9fe819a28bd74f54ef20fac4fe6beb5 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 2 Nov 2023 13:07:08 -0700 Subject: [PATCH 07/11] Error on staleness failure --- .github/workflows/staleness_check.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/staleness_check.yml b/.github/workflows/staleness_check.yml index 5911b75a10..9364d046c6 100644 --- a/.github/workflows/staleness_check.yml +++ b/.github/workflows/staleness_check.yml @@ -50,6 +50,7 @@ jobs: # tests along with user changes. Any stale files will be automatically fixed in a follow-up # commit. run: | + set -ex if [[ -z $COMMIT_TRIGGERED_RUN || -z $MAIN_RUN ]]; then bazel query 'attr(tags, "staleness_test", //...)' | xargs bazel test $BAZEL_FLAGS else From 47d91b7be6b65fe599179d12c7385e30b67e4979 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 2 Nov 2023 13:25:02 -0700 Subject: [PATCH 08/11] Revert "Bump rules_python" This reverts commit dd53f9f3627ed088fb3c88e30b22091a50044983. --- protobuf_deps.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index 35fe0f42c7..9a5001e7a9 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -101,8 +101,8 @@ def protobuf_deps(): _github_archive( name = "rules_python", repo = "https://github.com/bazelbuild/rules_python", - commit = "02b521fce3c7b36b05813aa986d72777cc3ee328", # 0.24.0 - sha256 = "f9e4f6acf82449324d56669bda4bdb28b48688ad2990d8b39fa5b93ed39c9ad1", + commit = "912a5051f51581784fd64094f6bdabf93f6d698f", # 0.14.0 + sha256 = "a3e4b4ade7c4a52e757b16a16e94d0b2640333062180cba577d81fac087a501d", ) if not native.existing_rule("rules_ruby"): From 884a1cde404b6a3043d1a82a251ae039bf2bdb67 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 2 Nov 2023 13:45:38 -0700 Subject: [PATCH 09/11] Switch back to 2019 due to Bazel 5 issues in 2022 --- .github/workflows/test_cpp.yml | 40 ++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 254e441cd3..117a79c066 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -319,6 +319,17 @@ jobs: uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 with: ref: ${{ inputs.safe-checkout }} + # Workaround for incompatibility between gcloud and windows-2019 runners. + - name: Install Python + if: ${{ matrix.python-version }} + uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - name: Use custom python for gcloud + if: ${{ matrix.python-version }} + run: echo "CLOUDSDK_PYTHON=${Python3_ROOT_DIR}\\python3" >> $GITHUB_ENV + shell: bash - name: Run tests uses: ./.github/actions/bazel with: @@ -335,13 +346,6 @@ jobs: os: macos-12 flags: -DCMAKE_CXX_STANDARD=14 - name: Windows CMake - os: windows-2022 - flags: >- - -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF - -Dprotobuf_BUILD_SHARED_LIBS=OFF - -Dprotobuf_BUILD_EXAMPLES=ON - vsversion: '2022' - - name: Windows CMake 2019 os: windows-2019 flags: >- -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF @@ -350,20 +354,34 @@ jobs: vsversion: '2019' # windows-2019 has python3.7 installed, which is incompatible with the latest gcloud python-version: '3.8' + # TODO(b/285566773) Re-enable this test. + # This is broken due to a github runner update. + # See https://github.com/actions/runner-images/issues/7662 for more details + #- name: Windows CMake 2022 + # os: windows-2022 + # flags: >- + # -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF + # -Dprotobuf_BUILD_SHARED_LIBS=OFF + # -Dprotobuf_BUILD_EXAMPLES=ON + # vsversion: '2022' - name: Windows CMake Shared - os: windows-2022 + os: windows-2019 flags: >- -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_BUILD_SHARED_LIBS=ON - vsversion: '2022' + vsversion: '2019' + # windows-2019 has python3.7 installed, which is incompatible with the latest gcloud + python-version: '3.8' - name: Windows CMake Install - os: windows-2022 + os: windows-2019 install-flags: -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_BUILD_TESTS=OFF flags: >- -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_REMOVE_INSTALLED_HEADERS=ON -Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF - vsversion: '2022' + vsversion: '2019' + # windows-2019 has python3.7 installed, which is incompatible with the latest gcloud + python-version: '3.8' name: ${{ matrix.name }} runs-on: ${{ matrix.os }} steps: From 422578de555f987e6a42ba31c97e203d32a6170e Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 2 Nov 2023 13:50:14 -0700 Subject: [PATCH 10/11] Remove pip setup and the one test depending on it --- WORKSPACE | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 4c403ac43b..31acd315bf 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -99,18 +99,6 @@ system_python( minimum_python_version = "3.7", ) -load("@system_python//:pip.bzl", "pip_parse") -pip_parse( - name="pip_deps", - requirements = "@upb//python:requirements.txt", - requirements_overrides = { - "3.11": "@upb//python:requirements_311.txt", - }, -) - -load("@pip_deps//:requirements.bzl", "install_deps") -install_deps() - load("@utf8_range//:workspace_deps.bzl", "utf8_range_deps") utf8_range_deps() From f0aaf21c79cfc8872818f0639141c2f118665921 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 2 Nov 2023 14:02:13 -0700 Subject: [PATCH 11/11] Followup fixes --- .github/workflows/test_cpp.yml | 10 ++++------ .github/workflows/test_csharp.yml | 2 +- .github/workflows/test_python.yml | 4 ++-- python/BUILD.bazel | 17 +++++++++-------- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 117a79c066..cfac011b7e 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -310,7 +310,7 @@ jobs: os: macos-12 bazel: //src/... - name: Windows - os: windows-2022 + os: windows-2019 bazel: //src/... @com_google_protobuf_examples//... --test_tag_filters=-conformance --build_tag_filters=-conformance name: ${{ matrix.name }} Bazel runs-on: ${{ matrix.os }} @@ -321,14 +321,13 @@ jobs: ref: ${{ inputs.safe-checkout }} # Workaround for incompatibility between gcloud and windows-2019 runners. - name: Install Python - if: ${{ matrix.python-version }} uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + if: runner.os == 'Windows' with: - python-version: ${{ matrix.python-version }} - cache: pip + python-version: '3.8' - name: Use custom python for gcloud - if: ${{ matrix.python-version }} run: echo "CLOUDSDK_PYTHON=${Python3_ROOT_DIR}\\python3" >> $GITHUB_ENV + if: runner.os == 'Windows' shell: bash - name: Run tests uses: ./.github/actions/bazel @@ -397,7 +396,6 @@ jobs: uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 with: python-version: ${{ matrix.python-version }} - cache: pip - name: Use custom python for gcloud if: ${{ matrix.python-version }} run: echo "CLOUDSDK_PYTHON=${Python3_ROOT_DIR}\\python3" >> $GITHUB_ENV diff --git a/.github/workflows/test_csharp.yml b/.github/workflows/test_csharp.yml index 4d5770a9cc..7536f85b3d 100644 --- a/.github/workflows/test_csharp.yml +++ b/.github/workflows/test_csharp.yml @@ -37,7 +37,7 @@ jobs: - name: Setup dotnet uses: actions/setup-dotnet@607fce577a46308457984d59e4954e075820f10a # v3.0.3 with: - dotnet-version: '6.0.x' + dotnet-version: '3.1.x' - name: Build run: dotnet build csharp/src/Google.Protobuf.sln diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml index 3547635277..18767bff5c 100644 --- a/.github/workflows/test_python.yml +++ b/.github/workflows/test_python.yml @@ -17,7 +17,7 @@ jobs: version: [ "3.7", "3.8", "3.9", "3.10" ] include: - type: Pure - targets: //python/... @upb//python/... //python:python_version + targets: //python/... //python:python_version flags: --define=use_fast_cpp_protos=false - type: C++ targets: //python/... //python:python_version @@ -54,7 +54,7 @@ jobs: version: [ "3.10" ] include: - type: Pure - targets: //python/... @upb//python/... //python:python_version + targets: //python/... //python:python_version - type: C++ targets: //python/... //python:python_version flags: --define=use_fast_cpp_protos=true diff --git a/python/BUILD.bazel b/python/BUILD.bazel index f6fe071152..0197282399 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -8,7 +8,8 @@ load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") load("@rules_python//python:defs.bzl", "py_library") -load("@pip_deps//:requirements.bzl", "requirement") +# It's no longer feasible to test this in our CI, and python is out of it's support window. +#load("@pip_deps//:requirements.bzl", "requirement") load("//:protobuf.bzl", "internal_py_proto_library") load("//build_defs:arch_tests.bzl", "aarch64_test", "x86_64_test") load("//build_defs:cpp_opts.bzl", "COPTS") @@ -325,13 +326,13 @@ internal_py_test( data = ["//src/google/protobuf:testdata"], ) -internal_py_test( - name = "numpy_test", - srcs = ["google/protobuf/internal/numpy_test.py"], - deps = [ - requirement("numpy"), - ], -) +#internal_py_test( +# name = "numpy_test", +# srcs = ["google/protobuf/internal/numpy_test.py"], +# deps = [ +# requirement("numpy"), +# ], +#) internal_py_test( name = "proto_builder_test",