From 769b27479aa4d610240c488b40e217e1362505ea Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Wed, 20 Dec 2023 15:26:00 -0800 Subject: [PATCH] [Deps] Update protobuf & upb to 3.25.1 (#35315) On top of https://github.com/grpc/grpc/pull/35285, upb is also upgraded in this PR. Closes #35315 PiperOrigin-RevId: 592672254 --- bazel/grpc_deps.bzl | 8 +- build_handwritten.yaml | 2 +- src/csharp/build/dependencies.props | 2 +- .../!ProtoCompiler-gRPCCppPlugin.podspec | 2 +- .../!ProtoCompiler-gRPCPlugin.podspec | 2 +- src/objective-c/!ProtoCompiler.podspec | 2 +- third_party/README.md | 6 +- third_party/protobuf | 2 +- third_party/protobuf.patch | 2 +- third_party/upb/.bazelci/presubmit.yml | 16 -- third_party/upb/.bazelignore | 1 - third_party/upb/.bazelrc | 60 ------ third_party/upb/.clang-format | 3 - .../actions/setup-bazel-cache/action.yml | 21 --- .../upb/.github/workflows/bazel_tests.yml | 101 ---------- .../upb/.github/workflows/clang_format.yml | 22 --- .../upb/.github/workflows/generate_files.yml | 24 --- .../upb/.github/workflows/mergeable.yml | 15 -- .../upb/.github/workflows/python_tests.yml | 173 ------------------ third_party/upb/.gitignore | 6 - third_party/upb/upb/reflection/def_pool.h | 4 +- tools/distrib/python/grpc_version.py | 2 +- .../python/grpcio_tools/grpc_version.py | 2 +- .../python/grpcio_tools/protoc_lib_deps.py | 2 +- .../distrib/python/xds_protos/grpc_version.py | 2 +- tools/run_tests/sanity/check_submodules.sh | 2 +- 26 files changed, 21 insertions(+), 463 deletions(-) delete mode 100644 third_party/upb/.bazelci/presubmit.yml delete mode 100644 third_party/upb/.bazelignore delete mode 100644 third_party/upb/.bazelrc delete mode 100644 third_party/upb/.clang-format delete mode 100644 third_party/upb/.github/actions/setup-bazel-cache/action.yml delete mode 100644 third_party/upb/.github/workflows/bazel_tests.yml delete mode 100644 third_party/upb/.github/workflows/clang_format.yml delete mode 100644 third_party/upb/.github/workflows/generate_files.yml delete mode 100644 third_party/upb/.github/workflows/mergeable.yml delete mode 100644 third_party/upb/.github/workflows/python_tests.yml delete mode 100644 third_party/upb/.gitignore diff --git a/bazel/grpc_deps.bzl b/bazel/grpc_deps.bzl index 1a08956fccb..6765a8b6805 100644 --- a/bazel/grpc_deps.bzl +++ b/bazel/grpc_deps.bzl @@ -258,12 +258,12 @@ def grpc_deps(): if "com_google_protobuf" not in native.existing_rules(): http_archive( name = "com_google_protobuf", - sha256 = "7ed5fc41fe1614e551025f8e14b79b026a015b3ed337d38920c586f3ea35d818", - strip_prefix = "protobuf-6b5d8db01fe47478e8d400f550e797e6230d464e", + sha256 = "70f480fe9cb0c6829dbf6be3c388103313aacb65de667b86d981bbc9eaedb905", + strip_prefix = "protobuf-7f94235e552599141950d7a4a3eaf93bc87d1b22", urls = [ # https://github.com/protocolbuffers/protobuf/commits/v25.0 - "https://storage.googleapis.com/grpc-bazel-mirror/github.com/protocolbuffers/protobuf/archive/6b5d8db01fe47478e8d400f550e797e6230d464e.tar.gz", - "https://github.com/protocolbuffers/protobuf/archive/6b5d8db01fe47478e8d400f550e797e6230d464e.tar.gz", + "https://storage.googleapis.com/grpc-bazel-mirror/github.com/protocolbuffers/protobuf/archive/7f94235e552599141950d7a4a3eaf93bc87d1b22.tar.gz", + "https://github.com/protocolbuffers/protobuf/archive/7f94235e552599141950d7a4a3eaf93bc87d1b22.tar.gz", ], patches = [ "@com_github_grpc_grpc//third_party:protobuf.patch", diff --git a/build_handwritten.yaml b/build_handwritten.yaml index a700fdd2da7..137a70eeabf 100644 --- a/build_handwritten.yaml +++ b/build_handwritten.yaml @@ -15,7 +15,7 @@ settings: core_version: 37.0.0 csharp_major_version: 2 g_stands_for: grand - protobuf_version: 3.25.0 + protobuf_version: 3.25.1 version: 1.61.0-dev configs: asan: diff --git a/src/csharp/build/dependencies.props b/src/csharp/build/dependencies.props index 11a1cfa9faf..9b9012c9063 100644 --- a/src/csharp/build/dependencies.props +++ b/src/csharp/build/dependencies.props @@ -2,6 +2,6 @@ 2.61.0-dev - 3.25.0 + 3.25.1 diff --git a/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec index 2a0a4515b76..50c928143f8 100644 --- a/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCCppPlugin.podspec @@ -100,7 +100,7 @@ Pod::Spec.new do |s| s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.25.0' + s.dependency '!ProtoCompiler', '3.25.1' s.ios.deployment_target = '10.0' s.osx.deployment_target = '10.12' diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 684d14ea056..d8affc2c6f8 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -102,7 +102,7 @@ Pod::Spec.new do |s| s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.25.0' + s.dependency '!ProtoCompiler', '3.25.1' s.ios.deployment_target = '10.0' s.osx.deployment_target = '10.12' diff --git a/src/objective-c/!ProtoCompiler.podspec b/src/objective-c/!ProtoCompiler.podspec index e931b00f0db..b26f9ef9260 100644 --- a/src/objective-c/!ProtoCompiler.podspec +++ b/src/objective-c/!ProtoCompiler.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler' - v = '3.25.0' + v = '3.25.1' s.version = v s.summary = 'The Protobuf Compiler (protoc) generates Objective-C files from .proto files' s.description = <<-DESC diff --git a/third_party/README.md b/third_party/README.md index 136ca7502d5..334ecf09c9d 100644 --- a/third_party/README.md +++ b/third_party/README.md @@ -123,10 +123,10 @@ In addition, please perform the following two steps to generate the Python `xds- Since upb is vendored in the gRPC repo, you cannot use submodule to update it. Please follow the steps below; 1. Update third_party/upb directory by running - `export GRPC_ROOT=~/git/grpc` - - `wget https://github.com/protocolbuffers/upb/archive/refs/heads/main.zip` + - `wget https://github.com/protocolbuffers/protobuf/releases/download/v25.1/protobuf-25.1.zip` - `rm -rf $GRPC_ROOT/third_party/upb` - - `unzip main.zip -d $GRPC_ROOT/third_party` - - `mv $GRPC_ROOT/third_party/upb-main $GRPC_ROOT/third_party/upb` + - `unzip protobuf-25.1.zip -d /tmp/protobuf` + - `cp -r /tmp/protobuf/protobuf-25.1/upb $GRPC_ROOT/third_party/upb` 2. Update the dependency in `grpc_deps.bzl` to the same commit 3. Populate the bazel download mirror by running `bazel/update_mirror.sh` 4. Run `tools/buildgen/generate_projects.sh` to regenerate the generated files diff --git a/third_party/protobuf b/third_party/protobuf index 6b5d8db01fe..7f94235e552 160000 --- a/third_party/protobuf +++ b/third_party/protobuf @@ -1 +1 @@ -Subproject commit 6b5d8db01fe47478e8d400f550e797e6230d464e +Subproject commit 7f94235e552599141950d7a4a3eaf93bc87d1b22 diff --git a/third_party/protobuf.patch b/third_party/protobuf.patch index a7b1cad4a0e..7feb79b06a0 100644 --- a/third_party/protobuf.patch +++ b/third_party/protobuf.patch @@ -5,7 +5,7 @@ index 45a6c20c5..c28dd8852 100755 @@ -8,3 +8,9 @@ # Copyright 2007 Google Inc. All Rights Reserved. - __version__ = '4.25.0' + __version__ = '4.25.1' + +if __name__ != '__main__': + try: diff --git a/third_party/upb/.bazelci/presubmit.yml b/third_party/upb/.bazelci/presubmit.yml deleted file mode 100644 index c7630471420..00000000000 --- a/third_party/upb/.bazelci/presubmit.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -tasks: - ubuntu: - platform: ubuntu2004 - shell_commands: - - "sudo apt -y update && sudo apt -y install libreadline-dev cmake rsync" - build_flags: - - "--incompatible_disallow_empty_glob" - test_targets: - - //... - macos: - platform: macos - build_flags: - - "--incompatible_disallow_empty_glob" - test_targets: - - //... diff --git a/third_party/upb/.bazelignore b/third_party/upb/.bazelignore deleted file mode 100644 index e35d8850c96..00000000000 --- a/third_party/upb/.bazelignore +++ /dev/null @@ -1 +0,0 @@ -_build diff --git a/third_party/upb/.bazelrc b/third_party/upb/.bazelrc deleted file mode 100644 index 8fc9ca820be..00000000000 --- a/third_party/upb/.bazelrc +++ /dev/null @@ -1,60 +0,0 @@ -# temporary fix for https://github.com/bazelbuild/bazel/issues/12905 on macOS -build --features=-debug_prefix_map_pwd_is_dot - -# Pin to C++17 -build --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 -build:cpp17_msvc --cxxopt=/std:c++17 --host_cxxopt=/std:c++17 - -# Use our custom-configured c++ toolchain. - -build:m32 --copt=-m32 --linkopt=-m32 -build:asan --copt=-fsanitize=address --linkopt=-fsanitize=address -build:msan --copt=-fsanitize=memory --linkopt=-fsanitize=memory - -# For Valgrind, we have to disable checks of "possible" leaks because the Python -# interpreter does the sorts of things that flag Valgrind "possible" leak checks. -# Ideally we could enforce a stricter check for the non-Python tests, but I don't -# know of an easy way to do that. -# -# We also have to disable pymalloc to avoid triggering Valgrind. -build:valgrind --run_under='valgrind --leak-check=full --track-origins=yes --trace-children=yes --show-leak-kinds=all --error-exitcode=1 --num-callers=500 ' --action_env=PYTHONMALLOC=malloc - -build:ubsan --copt=-fsanitize=undefined --linkopt=-fsanitize=undefined --action_env=UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 -# Workaround for the fact that Bazel links with $CC, not $CXX -# https://github.com/bazelbuild/bazel/issues/11122#issuecomment-613746748 -build:ubsan --copt=-fno-sanitize=function --copt=-fno-sanitize=vptr -# Workaround for https://bugs.llvm.org/show_bug.cgi?id=16404 -build:ubsan --linkopt=--rtlib=compiler-rt --linkopt=-lunwind - -build:Werror --copt=-Werror -build:Werror --per_file_copt=json/parser@-Wno-error -build:Werror --per_file_copt=com_google_protobuf@-Wno-error - -# GCC's -fanalyzer, a deeper static analysis than normal warnings. -build:analyzer --copt=-fanalyzer --copt=-Werror -build:analyzer --per_file_copt=json/parser@-fno-analyzer -build:analyzer --per_file_copt=com_google_protobuf@-fno-analyzer -build:analyzer --per_file_copt=com_github_google_benchmark@-fno-analyzer - -# --config=asan-libfuzzer -build:asan-libfuzzer --action_env=CC=clang -build:asan-libfuzzer --action_env=CXX=clang++ -build:asan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing//fuzzing/engines:libfuzzer -build:asan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_instrumentation=libfuzzer -build:asan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_sanitizer=asan - -# --config=msan-libfuzzer -build:msan-libfuzzer --action_env=CC=clang -build:msan-libfuzzer --action_env=CXX=clang++ -build:msan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing//fuzzing/engines:libfuzzer -build:msan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_instrumentation=libfuzzer -build:msan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_sanitizer=msan - -# --config=ubsan-libfuzzer -build:ubsan-libfuzzer --action_env=CC=clang -build:ubsan-libfuzzer --action_env=CXX=clang++ -build:ubsan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing//fuzzing/engines:libfuzzer -build:ubsan-libfuzzer --@rules_fuzzing//fuzzing:cc_engine_instrumentation=libfuzzer -build:ubsan-libfuzzer --copt=-fsanitize=undefined -build:ubsan-libfuzzer --linkopt=-fsanitize=undefined -build:ubsan-libfuzzer --linkopt=-fsanitize-link-c++-runtime diff --git a/third_party/upb/.clang-format b/third_party/upb/.clang-format deleted file mode 100644 index 24d97f2b22d..00000000000 --- a/third_party/upb/.clang-format +++ /dev/null @@ -1,3 +0,0 @@ -BasedOnStyle: Google -DerivePointerAlignment: false -PointerAlignment: Left diff --git a/third_party/upb/.github/actions/setup-bazel-cache/action.yml b/third_party/upb/.github/actions/setup-bazel-cache/action.yml deleted file mode 100644 index ac9ea833297..00000000000 --- a/third_party/upb/.github/actions/setup-bazel-cache/action.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: "Setup Bazel Caching" -description: "Sets up Bazel caching" -inputs: - cache_url: - description: "URL of the Bazel cache to read/write" - required: false - default: https://storage.googleapis.com/protobuf-bazel-cache/upb - read_only: - description: "If true, we can read from the cache but not write it." - required: false - default: ${{ github.event.pull_request.head.repo.full_name != 'protocolbuffers/upb' }} -outputs: - cache_args: - description: "Caching related arguments to pass to 'bazel build" - value: --remote_cache=${{ inputs.cache_url }} ${{ steps.set_auth_arg.outputs.auth_arg }} -runs: - using: "composite" - steps: - - id: set_auth_arg - run: echo auth_arg=${{ inputs.read_only == 'true' && '--remote_upload_local_results=false' || '--google_default_credentials' }} >> $GITHUB_OUTPUT - shell: bash diff --git a/third_party/upb/.github/workflows/bazel_tests.yml b/third_party/upb/.github/workflows/bazel_tests.yml deleted file mode 100644 index 2cf83dfa111..00000000000 --- a/third_party/upb/.github/workflows/bazel_tests.yml +++ /dev/null @@ -1,101 +0,0 @@ -name: Bazel Tests - -on: - push: - branches: - - main - - '[0-9]+.x' - pull_request: - branches: - - main - - '[0-9]+.x' - workflow_dispatch: - -jobs: - - ubuntu: - runs-on: ${{ matrix.os }} - - strategy: - fail-fast: false # Don't cancel all jobs if one fails. - matrix: - include: - - { NAME: "Fastbuild", BAZEL: bazel, CC: clang, os: ubuntu-20-large, flags: "" } - - { NAME: "Optimized", BAZEL: bazel, CC: clang, os: ubuntu-20-large, flags: "-c opt" } # Some warnings only fire with -c opt - - { NAME: "GCC Optimized", BAZEL: bazel, CC: gcc-12, os: ubuntu-22.04, flags: "-c opt" } - - { NAME: "FastTable", BAZEL: bazel, CC: clang, os: ubuntu-20-large, flags: "--//:fasttable_enabled=true -- -cmake:test_generated_files" } - - { NAME: "ASAN", BAZEL: bazel, CC: clang, os: ubuntu-20-large, flags: "--config=asan -c dbg -- -benchmarks:benchmark -python/..." } - - { NAME: "UBSAN", BAZEL: bazel, CC: clang, os: ubuntu-20-large, flags: "--config=ubsan -c dbg -- -benchmarks:benchmark -python/... -lua/...", install: "libunwind-dev" } - - { NAME: "32-bit", BAZEL: bazel, CC: clang, os: ubuntu-20-large, flags: "--copt=-m32 --linkopt=-m32 -- -... benchmarks:benchmark ", install: "g++-multilib" } - - { NAME: "Windows", BAZEL: bazel, os: windows-2019, startup-flags: "--output_user_root=C:/tmp", flags: "--config=cpp17_msvc", targets: "upb/... upbc/... python/... protos/... protos_generator/..." } - - { NAME: "macOS", BAZEL: bazel, CC: clang, os: macos-11 } - # Current github runners are all Intel based, so just build/compile for Apple Silicon to detect issues there. - - { NAME: "macOS ARM (build only)", BAZEL: bazel, BAZEL_CMD: build, CC: clang, os: macos-11, flags: "--cpu=darwin_arm64"} - # We support two Bazel versions back per https://opensource.google/documentation/policies/cplusplus-support - - { NAME: "Bazel 5.3.0", BAZEL: bazel-5.3.0-linux-x86_64, CC: clang, os: ubuntu-20-large } - - { NAME: "Bazel 6.1.0", BAZEL: bazel-6.1.0-linux-x86_64, CC: clang, os: ubuntu-20-large } - - name: ${{ matrix.NAME }} - - steps: - - uses: actions/checkout@v2 - - name: Set up Cloud SDK - uses: google-github-actions/auth@v0 - with: - credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }} - export_environment_variables: true - if: ${{ github.event.pull_request.head.repo.full_name == 'protocolbuffers/upb' }} - - name: Download historical Bazel version - run: | - FILENAME=$HOME/bin/${{ matrix.BAZEL }} - VERSION=$(echo ${{ matrix.BAZEL }} | cut -d- -f 2 ) - mkdir -p $HOME/bin - echo $HOME/bin >> $GITHUB_PATH - wget -O $FILENAME https://github.com/bazelbuild/bazel/releases/download/$VERSION/${{ matrix.BAZEL }} - chmod a+x $FILENAME - if: ${{ matrix.BAZEL != 'bazel' }} - - name: Check compiler versions - if: matrix.CC - run: ${{ matrix.CC }} --version - - name: Check Bazel versions - run: ${{ matrix.BAZEL }} --version - - id: bazel-cache - name: Set up Bazel caching - uses: ./.github/actions/setup-bazel-cache - - name: Setup Python venv - if: ${{ runner.os != 'Windows' }} - run: rm -rf /tmp/venv && python3 -m venv /tmp/venv && source /tmp/venv/bin/activate && python3 --version - - name: Install dependencies - run: sudo apt update && sudo apt install -y ${{ matrix.install }} - if: matrix.install != '' - - name: Install numpy - run: pip3 install numpy - - name: Setup environment variables - if: matrix.CC - run: echo "CC=${{ matrix.CC }}" >> $GITHUB_ENV - - name: Run tests - run: cd ${{ github.workspace }} && ${{ matrix.BAZEL }} ${{ matrix.startup-flags }} ${{ matrix.BAZEL_CMD || 'test' }} --test_output=errors ${{ steps.bazel-cache.outputs.cache_args }} ${{ matrix.targets || '...' }} ${{ matrix.flags }} - - no-python: - runs-on: ubuntu-20-large - - strategy: - fail-fast: false # Don't cancel all jobs if one fails. - - name: "No System Python" - - steps: - - uses: actions/checkout@v2 - - name: Set up Cloud SDK - uses: google-github-actions/auth@v0 - with: - credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }} - export_environment_variables: true - if: ${{ github.event.pull_request.head.repo.full_name == 'protocolbuffers/upb' }} - - id: bazel-cache - name: Set up Bazel caching - uses: ./.github/actions/setup-bazel-cache - - name: Uninstall python - run: which python3 && sudo mv `which python3` /tmp && ! which python3 - - name: Run tests - run: cd ${{ github.workspace }} && bazel test --test_output=errors ${{ steps.bazel-cache.outputs.cache_args }} //python/... -- -//python/dist:source_wheel diff --git a/third_party/upb/.github/workflows/clang_format.yml b/third_party/upb/.github/workflows/clang_format.yml deleted file mode 100644 index 51035005865..00000000000 --- a/third_party/upb/.github/workflows/clang_format.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Check ClangFormat - -on: - push: - branches: - - main - - '[0-9]+.x' - pull_request: - branches: - - main - - '[0-9]+.x' - workflow_dispatch: - -jobs: - check_clang_format: - runs-on: ubuntu-20-large - steps: - - uses: actions/checkout@v2 - - name: Run ClangFormat - run: find . | grep -E '\.(c|h|cc)$' | grep -E -v '^./(third_party|cmake)' | xargs clang-format -i - - name: Check for differences - run: git diff --exit-code diff --git a/third_party/upb/.github/workflows/generate_files.yml b/third_party/upb/.github/workflows/generate_files.yml deleted file mode 100644 index ce9d4da3e28..00000000000 --- a/third_party/upb/.github/workflows/generate_files.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Generate Files - -# After any push to the main branch, re-generate pregenerated files. -on: - push: - branches: - - main - - '[0-9]+.x' - -jobs: - generate: - if: github.repository == 'protocolbuffers/upb' - runs-on: ubuntu-22-large - - steps: - - uses: actions/checkout@v3 - with: - # Note: this token has an expiration date, so if the workflow starts - # failing then you may need to generate a fresh token. - token: ${{ secrets.BOT_ACCESS_TOKEN }} - - name: Configure name and email address in Git - run: cd ${{ github.workspace }} && git config user.name "Protobuf Team Bot" && git config user.email "protobuf-team-bot@google.com" - - name: Commit and push update - run: cd ${{ github.workspace }} && ./cmake/push_auto_update.sh diff --git a/third_party/upb/.github/workflows/mergeable.yml b/third_party/upb/.github/workflows/mergeable.yml deleted file mode 100644 index 255bc1c92c0..00000000000 --- a/third_party/upb/.github/workflows/mergeable.yml +++ /dev/null @@ -1,15 +0,0 @@ -mergeable: - pull_requests: - label: - and: - - must_exclude: - regex: '^disposition/DO NOT MERGE' - message: 'Pull request marked not mergeable' - - must_include: - regex: 'mergeable:force-allow' - message: 'Pull requests should not be merged directly and should instead - be handled by Copybara. - - To enable Github merges, add the `mergeable:force-allow` label and get a second - approval. This should almost never be used except for releases or as a break glass - measure after discussing with the team.' \ No newline at end of file diff --git a/third_party/upb/.github/workflows/python_tests.yml b/third_party/upb/.github/workflows/python_tests.yml deleted file mode 100644 index 42d93a739b6..00000000000 --- a/third_party/upb/.github/workflows/python_tests.yml +++ /dev/null @@ -1,173 +0,0 @@ -name: Python Tests - -on: - push: - branches: - - main - - '[0-9]+.x' - pull_request: - branches: - - main - - '[0-9]+.x' - workflow_dispatch: - -jobs: - build_wheels: - name: Build Wheels - runs-on: ubuntu-large - if: ${{ github.event.pull_request.head.repo.full_name == 'protocolbuffers/upb' }} - env: - # Bazel 5.4.0. Once we have moved to toolchains, we can update to Bazel 6.x. - DOCKER_IMAGE: us-docker.pkg.dev/protobuf-build/release-containers/linux/apple@sha256:bb1d14738449916d489c1cbb062508c4dca5bd265ea3e67a2628ae40912b9b00 - - steps: - - uses: actions/checkout@v2 - - name: Set up Cloud SDK - uses: google-github-actions/auth@v0 - with: - credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }} - export_environment_variables: true - - name: Use gcloud CLI - run: gcloud info - - name: Configure Docker - run: gcloud auth configure-docker -q us-docker.pkg.dev - - name: Pull Docker Image - run: docker pull $DOCKER_IMAGE - - name: Check Bazel version - run: cd ${{ github.workspace }} && docker run -v$PWD:/workspace $DOCKER_IMAGE --version - - id: bazel-cache - name: Set up Bazel caching - uses: ./.github/actions/setup-bazel-cache - - name: Build Wheels - run: cd ${{ github.workspace }} && docker run -e GOOGLE_APPLICATION_CREDENTIALS=/workspace/$(basename $GOOGLE_APPLICATION_CREDENTIALS) -v$PWD:/workspace $DOCKER_IMAGE build $BAZEL_CACHE --crosstool_top=@com_google_protobuf//toolchain:clang_suite --@com_google_protobuf//toolchain:release=true --symlink_prefix=/ -c dbg python/dist ${{ steps.bazel-cache.outputs.cache_args }} python/dist:test_wheel python/dist:source_wheel - - name: Move Wheels - run: mkdir wheels && find _build/out \( -name 'protobuf*.whl' -o -name 'protobuf-*.tar.gz' \) -exec mv '{}' wheels ';' - - uses: actions/upload-artifact@v3 - with: - name: python-wheels - path: wheels/ - - uses: actions/upload-artifact@v3 - with: - name: requirements - # Tests shouldn't have access to the whole upb repo, upload the one file we need - path: python/requirements.txt - - test_wheels: - name: Test Wheels - needs: build_wheels - strategy: - fail-fast: false # Don't cancel all jobs if one fails. - matrix: - include: - # Linux and Mac use the limited API, so all Python versions will use a single wheel. - # As a result we can just test the oldest and newest supported Python versions and assume - # this gives us sufficient test coverage. - - { os: ubuntu-18-large, python-version: "3.7", architecture: x64, type: 'binary' } - - { os: macos-11, python-version: "3.7", architecture: x64, type: 'binary' } - - { os: ubuntu-20-large, python-version: "3.10", architecture: x64, type: 'binary' } - - { os: macos-12, python-version: "3.10", architecture: x64, type: 'binary' } - - { os: ubuntu-18-large, python-version: "3.7", architecture: x64, type: 'source' } - - { os: macos-11, python-version: "3.7", architecture: x64, type: 'source' } - - { os: ubuntu-20-large, python-version: "3.10", architecture: x64, type: 'source' } - - { os: macos-12, python-version: "3.10", architecture: x64, type: 'source' } - - # Windows uses the full API up until Python 3.10, so each of these - # jobs tests a distinct binary wheel. - - { os: windows-2019-large, python-version: "3.7", architecture: x86, type: 'binary' } - - { os: windows-2019-large, python-version: "3.8", architecture: x86, type: 'binary' } - - { os: windows-2019-large, python-version: "3.9", architecture: x86, type: 'binary' } - - { os: windows-2019-large, python-version: "3.10", architecture: x86, type: 'binary' } - - { os: windows-2019-large, python-version: "3.7", architecture: x64, type: 'binary' } - - { os: windows-2019-large, python-version: "3.8", architecture: x64, type: 'binary' } - - { os: windows-2019-large, python-version: "3.9", architecture: x64, type: 'binary' } - - { os: windows-2019-large, python-version: "3.10", architecture: x64, type: 'binary' } - runs-on: ${{ matrix.os }} - defaults: - run: - shell: bash - steps: - - name: Download Wheels - uses: actions/download-artifact@v3 - with: - name: python-wheels - path: wheels - - name: Download Requirements - uses: actions/download-artifact@v3 - with: - name: requirements - path: requirements - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - architecture: ${{ matrix.architecture }} - - name: Setup Python venv - run: | - python -m pip install --upgrade pip - python -m venv env - # Windows uses 'Scripts' instead of 'bin' - source env/bin/activate || source env/Scripts/activate - echo "VIRTUAL ENV:" $VIRTUAL_ENV - - name: Install tzdata - run: pip install tzdata - # Only needed on Windows, Linux ships with tzdata. - if: ${{ contains(matrix.os, 'windows') }} - - name: Install requirements - run: pip install -r requirements/requirements.txt - - name: Install Protobuf Binary Wheel - run: pip install -vvv --no-index --find-links wheels protobuf - if: ${{ matrix.type == 'binary' }} - - name: Install Protobuf Source Wheel - run: | - cd wheels - tar -xzvf *.tar.gz - cd protobuf-*/ - pip install . - if: ${{ matrix.type == 'source' }} - - name: Test that module is importable - run: python -v -c 'from google._upb import _message; assert "google._upb._message.MessageMeta" in str(_message.MessageMeta)' - if: ${{ !matrix.pure_python }} - - name: Install Protobuf Test Wheel - run: pip install -vvv --no-index --find-links wheels protobuftests - - name: Run the unit tests - run: | - TESTS=$(pip show -f protobuftests | grep pb_unit_tests.*py$ | sed 's,/,.,g' | sed 's,\\,.,g' | sed -E 's,.py$,,g') - for test in $TESTS; do - python -m unittest -v $test - done - - test_pure_python_wheels: - name: Test Pure Python Wheels - needs: build_wheels - strategy: - fail-fast: false # Don't cancel all jobs if one fails. - matrix: - python-version: ["3.7", "3.10"] - runs-on: ubuntu-large - - steps: - - name: Download Wheels - uses: actions/download-artifact@v3 - with: - name: python-wheels - path: wheels - - name: Delete Binary Wheels - run: find wheels -type f | grep -v none-any | xargs rm - - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Setup Python venv - run: | - python -m pip install --upgrade pip - python -m venv env - source env/bin/activate - echo "VIRTUAL ENV:" $VIRTUAL_ENV - - name: Install numpy - run: pip install numpy - - name: Install Protobuf Wheels - run: pip install -vvv --no-index --find-links wheels protobuf protobuftests - - name: Run the unit tests - run: | - TESTS=$(pip show -f protobuftests | grep _test.py | sed 's,/,.,g' | sed -E 's,.py$,,g') - for test in $TESTS; do - python -m unittest -v $test - done diff --git a/third_party/upb/.gitignore b/third_party/upb/.gitignore deleted file mode 100644 index 79d6c53f831..00000000000 --- a/third_party/upb/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -*.sw? -obj/ -lib/ -bazel-* -_build -.vscode diff --git a/third_party/upb/upb/reflection/def_pool.h b/third_party/upb/upb/reflection/def_pool.h index f21a0dbd2db..513e29de821 100644 --- a/third_party/upb/upb/reflection/def_pool.h +++ b/third_party/upb/upb/reflection/def_pool.h @@ -48,7 +48,7 @@ const upb_FileDef* upb_DefPool_FindFileByNameWithSize(const upb_DefPool* s, const upb_FieldDef* upb_DefPool_FindExtensionByMiniTable( const upb_DefPool* s, const upb_MiniTableExtension* ext); -const upb_FieldDef* upb_DefPool_FindExtensionByName(const upb_DefPool* s, +UPB_API const upb_FieldDef* upb_DefPool_FindExtensionByName(const upb_DefPool* s, const char* sym); const upb_FieldDef* upb_DefPool_FindExtensionByNameWithSize( @@ -71,7 +71,7 @@ UPB_API const upb_FileDef* upb_DefPool_AddFile( upb_DefPool* s, const UPB_DESC(FileDescriptorProto) * file_proto, upb_Status* status); -const upb_ExtensionRegistry* upb_DefPool_ExtensionRegistry( +UPB_API const upb_ExtensionRegistry* upb_DefPool_ExtensionRegistry( const upb_DefPool* s); const upb_FieldDef** upb_DefPool_GetAllExtensions(const upb_DefPool* s, diff --git a/tools/distrib/python/grpc_version.py b/tools/distrib/python/grpc_version.py index 7c78587c40e..b19fce51a57 100644 --- a/tools/distrib/python/grpc_version.py +++ b/tools/distrib/python/grpc_version.py @@ -15,4 +15,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! VERSION = '1.61.0.dev0' -PROTOBUF_VERSION = '3.25.0' +PROTOBUF_VERSION = '3.25.1' diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index f7f3637710e..ba4c8519643 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -15,4 +15,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! VERSION = '1.61.0.dev0' -PROTOBUF_VERSION = '3.25.0' +PROTOBUF_VERSION = '3.25.1' diff --git a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py index edb4c940321..b13282749e8 100644 --- a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py +++ b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py @@ -320,4 +320,4 @@ CC_INCLUDES=[ ] PROTO_INCLUDE='third_party/protobuf/src' -PROTOBUF_SUBMODULE_VERSION="6b5d8db01fe47478e8d400f550e797e6230d464e" +PROTOBUF_SUBMODULE_VERSION="7f94235e552599141950d7a4a3eaf93bc87d1b22" diff --git a/tools/distrib/python/xds_protos/grpc_version.py b/tools/distrib/python/xds_protos/grpc_version.py index 008611094fc..aa2e7c91051 100644 --- a/tools/distrib/python/xds_protos/grpc_version.py +++ b/tools/distrib/python/xds_protos/grpc_version.py @@ -15,4 +15,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! VERSION = '1.61.0.dev0' -PROTOBUF_VERSION = '3.25.0' +PROTOBUF_VERSION = '3.25.1' diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index ffe5b6a9bd8..ff2b3b85fbe 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -35,7 +35,7 @@ third_party/googleapis 2f9af297c84c55c8b871ba4495e01ade42476c92 third_party/googletest 2dd1c131950043a8ad5ab0d2dda0e0970596586a third_party/opencensus-proto 4aa53e15cbf1a47bc9087e6cfdca214c1eea4e89 third_party/opentelemetry 60fa8754d890b5c55949a8c68dcfd7ab5c2395df -third_party/protobuf 6b5d8db01fe47478e8d400f550e797e6230d464e +third_party/protobuf 7f94235e552599141950d7a4a3eaf93bc87d1b22 third_party/protoc-gen-validate fab737efbb4b4d03e7c771393708f75594b121e4 third_party/re2 0c5616df9c0aaa44c9440d87422012423d91c7d1 third_party/xds e9ce68804cb4e64cab5a52e3c8baf840d4ff87b7