[Deps] Removed vendored python deps from Bazel builds (#38692)

With the Bazel build transitioning to BzlMod, all vendored Bazel Python targets must be removed.  These targets are not available in the BCR and need be managed via a pip `requirements.bazel.txt` file. gRPC already uses this approach, so it is going to be extended to include those target. (e.g. see [test utils](e06ad82c3f/test/cpp/naming/utils/BUILD (L27-L45)) to understand how those targets are used)

Additionally, the generation of the `requirements.bazel.lock` file has been improved.  Because this file is a lock file, including all transitive dependencies, manual maintenance is not managable.  Its new source file, `requirements.bazel.txt` is created to list only the direct dependencies used by gRPC, along with instructions for generating the full lock file.  This source file omits specific version requirements to use the latest available versions, but version constraints can be added as needed.

Closes #38692

PiperOrigin-RevId: 724427578
pull/38704/head
Esun Kim 2 weeks ago committed by Copybara-Service
parent 1f97f75401
commit 68b5dc5125
  1. 1
      .github/labeler.yml
  2. 2
      MODULE.bazel
  3. 2
      WORKSPACE
  4. 60
      bazel/grpc_deps.bzl
  5. 116
      requirements.bazel.lock
  6. 82
      requirements.bazel.txt
  7. 2
      templates/MODULE.bazel.template
  8. 12
      test/cpp/naming/utils/BUILD
  9. 12
      third_party/BUILD
  10. 7
      third_party/constantly.BUILD
  11. 10
      third_party/incremental.BUILD
  12. 15
      third_party/twisted.BUILD
  13. 10
      third_party/yaml.BUILD
  14. 13
      third_party/zope_interface.BUILD
  15. 2
      tools/distrib/python/docgen.py
  16. 17
      tools/run_tests/sanity/check_bazel_workspace.py

@ -46,6 +46,7 @@ lang/python:
- any-glob-to-any-file:
- bazel/python_rules.bzl
- examples/python/**
- requirements.bazel.lock
- requirements.bazel.txt
- src/compiler/python*
- all-globs-to-any-file:

@ -85,7 +85,7 @@ pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
pip.parse(
hub_name = "grpc_python_dependencies",
python_version = python_version,
requirements_lock = "//:requirements.bazel.txt",
requirements_lock = "//:requirements.bazel.lock",
)
for python_version in PYTHON_VERSIONS
]

@ -46,7 +46,7 @@ load("@rules_python//python:pip.bzl", "pip_parse")
pip_parse(
name = "grpc_python_dependencies",
requirements_lock = "@com_github_grpc_grpc//:requirements.bazel.txt",
requirements_lock = "@com_github_grpc_grpc//:requirements.bazel.lock",
)
load("@grpc_python_dependencies//:requirements.bzl", "install_deps")

@ -368,66 +368,6 @@ def grpc_test_only_deps():
Loads dependencies that are only needed to run grpc library's tests.
"""
if "com_github_twisted_twisted" not in native.existing_rules():
http_archive(
name = "com_github_twisted_twisted",
sha256 = "ca17699d0d62eafc5c28daf2c7d0a18e62ae77b4137300b6c7d7868b39b06139",
strip_prefix = "twisted-twisted-17.5.0",
urls = [
"https://storage.googleapis.com/grpc-bazel-mirror/github.com/twisted/twisted/archive/twisted-17.5.0.zip",
"https://github.com/twisted/twisted/archive/twisted-17.5.0.zip",
],
build_file = "@com_github_grpc_grpc//third_party:twisted.BUILD",
)
if "com_github_yaml_pyyaml" not in native.existing_rules():
http_archive(
name = "com_github_yaml_pyyaml",
sha256 = "e34d97db6d846f5e2ad51417fd646e7ce6a3a70726ccea2a857e0580a7155f39",
strip_prefix = "pyyaml-6.0.1",
urls = [
"https://storage.googleapis.com/grpc-bazel-mirror/github.com/yaml/pyyaml/archive/6.0.1.zip",
"https://github.com/yaml/pyyaml/archive/6.0.1.zip",
],
build_file = "@com_github_grpc_grpc//third_party:yaml.BUILD",
)
if "com_github_twisted_incremental" not in native.existing_rules():
http_archive(
name = "com_github_twisted_incremental",
sha256 = "f0ca93359ee70243ff7fbf2d904a6291810bd88cb80ed4aca6fa77f318a41a36",
strip_prefix = "incremental-incremental-17.5.0",
urls = [
"https://storage.googleapis.com/grpc-bazel-mirror/github.com/twisted/incremental/archive/incremental-17.5.0.zip",
"https://github.com/twisted/incremental/archive/incremental-17.5.0.zip",
],
build_file = "@com_github_grpc_grpc//third_party:incremental.BUILD",
)
if "com_github_zopefoundation_zope_interface" not in native.existing_rules():
http_archive(
name = "com_github_zopefoundation_zope_interface",
sha256 = "e9579fc6149294339897be3aa9ecd8a29217c0b013fe6f44fcdae00e3204198a",
strip_prefix = "zope.interface-4.4.3",
urls = [
"https://storage.googleapis.com/grpc-bazel-mirror/github.com/zopefoundation/zope.interface/archive/4.4.3.zip",
"https://github.com/zopefoundation/zope.interface/archive/4.4.3.zip",
],
build_file = "@com_github_grpc_grpc//third_party:zope_interface.BUILD",
)
if "com_github_twisted_constantly" not in native.existing_rules():
http_archive(
name = "com_github_twisted_constantly",
sha256 = "2702cd322161a579d2c0dbf94af4e57712eedc7bd7bbbdc554a230544f7d346c",
strip_prefix = "constantly-15.1.0",
urls = [
"https://storage.googleapis.com/grpc-bazel-mirror/github.com/twisted/constantly/archive/15.1.0.zip",
"https://github.com/twisted/constantly/archive/15.1.0.zip",
],
build_file = "@com_github_grpc_grpc//third_party:constantly.BUILD",
)
if "com_google_libprotobuf_mutator" not in native.existing_rules():
http_archive(
name = "com_google_libprotobuf_mutator",

@ -0,0 +1,116 @@
#
# This file is autogenerated by pip-compile with Python 3.8
# by the following command:
#
# pip-compile --allow-unsafe requirements.bazel.in
#
absl-py==2.1.0
# via -r requirements.bazel.in
attrs==25.1.0
# via twisted
automat==24.8.1
# via twisted
cachetools==5.5.1
# via google-auth
certifi==2025.1.31
# via
# -r requirements.bazel.in
# requests
chardet==5.2.0
# via -r requirements.bazel.in
charset-normalizer==3.4.1
# via requests
constantly==23.10.4
# via twisted
deprecated==1.2.18
# via
# opentelemetry-api
# opentelemetry-semantic-conventions
gevent==24.2.1
# via -r requirements.bazel.in
google-auth==2.38.0
# via -r requirements.bazel.in
googleapis-common-protos==1.66.0
# via -r requirements.bazel.in
greenlet==3.1.1
# via gevent
hyperlink==21.0.0
# via twisted
idna==3.10
# via
# -r requirements.bazel.in
# hyperlink
# requests
importlib-metadata==8.5.0
# via opentelemetry-api
incremental==24.7.2
# via twisted
opentelemetry-api==1.30.0
# via
# -r requirements.bazel.in
# opentelemetry-exporter-prometheus
# opentelemetry-resourcedetector-gcp
# opentelemetry-sdk
# opentelemetry-semantic-conventions
opentelemetry-exporter-prometheus==0.51b0
# via -r requirements.bazel.in
opentelemetry-resourcedetector-gcp==1.9.0a0
# via -r requirements.bazel.in
opentelemetry-sdk==1.30.0
# via
# -r requirements.bazel.in
# opentelemetry-exporter-prometheus
# opentelemetry-resourcedetector-gcp
opentelemetry-semantic-conventions==0.51b0
# via opentelemetry-sdk
prometheus-client==0.21.1
# via opentelemetry-exporter-prometheus
protobuf==5.29.3
# via
# -r requirements.bazel.in
# googleapis-common-protos
pyasn1==0.6.1
# via
# pyasn1-modules
# rsa
pyasn1-modules==0.4.1
# via google-auth
pyyaml==6.0.2
# via -r requirements.bazel.in
requests==2.32.3
# via
# -r requirements.bazel.in
# opentelemetry-resourcedetector-gcp
rsa==4.9
# via google-auth
tomli==2.2.1
# via incremental
twisted==24.11.0
# via -r requirements.bazel.in
typing-extensions==4.12.2
# via
# automat
# opentelemetry-resourcedetector-gcp
# opentelemetry-sdk
# twisted
urllib3==2.2.3
# via
# -r requirements.bazel.in
# requests
wrapt==1.17.2
# via deprecated
zipp==3.20.2
# via importlib-metadata
zope-event==5.0
# via gevent
zope-interface==7.2
# via
# gevent
# twisted
# The following packages are considered to be unsafe in a requirements file:
setuptools==75.3.0
# via
# incremental
# zope-event
# zope-interface

@ -1,41 +1,41 @@
# GRPC Python setup requirements
# The requirements listed below are solely for Bazel tests and should not be used elsewhere.
absl-py==1.4.0
cachetools==5.3.2
certifi==2023.7.22
chardet==3.0.4
charset-normalizer==3.3.2
coverage==4.5.4
cython==3.0.0
Deprecated==1.2.14
gevent==22.08.0
google-api-core==2.24.0
google-auth==2.23.4
google-cloud-monitoring==2.26.0
google-cloud-trace==1.15.0
googleapis-common-protos==1.63.1
greenlet==1.1.3.post0
idna==2.7
importlib-metadata==6.11.0
oauth2client==4.1.0
opencensus-context==0.1.3
opentelemetry-api==1.25.0
opentelemetry-exporter-prometheus==0.46b0
opentelemetry-resourcedetector-gcp==1.6.0a0
opentelemetry-sdk==1.25.0
opentelemetry-semantic-conventions==0.46b0
prometheus_client==0.20.0
proto-plus==1.25.0
protobuf>=5.27.1,<6.0dev
pyasn1-modules==0.3.0
pyasn1==0.5.0
requests==2.25.1
rsa==4.9
setuptools==44.1.1
typing-extensions==4.9.0
urllib3==1.26.18
wheel==0.38.1
wrapt==1.16.0
zipp==3.17.0
zope.event==4.5.0
zope.interface==6.1
# Copyright 2025 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This file provides the source for generating requirements.bazel.lock.
# Instructions for generation follow:
# Use the oldest supported version of Python
#
# $ docker run -it --rm -v $(pwd):/grpc python:3.8 /bin/bash
# # cd grpc
# # pip install pip-tools
# # pip-compile --allow-unsafe requirements.bazel.txt > requirements.bazel.lock
# # exit
# $ sudo chown $USER requirements.bazel.lock
absl-py
certifi
chardet
gevent
google-auth
googleapis-common-protos
idna
opentelemetry-api
opentelemetry-exporter-prometheus
opentelemetry-resourcedetector-gcp
opentelemetry-sdk
protobuf
pyyaml # for DNS test
requests
twisted # for DNS test
urllib3

@ -87,7 +87,7 @@
pip.parse(
hub_name = "grpc_python_dependencies",
python_version = python_version,
requirements_lock = "//:requirements.bazel.txt",
requirements_lock = "//:requirements.bazel.lock",
)
for python_version in PYTHON_VERSIONS
]

@ -28,21 +28,19 @@ grpc_py_binary(
name = "dns_server",
testonly = True,
srcs = ["dns_server.py"],
external_deps = [
"twisted",
"yaml",
],
python_version = "PY3",
deps = [
requirement("twisted"),
requirement("pyyaml"),
],
)
grpc_py_binary(
name = "dns_resolver",
testonly = True,
srcs = ["dns_resolver.py"],
external_deps = [
"twisted",
],
python_version = "PY3",
deps = [requirement("twisted")],
)
grpc_py_binary(

12
third_party/BUILD vendored

@ -208,15 +208,3 @@ alias(
actual = "@com_google_googleapis//google/logging/v2:logging_cc_proto",
tags = ["manual"],
)
alias(
name = "twisted",
actual = "@com_github_twisted_twisted//:twisted",
tags = ["manual"],
)
alias(
name = "yaml",
actual = "@com_github_yaml_pyyaml//:yaml",
tags = ["manual"],
)

@ -1,7 +0,0 @@
py_library(
name = "constantly",
srcs = glob(["constantly/*.py"]),
visibility = [
"//visibility:public",
],
)

@ -1,10 +0,0 @@
py_library(
name = "incremental",
srcs = glob(["src/incremental/*.py"]),
imports = [
"src",
],
visibility = [
"//visibility:public",
],
)

@ -1,15 +0,0 @@
py_library(
name = "twisted",
srcs = glob(["src/twisted/**/*.py"]),
imports = [
"src",
],
visibility = [
"//visibility:public",
],
deps = [
"@com_github_twisted_incremental//:incremental",
"@com_github_twisted_constantly//:constantly",
"@com_github_zopefoundation_zope_interface//:zope_interface",
],
)

@ -1,10 +0,0 @@
py_library(
name = "yaml",
srcs = glob(["lib/yaml/*.py"]),
imports = [
"lib",
],
visibility = [
"//visibility:public",
],
)

@ -1,13 +0,0 @@
py_library(
name = "zope_interface",
srcs = glob([
"src/zope/interface/*.py",
"src/zope/interface/common/*.py",
]),
imports = [
"src",
],
visibility = [
"//visibility:public",
],
)

@ -38,7 +38,7 @@ SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
PROJECT_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, "..", "..", ".."))
SETUP_PATH = os.path.join(PROJECT_ROOT, "setup.py")
REQUIREMENTS_PATH = os.path.join(PROJECT_ROOT, "requirements.bazel.txt")
REQUIREMENTS_PATH = os.path.join(PROJECT_ROOT, "requirements.bazel.lock")
DOC_PATH = os.path.join(PROJECT_ROOT, "doc/build")
if "VIRTUAL_ENV" in os.environ:

@ -38,13 +38,6 @@ git_submodule_hashes = {
_BAZEL_SKYLIB_DEP_NAME = "bazel_skylib"
_BAZEL_TOOLCHAINS_DEP_NAME = "bazel_toolchains"
_BAZEL_COMPDB_DEP_NAME = "bazel_compdb"
_TWISTED_TWISTED_DEP_NAME = "com_github_twisted_twisted"
_YAML_PYYAML_DEP_NAME = "com_github_yaml_pyyaml"
_TWISTED_INCREMENTAL_DEP_NAME = "com_github_twisted_incremental"
_ZOPEFOUNDATION_ZOPE_INTERFACE_DEP_NAME = (
"com_github_zopefoundation_zope_interface"
)
_TWISTED_CONSTANTLY_DEP_NAME = "com_github_twisted_constantly"
_GRPC_DEP_NAMES = [
"platforms",
@ -63,11 +56,6 @@ _GRPC_DEP_NAMES = [
_BAZEL_SKYLIB_DEP_NAME,
_BAZEL_TOOLCHAINS_DEP_NAME,
_BAZEL_COMPDB_DEP_NAME,
_TWISTED_TWISTED_DEP_NAME,
_YAML_PYYAML_DEP_NAME,
_TWISTED_INCREMENTAL_DEP_NAME,
_ZOPEFOUNDATION_ZOPE_INTERFACE_DEP_NAME,
_TWISTED_CONSTANTLY_DEP_NAME,
"bazel_features",
"rules_proto",
"io_bazel_rules_go",
@ -94,11 +82,6 @@ _GRPC_BAZEL_ONLY_DEPS = [
_BAZEL_SKYLIB_DEP_NAME,
_BAZEL_TOOLCHAINS_DEP_NAME,
_BAZEL_COMPDB_DEP_NAME,
_TWISTED_TWISTED_DEP_NAME,
_YAML_PYYAML_DEP_NAME,
_TWISTED_INCREMENTAL_DEP_NAME,
_ZOPEFOUNDATION_ZOPE_INTERFACE_DEP_NAME,
_TWISTED_CONSTANTLY_DEP_NAME,
"bazel_features",
"rules_proto",
"io_bazel_rules_go",

Loading…
Cancel
Save