diff --git a/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh b/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh index fef395557ff..ad249215de5 100644 --- a/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh +++ b/tools/internal_ci/linux/grpc_xds_k8s_install_test_driver.sh @@ -216,7 +216,7 @@ test_driver_get_source() { } ####################################### -# Install Python modules from required in $TEST_DRIVER_FULL_DIR/requirements.txt +# Install Python modules from required in $TEST_DRIVER_FULL_DIR/requirements.lock # to Python virtual environment. Creates and activates Python venv if necessary. # Globals: # TEST_DRIVER_FULL_DIR @@ -245,7 +245,7 @@ test_driver_pip_install() { source "${venv_dir}/bin/activate" fi - python3 -m pip install -r requirements.txt + python3 -m pip install -r requirements.lock echo "Installed Python packages:" python3 -m pip list } diff --git a/tools/run_tests/xds_k8s_test_driver/README.md b/tools/run_tests/xds_k8s_test_driver/README.md index 3f5680184ba..492e3926eef 100644 --- a/tools/run_tests/xds_k8s_test_driver/README.md +++ b/tools/run_tests/xds_k8s_test_driver/README.md @@ -180,7 +180,7 @@ python3.7 -m venv venv . ./venv/bin/activate # Install requirements -pip install -r requirements.txt +pip install -r requirements.lock # Generate protos python -m grpc_tools.protoc --proto_path=../../../ \ @@ -207,6 +207,22 @@ from your dev environment. You need: 1. Install additional dev packages: `pip install -r requirements-dev.txt` 2. Use `./bin/yapf.sh` and `./bin/isort.sh` helpers to auto-format code. +### Updating Python Dependencies + +We track our Python-level dependencies using three different files: + +- `requirements.txt` +- `dev-requirements.txt` +- `requirements.lock` + +`requirements.txt` lists modules without specific versions supplied, though +versions ranges may be specified. `requirements.lock` is generated from +`requirements.txt` and _does_ specify versions for every dependency in the +transitive dependency tree. + +When updating `requirements.txt`, you must also update `requirements.lock`. To +do this, navigate to this directory and run `./bin/freeze.sh`. + ### Setup test configuration There are many arguments to be passed into the test run. You can save the diff --git a/tools/run_tests/xds_k8s_test_driver/bin/freeze.sh b/tools/run_tests/xds_k8s_test_driver/bin/freeze.sh new file mode 100755 index 00000000000..2674ab58deb --- /dev/null +++ b/tools/run_tests/xds_k8s_test_driver/bin/freeze.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Copyright 2023 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -exo pipefail + +VENV_NAME="venv-$(mktemp -d)" +readonly VENV_NAME + +python3 -m virtualenv "${VENV_NAME}" + +"${VENV_NAME}"/bin/pip install -r requirements.txt +"${VENV_NAME}"/bin/pip freeze > requirements.lock + +rm -rf "${VENV_NAME}" diff --git a/tools/run_tests/xds_k8s_test_driver/requirements-dev.txt b/tools/run_tests/xds_k8s_test_driver/requirements-dev.txt index 27dc27bd3dd..bf47c5a06a7 100644 --- a/tools/run_tests/xds_k8s_test_driver/requirements-dev.txt +++ b/tools/run_tests/xds_k8s_test_driver/requirements-dev.txt @@ -1,4 +1,4 @@ --r requirements.txt +-r requirements.lock yapf==0.30.0 # Mirrors yapf version set in https://github.com/grpc/grpc/blob/master/tools/distrib/yapf_code.sh isort~=5.9 # TODO(https://github.com/grpc/grpc/pull/25872): mypy diff --git a/tools/run_tests/xds_k8s_test_driver/requirements.lock b/tools/run_tests/xds_k8s_test_driver/requirements.lock new file mode 100644 index 00000000000..733cb8688c0 --- /dev/null +++ b/tools/run_tests/xds_k8s_test_driver/requirements.lock @@ -0,0 +1,40 @@ +absl-py==0.15.0 +cachetools==5.2.1 +certifi==2022.12.7 +charset-normalizer==3.0.1 +google-api-core==2.11.0 +google-api-python-client==1.12.11 +google-auth==2.16.0 +google-auth-httplib2==0.1.0 +google-cloud-secret-manager==2.15.0 +googleapis-common-protos==1.58.0 +grpc-google-iam-v1==0.12.6 +grpcio==1.51.1 +grpcio-channelz==1.48.2 +grpcio-health-checking==1.48.2 +grpcio-status==1.48.2 +grpcio-tools==1.48.2 +httplib2==0.21.0 +idna==3.4 +kubernetes==12.0.1 +Mako==1.2.4 +MarkupSafe==2.1.2 +oauthlib==3.2.2 +packaging==21.3 +proto-plus==1.22.2 +protobuf==3.20.3 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +Pygments==2.14.0 +pyparsing==3.0.9 +python-dateutil==2.8.2 +PyYAML==5.4.1 +requests==2.28.2 +requests-oauthlib==1.3.1 +rsa==4.9 +six==1.16.0 +tenacity==6.3.1 +uritemplate==3.0.1 +urllib3==1.26.14 +websocket-client==1.4.2 +xds-protos==0.0.11