From 3b5b20682bf64e2f7275adec6983d6e390e7caf2 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Thu, 2 Jun 2016 20:27:20 -0700 Subject: [PATCH] Make running individual Python tests less painful Before this change, running Python tests individually required building a tox environment via the run_tests script and then specifying long environment variables to filter out just the test we wanted to run (and then we wouldn't be able to get the output on interrupt, nor would we have an easy way of determining the PID of the process for debugger attachment). Now invoking the build_python.sh script creates a workable python virtual environment that includes all necessary libraries and tests (s.t. running a single test is now possible by just knowing the module name). This does not change existing supported means of running tests (e.g. through run_tests.py). An additional way of running individual tests has been introduced. Following invocation of `./tools/run_tests/build_python.sh` (or run_tests.py), one may invoke ./$VENV/bin/python -m $TEST_MODULE_NAME and acquire a single running process that *is* the test process (rather than a parent of the process). $VENV is the virtual environment name specified to `build_python.sh` (defaults to `py27`) and $TEST_MODULE_NAME is what it says on the tin. --- PYTHON-MANIFEST.in | 1 - tools/run_tests/build_python.sh | 71 +++++++++++++++---- .../performance/run_worker_python.sh | 2 +- tools/run_tests/run_interop_tests.py | 14 ++-- tools/run_tests/run_python.sh | 17 +---- tools/run_tests/run_tests.py | 60 +++++++++++----- 6 files changed, 114 insertions(+), 51 deletions(-) diff --git a/PYTHON-MANIFEST.in b/PYTHON-MANIFEST.in index 3ebba6ec3fc..635e77b875c 100644 --- a/PYTHON-MANIFEST.in +++ b/PYTHON-MANIFEST.in @@ -1,6 +1,5 @@ recursive-include src/python/grpcio/grpc *.c *.h *.py *.pyx *.pxd *.pxi *.python *.pem recursive-exclude src/python/grpcio/grpc/_cython *.so *.pyd -graft src/python/grpcio/tests graft src/python/grpcio/grpcio.egg-info graft src/core graft src/boringssl diff --git a/tools/run_tests/build_python.sh b/tools/run_tests/build_python.sh index 3b1c7d2d40e..9d2813fa192 100755 --- a/tools/run_tests/build_python.sh +++ b/tools/run_tests/build_python.sh @@ -33,25 +33,70 @@ set -ex # change to grpc repo root cd $(dirname $0)/../.. -TOX_PYTHON_ENV="$1" -PY_VERSION="${TOX_PYTHON_ENV: -2}" +PYTHON=${1:-python2.7} +VENV=${2:-py27} +VENV_RELATIVE_PYTHON=${3:-bin/python} +TOOLCHAIN=${4:-unix} ROOT=`pwd` export CFLAGS="-I$ROOT/include -std=gnu99 -fno-wrapv" export GRPC_PYTHON_BUILD_WITH_CYTHON=1 -if [ "$CONFIG" = "gcov" ] -then +# If ccache is available, use it... unless we're on Mac, then all hell breaks +# loose because Python does hacky things to support other hacky things done to +# hacky things on Mac OS X +PLATFORM=`uname -s` +if [ "${PLATFORM/Darwin}" = "$PLATFORM" ]; then + # We're not on Darwin (Mac OS X) + if [ -x "$(command -v ccache)" ]; then + if [ -x "$(command -v gcc)" ]; then + export CC='ccache gcc' + elif [ -x "$(command -v clang)" ]; then + export CC='ccache clang' + fi + fi +fi + +# Find `realpath` +if [ -x "$(command -v realpath)" ]; then + export REALPATH=realpath +elif [ -x "$(command -v grealpath)" ]; then + export REALPATH=grealpath +else + echo 'Couldn'"'"'t find `realpath` or `grealpath`' + exit 1 +fi + +if [ "$CONFIG" = "gcov" ]; then export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1 fi -tox -e ${TOX_PYTHON_ENV} --notest +($PYTHON -m virtualenv $VENV || true) +VENV_PYTHON=`$REALPATH -s "$VENV/$VENV_RELATIVE_PYTHON"` + +# pip-installs the directory specified. Used because on MSYS the vanilla Windows +# Python gets confused when parsing paths. +pip_install_dir() { + PWD=`pwd` + cd $1 + ($VENV_PYTHON setup.py build_ext -c $TOOLCHAIN || true) + # install the dependencies + $VENV_PYTHON -m pip install --upgrade . + # ensure that we've reinstalled the test packages + $VENV_PYTHON -m pip install --upgrade --force-reinstall --no-deps . + cd $PWD +} -$ROOT/.tox/${TOX_PYTHON_ENV}/bin/pip install cython -$ROOT/.tox/${TOX_PYTHON_ENV}/bin/pip install $ROOT -$ROOT/.tox/${TOX_PYTHON_ENV}/bin/python $ROOT/tools/distrib/python/make_grpcio_tools.py -$ROOT/.tox/${TOX_PYTHON_ENV}/bin/pip install $ROOT/tools/distrib/python/grpcio_tools -$ROOT/.tox/${TOX_PYTHON_ENV}/bin/python $ROOT/src/python/grpcio_health_checking/setup.py preprocess -$ROOT/.tox/${TOX_PYTHON_ENV}/bin/pip install $ROOT/src/python/grpcio_health_checking -$ROOT/.tox/${TOX_PYTHON_ENV}/bin/python $ROOT/src/python/grpcio_tests/setup.py preprocess -$ROOT/.tox/${TOX_PYTHON_ENV}/bin/python $ROOT/src/python/grpcio_tests/setup.py build_proto_modules +$VENV_PYTHON -m pip install --upgrade pip setuptools +$VENV_PYTHON -m pip install cython +pip_install_dir $ROOT +$VENV_PYTHON $ROOT/tools/distrib/python/make_grpcio_tools.py +pip_install_dir $ROOT/tools/distrib/python/grpcio_tools +# TODO(atash) figure out namespace packages and grpcio-tools and auditwheel +# etc... +pip_install_dir $ROOT +$VENV_PYTHON $ROOT/src/python/grpcio_health_checking/setup.py preprocess +pip_install_dir $ROOT/src/python/grpcio_health_checking +$VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py preprocess +$VENV_PYTHON $ROOT/src/python/grpcio_tests/setup.py build_proto_modules +pip_install_dir $ROOT/src/python/grpcio_tests diff --git a/tools/run_tests/performance/run_worker_python.sh b/tools/run_tests/performance/run_worker_python.sh index 2bca5e16952..3b8ba6f4e4a 100755 --- a/tools/run_tests/performance/run_worker_python.sh +++ b/tools/run_tests/performance/run_worker_python.sh @@ -32,4 +32,4 @@ set -ex cd $(dirname $0)/../../.. -PYTHONPATH=src/python/grpcio_tests:src/python/grpcio:src/python/gens .tox/py27/bin/python src/python/grpcio_tests/tests/qps/qps_worker.py $@ +PYTHONPATH=src/python/grpcio_tests:src/python/grpcio:src/python/gens py27/bin/python src/python/grpcio_tests/tests/qps/qps_worker.py $@ diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index adf9adaf136..13a4a49325a 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -304,8 +304,11 @@ class PythonLanguage: def client_cmd(self, args): return [ - 'tox -einterop_client --', - ' '.join(args) + 'py27/bin/python', + 'src/python/grpcio_tests/setup.py', + 'run_interop', + '--client', + '--args="{}"'.format(' '.join(args)) ] def cloud_to_prod_env(self): @@ -313,8 +316,11 @@ class PythonLanguage: def server_cmd(self, args): return [ - 'tox -einterop_server --', - ' '.join(args) + ' --use_tls=true' + 'py27/bin/python', + 'src/python/grpcio_tests/setup.py', + 'run_interop', + '--server', + '--args="{}"'.format(' '.join(args) + ' --use_tls=true') ] def global_env(self): diff --git a/tools/run_tests/run_python.sh b/tools/run_tests/run_python.sh index b05efd1c76a..17e0186f2a8 100755 --- a/tools/run_tests/run_python.sh +++ b/tools/run_tests/run_python.sh @@ -33,24 +33,11 @@ set -ex # change to grpc repo root cd $(dirname $0)/../.. -TOX_PYTHON_ENV="$1" +PYTHON=`realpath -s "${1:-py27/bin/python}"` ROOT=`pwd` -export LD_LIBRARY_PATH=$ROOT/libs/$CONFIG -export DYLD_LIBRARY_PATH=$ROOT/libs/$CONFIG -export PATH=$ROOT/bins/$CONFIG:$ROOT/bins/$CONFIG/protobuf:$PATH -export CFLAGS="-I$ROOT/include -std=c89" -export LDFLAGS="-L$ROOT/libs/$CONFIG" -export GRPC_PYTHON_BUILD_WITH_CYTHON=1 -export GRPC_PYTHON_USE_PRECOMPILED_BINARIES=0 -if [ "$CONFIG" = "gcov" ] -then - export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1 - tox -e ${TOX_PYTHON_ENV} -else - $ROOT/.tox/${TOX_PYTHON_ENV}/bin/python $ROOT/src/python/grpcio_tests/setup.py test_lite -fi +$PYTHON $ROOT/src/python/grpcio_tests/setup.py test_lite mkdir -p $ROOT/reports rm -rf $ROOT/reports/python-coverage diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index f32a621ee46..fbc5729c758 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -32,11 +32,13 @@ import argparse import ast +import collections import glob import itertools import json import multiprocessing import os +import os.path import platform import random import re @@ -372,12 +374,20 @@ class PhpLanguage(object): return 'php' +class PythonConfig(collections.namedtuple('PythonConfig', [ + 'python', 'venv', 'venv_relative_python', 'toolchain',])): + + @property + def venv_python(self): + return os.path.abspath('{}/{}'.format(self.venv, self.venv_relative_python)) + + class PythonLanguage(object): def configure(self, config, args): self.config = config self.args = args - self._tox_envs = self._get_tox_envs(self.args.compiler) + self.pythons = self._get_pythons(self.args.compiler) def test_specs(self): # load list of known test suites @@ -386,33 +396,42 @@ class PythonLanguage(object): environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS) if self.config.build_config != 'gcov': return [self.config.job_spec( - ['tools/run_tests/run_python.sh', tox_env], + ['tools/run_tests/run_python.sh', config.venv_python], + None, environ=dict(environment.items() + [('GRPC_PYTHON_TESTRUNNER_FILTER', suite_name)]), - shortname='%s.test.%s' % (tox_env, suite_name), + shortname='%s.test.%s' % (config.venv, suite_name), timeout_seconds=5*60) for suite_name in tests_json - for tox_env in self._tox_envs] + for config in self.pythons] else: - return [self.config.job_spec(['tools/run_tests/run_python.sh', tox_env], - environ=environment, - shortname='%s.test.coverage' % tox_env, - timeout_seconds=15*60) - for tox_env in self._tox_envs] + return [self.config.job_spec( + ['tools/run_tests/run_python.sh', config.venv_python], + None, + environ=environment, + shortname='%s.test.coverage' % config.venv, + timeout_seconds=15*60) + for config in self.pythons] def pre_build_steps(self): return [] def make_targets(self): - return ['static_c', 'grpc_python_plugin', 'shared_c'] + return [] def make_options(self): return [] def build_steps(self): - return [['tools/run_tests/build_python.sh', tox_env] - for tox_env in self._tox_envs] + return [ + [ + 'tools/run_tests/build_python.sh', + config.python, config.venv, + config.venv_relative_python, config.toolchain + ] + for config in self.pythons + ] def post_tests_steps(self): return [] @@ -423,14 +442,21 @@ class PythonLanguage(object): def dockerfile_dir(self): return 'tools/dockerfile/test/python_jessie_%s' % _docker_arch_suffix(self.args.arch) - def _get_tox_envs(self, compiler): - """Returns name of tox environment based on selected compiler.""" + def _get_pythons(self, compiler): + if os.name == 'nt': + venv_relative_python = 'Scripts/python.exe' + toolchain = 'mingw32' + else: + venv_relative_python = 'bin/python' + toolchain = 'unix' + python27_config = PythonConfig('python2.7', 'py27', venv_relative_python, toolchain) + python34_config = PythonConfig('python3.4', 'py34', venv_relative_python, toolchain) if compiler == 'default': - return ('py27', 'py34') + return (python27_config, python34_config,) elif compiler == 'python2.7': - return ('py27',) + return (python27_config,) elif compiler == 'python3.4': - return ('py34',) + return (python34_config,) else: raise Exception('Compiler %s not supported.' % compiler)