From e9a253477d06041b6cd137dd0b40c4e2bc658563 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 9 Jan 2015 13:34:53 -0800 Subject: [PATCH 1/6] Fix win32 build error. --- include/grpc/support/port_platform.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/grpc/support/port_platform.h b/include/grpc/support/port_platform.h index 27a7b5529f7..978b3377243 100644 --- a/include/grpc/support/port_platform.h +++ b/include/grpc/support/port_platform.h @@ -126,7 +126,7 @@ #error Must define exactly one of GPR_ARCH_32, GPR_ARCH_64 #endif -#if defined(GPR_CPU_LINUX) + defined(GPR_CPU_POSIX) != 1 +#if defined(GPR_CPU_LINUX) + defined(GPR_CPU_POSIX) + defined(GPR_WIN32) != 1 #error Must define exactly one of GPR_CPU_LINUX, GPR_CPU_POSIX #endif From 93d970a9eff403579e7f5205692f92902ba9f91c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 9 Jan 2015 13:35:04 -0800 Subject: [PATCH 2/6] Fix win32 build error. --- src/core/iomgr/pollset.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/iomgr/pollset.h b/src/core/iomgr/pollset.h index 7374a4ec131..36d80d5c297 100644 --- a/src/core/iomgr/pollset.h +++ b/src/core/iomgr/pollset.h @@ -35,6 +35,7 @@ #define __GRPC_INTERNAL_IOMGR_POLLSET_H_ #include +#include /* A grpc_pollset is a set of file descriptors that a higher level item is interested in. For example: From 5d191a4bf48ae7c02ed6047aa02898901741c326 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 12 Jan 2015 13:38:32 -0800 Subject: [PATCH 3/6] Add build outputs to .gitignore. --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index e69de29bb2d..5202b53ad25 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1,6 @@ +bins +deps +libs +objs +*.pyc + From 3b08306c1319c55f2f5724104cb96982d895de9a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 12 Jan 2015 13:51:28 -0800 Subject: [PATCH 4/6] Cleanup and fixes to jobset.py --- tools/run_tests/jobset.py | 47 +++++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index d3a46b63e14..7e1ec366f96 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -39,6 +39,36 @@ _RUNNING = object() _KILLED = object() +_COLORS = { + 'red': 31, + 'green': 32, + 'yellow': 33, + } + + +_BEGINNING_OF_LINE = '\x1b[0G' +_CLEAR_LINE = '\x1b[2K' + + +_TAG_COLOR = { + 'FAILED': 'red', + 'PASSED': 'green', + 'START': 'yellow', + 'WAITING': 'yellow', + } + + +def message(tag, message, explanatory_text=None): + sys.stdout.write('%s%s\x1b[%dm%s\x1b[0m: %s%s' % ( + _BEGINNING_OF_LINE, + _CLEAR_LINE, + _COLORS[_TAG_COLOR[tag]], + tag, + message, + '\n%s\n' % explanatory_text if explanatory_text is not None else '')) + sys.stdout.flush() + + class Job(object): """Manages one job.""" @@ -49,9 +79,7 @@ class Job(object): stderr=subprocess.STDOUT, stdout=self._tempfile) self._state = _RUNNING - sys.stdout.write('\x1b[0G\x1b[2K\x1b[33mSTART\x1b[0m: %s' % - self._cmdline) - sys.stdout.flush() + message('START', self._cmdline) def state(self): """Poll current state of the job. Prints messages at completion.""" @@ -60,16 +88,10 @@ class Job(object): self._state = _FAILURE self._tempfile.seek(0) stdout = self._tempfile.read() - sys.stdout.write('\x1b[0G\x1b[2K\x1b[31mFAILED\x1b[0m: %s' - ' [ret=%d]\n' - '%s\n' % ( - self._cmdline, self._process.returncode, stdout)) - sys.stdout.flush() + message('FAILED', '%s [ret=%d]' % (self._cmdline, self._process.returncode), stdout) else: self._state = _SUCCESS - sys.stdout.write('\x1b[0G\x1b[2K\x1b[32mPASSED\x1b[0m: %s' % - self._cmdline) - sys.stdout.flush() + message('PASSED', '%s' % self._cmdline) return self._state def kill(self): @@ -107,7 +129,8 @@ class Jobset(object): dead.add(job) for job in dead: self._running.remove(job) - if not dead: return + if dead: return + message('WAITING', '%d jobs left' % len(self._running)) time.sleep(0.1) def cancelled(self): From b29797b142a016ec9821b65479309329e151f6b4 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 12 Jan 2015 13:51:54 -0800 Subject: [PATCH 5/6] Remove ?san from the default tests. At least until they stabilize some. --- tools/run_tests/run_tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 9234682bea7..c9aedc1679e 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -13,13 +13,14 @@ import watch_dirs # flags required for make for each configuration _CONFIGS = ['dbg', 'opt', 'tsan', 'msan', 'asan'] +_DEFAULT = ['dbg', 'opt'] # parse command line argp = argparse.ArgumentParser(description='Run grpc tests.') argp.add_argument('-c', '--config', choices=['all'] + _CONFIGS, nargs='+', - default=['all']) + default=_DEFAULT) argp.add_argument('-t', '--test-filter', nargs='*', default=['*']) argp.add_argument('-n', '--runs_per_test', default=1, type=int) argp.add_argument('-f', '--forever', From 738c334432bfe79104d4d301989fe30823364279 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 12 Jan 2015 14:28:33 -0800 Subject: [PATCH 6/6] Allow running tests under valgrind --- tools/run_tests/jobset.py | 7 ++-- tools/run_tests/run_tests.py | 62 ++++++++++++++++++++++++++---------- 2 files changed, 51 insertions(+), 18 deletions(-) diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index 7e1ec366f96..b7bcb7b02c1 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -8,7 +8,7 @@ import tempfile import time -_MAX_JOBS = 16 * multiprocessing.cpu_count() +_MAX_JOBS = 3 def shuffle_iteratable(it): @@ -108,6 +108,7 @@ class Jobset(object): self._check_cancelled = check_cancelled self._cancelled = False self._failures = 0 + self._completed = 0 def start(self, cmdline): """Start a job. Return True on success, False on failure.""" @@ -128,9 +129,11 @@ class Jobset(object): if st == _FAILURE: self._failures += 1 dead.add(job) for job in dead: + self._completed += 1 self._running.remove(job) if dead: return - message('WAITING', '%d jobs left' % len(self._running)) + message('WAITING', '%d jobs running, %d complete' % ( + len(self._running), self._completed)) time.sleep(0.1) def cancelled(self): diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index c9aedc1679e..2bd2e6d830c 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -11,14 +11,42 @@ import time import jobset import watch_dirs -# flags required for make for each configuration -_CONFIGS = ['dbg', 'opt', 'tsan', 'msan', 'asan'] + +# SimpleConfig: just compile with CONFIG=config, and run the binary to test +class SimpleConfig(object): + def __init__(self, config): + self.build_config = config + + def run_command(self, binary): + return [binary] + + +# ValgrindConfig: compile with some CONFIG=config, but use valgrind to run +class ValgrindConfig(object): + def __init__(self, config): + self.build_config = config + + def run_command(self, binary): + return ['valgrind', binary] + + +# different configurations we can run under +_CONFIGS = { + 'dbg': SimpleConfig('dbg'), + 'opt': SimpleConfig('opt'), + 'tsan': SimpleConfig('tsan'), + 'msan': SimpleConfig('msan'), + 'asan': SimpleConfig('asan'), + 'valgrind': ValgrindConfig('dbg'), + } + + _DEFAULT = ['dbg', 'opt'] # parse command line argp = argparse.ArgumentParser(description='Run grpc tests.') argp.add_argument('-c', '--config', - choices=['all'] + _CONFIGS, + choices=['all'] + sorted(_CONFIGS.keys()), nargs='+', default=_DEFAULT) argp.add_argument('-t', '--test-filter', nargs='*', default=['*']) @@ -30,10 +58,11 @@ argp.add_argument('-f', '--forever', args = argp.parse_args() # grab config -configs = [cfg - for cfg in itertools.chain.from_iterable( - _CONFIGS if x == 'all' else [x] - for x in args.config)] +run_configs = set(_CONFIGS[cfg] + for cfg in itertools.chain.from_iterable( + _CONFIGS.iterkeys() if x == 'all' else [x] + for x in args.config)) +build_configs = set(cfg.build_config for cfg in run_configs) filters = args.test_filter runs_per_test = args.runs_per_test forever = args.forever @@ -44,20 +73,21 @@ def _build_and_run(check_cancelled): # build latest, sharing cpu between the various makes if not jobset.run( (['make', - '-j', '%d' % max(multiprocessing.cpu_count() / len(configs), 1), + '-j', '%d' % (multiprocessing.cpu_count() + 1), 'buildtests_c', 'CONFIG=%s' % cfg] - for cfg in configs), check_cancelled): + for cfg in build_configs), check_cancelled): sys.exit(1) # run all the tests - jobset.run(([x] - for x in itertools.chain.from_iterable( - itertools.chain.from_iterable(itertools.repeat( - glob.glob('bins/%s/%s_test' % (config, filt)), - runs_per_test)) - for config in configs - for filt in filters)), check_cancelled) + jobset.run(( + config.run_command(x) + for config in run_configs + for filt in filters + for x in itertools.chain.from_iterable(itertools.repeat( + glob.glob('bins/%s/%s_test' % ( + config.build_config, filt)), + runs_per_test))), check_cancelled) if forever: