From 1408920ab936104f539a020f7707a5aa6189231b Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 27 Apr 2016 17:55:27 -0700 Subject: [PATCH] get rid of local qpsworkers before starting the tests --- tools/run_tests/performance/kill_workers.sh | 51 +++++++++++++++++++ .../performance/remote_host_prepare.sh | 7 ++- tools/run_tests/run_performance_tests.py | 22 +++++--- 3 files changed, 70 insertions(+), 10 deletions(-) create mode 100755 tools/run_tests/performance/kill_workers.sh diff --git a/tools/run_tests/performance/kill_workers.sh b/tools/run_tests/performance/kill_workers.sh new file mode 100755 index 00000000000..3eae8c31cb6 --- /dev/null +++ b/tools/run_tests/performance/kill_workers.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set -ex + +cd $(dirname $0)/../../.. + +# Make sure there are no pre-existing QPS workers around before starting +# the performance test suite + +# C++ +killall -9 qps_worker || true + +# C# +ps -C mono -o pid=,cmd= | grep QpsWorker | awk '{print $1}' | xargs kill -9 + +# Ruby +ps -C ruby -o pid=,cmd= | grep 'qps/worker.rb' | awk '{print $1}' | xargs kill -9 + +# Node +ps -C node -o pid=,cmd= | grep 'performance/worker.js' | awk '{print $1}' | xargs kill -9 + +# Java +jps | grep LoadWorker | awk '{print $1}' | xargs kill -9 diff --git a/tools/run_tests/performance/remote_host_prepare.sh b/tools/run_tests/performance/remote_host_prepare.sh index 17cfa1a5998..d7f539a74e8 100755 --- a/tools/run_tests/performance/remote_host_prepare.sh +++ b/tools/run_tests/performance/remote_host_prepare.sh @@ -41,10 +41,9 @@ ssh "${USER_AT_HOST}" "rm -rf ~/performance_workspace && mkdir -p ~/performance_ # could also kill jenkins. ssh "${USER_AT_HOST}" "killall -9 qps_worker mono node ruby || true" -# Kill all java LoadWorker processes. We can't just killall java -# as one of the processes might be jenkins. -ssh "${USER_AT_HOST}" 'kill -9 $(jps | grep LoadWorker | cut -f1 -d" ") || true' - # push the current sources to the slave and unpack it. scp ../grpc.tar "${USER_AT_HOST}:~/performance_workspace" ssh "${USER_AT_HOST}" "tar -xf ~/performance_workspace/grpc.tar -C ~/performance_workspace" + +# For consistency with local run, invoke the kill_workers script remotely. +ssh "${USER_AT_HOST}" "~/performance_workspace/grpc/tools/run_tests/performance/kill_workers.sh" diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index ada341abf54..fb3b0a1afdf 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -157,8 +157,9 @@ def archive_repo(languages): sys.exit(1) -def prepare_remote_hosts(hosts): - """Prepares remote hosts.""" +def prepare_remote_hosts(hosts, prepare_local=False): + """Prepares remote hosts (and maybe prepare localhost as well).""" + prepare_timeout = 5*60 prepare_jobs = [] for host in hosts: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host) @@ -167,13 +168,20 @@ def prepare_remote_hosts(hosts): cmdline=['tools/run_tests/performance/remote_host_prepare.sh'], shortname='remote_host_prepare.%s' % host, environ = {'USER_AT_HOST': user_at_host}, - timeout_seconds=5*60)) - jobset.message('START', 'Preparing remote hosts.', do_newline=True) + timeout_seconds=prepare_timeout)) + if prepare_local: + # Prepare localhost as well + prepare_jobs.append( + jobset.JobSpec( + cmdline=['tools/run_tests/performance/kill_workers.sh'], + shortname='local_prepare', + timeout_seconds=prepare_timeout)) + jobset.message('START', 'Preparing hosts.', do_newline=True) num_failures, _ = jobset.run( prepare_jobs, newline_on_success=True, maxjobs=10) if num_failures == 0: jobset.message('SUCCESS', - 'Remote hosts ready to start build.', + 'Prepare step completed successfully.', do_newline=True) else: jobset.message('FAILED', 'Failed to prepare remote hosts.', @@ -322,7 +330,9 @@ if args.remote_driver_host: if remote_hosts: archive_repo(languages=[str(l) for l in languages]) - prepare_remote_hosts(remote_hosts) + prepare_remote_hosts(remote_hosts, prepare_local=True) +else: + prepare_remote_hosts([], prepare_local=True) build_local = False if not args.remote_driver_host: