@ -30,6 +30,8 @@
""" Run tests in parallel. """
from __future__ import print_function
import argparse
import ast
import collections
@ -48,7 +50,7 @@ import sys
import tempfile
import traceback
import time
import urllib2
from six . moves import urllib
import uuid
import jobset
@ -93,7 +95,7 @@ class Config(object):
would like to run
"""
actual_environ = self . environ . copy ( )
for k , v in environ . iterite ms ( ) :
for k , v in environ . items ( ) :
actual_environ [ k ] = v
return jobset . JobSpec ( cmdline = self . tool_prefix + cmdline ,
shortname = shortname ,
@ -202,7 +204,7 @@ class CLanguage(object):
flaky = target . get ( ' flaky ' , False ) ,
environ = env ) )
elif self . args . regex == ' .* ' or self . platform == ' windows ' :
print ' \n WARNING: binary not found, skipping ' , binary
print ( ' \n WARNING: binary not found, skipping ' , binary )
return sorted ( out )
def make_targets ( self ) :
@ -393,7 +395,7 @@ class PythonLanguage(object):
return [ self . config . job_spec (
config . run ,
timeout_seconds = 5 * 60 ,
environ = dict ( environment . items ( ) +
environ = dict ( list ( environment . items ( ) ) +
[ ( ' GRPC_PYTHON_TESTRUNNER_FILTER ' , suite_name ) ] ) ,
shortname = ' %s .test. %s ' % ( config . name , suite_name ) , )
for suite_name in tests_json
@ -755,7 +757,7 @@ def _windows_arch_option(arch):
elif arch == ' x64 ' :
return ' /p:Platform=x64 '
else :
print ' Architecture %s not supported. ' % arch
print ( ' Architecture %s not supported. ' % arch )
sys . exit ( 1 )
@ -773,11 +775,11 @@ def _check_arch_option(arch):
elif runtime_arch == ' 32bit ' and arch == ' x86 ' :
return
else :
print ' Architecture %s does not match current runtime architecture. ' % arch
print ( ' Architecture %s does not match current runtime architecture. ' % arch )
sys . exit ( 1 )
else :
if args . arch != ' default ' :
print ' Architecture %s not supported on current platform. ' % args . arch
print ( ' Architecture %s not supported on current platform. ' % args . arch )
sys . exit ( 1 )
@ -791,7 +793,7 @@ def _windows_build_bat(compiler):
elif compiler == ' vs2010 ' :
return ' vsprojects \\ build_vs2010.bat '
else :
print ' Compiler %s not supported. ' % compiler
print ( ' Compiler %s not supported. ' % compiler )
sys . exit ( 1 )
@ -805,7 +807,7 @@ def _windows_toolset_option(compiler):
elif compiler == ' vs2010 ' :
return ' /p:PlatformToolset=v100 '
else :
print ' Compiler %s not supported. ' % compiler
print ( ' Compiler %s not supported. ' % compiler )
sys . exit ( 1 )
@ -816,7 +818,7 @@ def _docker_arch_suffix(arch):
elif arch == ' x86 ' :
return ' x86 '
else :
print ' Architecture %s not supported with current settings. ' % arch
print ( ' Architecture %s not supported with current settings. ' % arch )
sys . exit ( 1 )
@ -932,7 +934,7 @@ for spec in args.update_submodules:
branch = spec [ 1 ]
cwd = ' third_party/ %s ' % submodule
def git ( cmd , cwd = cwd ) :
print ' in %s : git %s ' % ( cwd , cmd )
print ( ' in %s : git %s ' % ( cwd , cmd ) )
subprocess . check_call ( ' git %s ' % cmd , cwd = cwd , shell = True )
git ( ' fetch ' )
git ( ' checkout %s ' % branch )
@ -943,8 +945,8 @@ if need_to_regenerate_projects:
if jobset . platform_string ( ) == ' linux ' :
subprocess . check_call ( ' tools/buildgen/generate_projects.sh ' , shell = True )
else :
print ' WARNING: may need to regenerate projects, but since we are not on '
print ' Linux this step is being skipped. Compilation MAY fail. '
print ( ' WARNING: may need to regenerate projects, but since we are not on ' )
print ( ' Linux this step is being skipped. Compilation MAY fail. ' )
# grab config
@ -971,18 +973,18 @@ for l in languages:
language_make_options = [ ]
if any ( language . make_options ( ) for language in languages ) :
if not ' gcov ' in args . config and len ( languages ) != 1 :
print ' languages with custom make options cannot be built simultaneously with other languages '
print ( ' languages with custom make options cannot be built simultaneously with other languages ' )
sys . exit ( 1 )
else :
language_make_options = next ( iter ( languages ) ) . make_options ( )
if args . use_docker :
if not args . travis :
print ' Seen --use_docker flag, will run tests under docker. '
print
print ' IMPORTANT: The changes you are testing need to be locally committed '
print ' because only the committed changes in the current branch will be '
print ' copied to the docker environment. '
print ( ' Seen --use_docker flag, will run tests under docker. ' )
print ( ' ' )
print ( ' IMPORTANT: The changes you are testing need to be locally committed ' )
print ( ' because only the committed changes in the current branch will be ' )
print ( ' copied to the docker environment. ' )
time . sleep ( 5 )
dockerfile_dirs = set ( [ l . dockerfile_dir ( ) for l in languages ] )
@ -1066,7 +1068,7 @@ build_steps = list(set(
for l in languages
for cmdline in l . pre_build_steps ( ) ) )
if make_targets :
make_commands = itertools . chain . from_iterable ( make_jobspec ( build_config , list ( targets ) , makefile ) for ( makefile , targets ) in make_targets . iterite ms ( ) )
make_commands = itertools . chain . from_iterable ( make_jobspec ( build_config , list ( targets ) , makefile ) for ( makefile , targets ) in make_targets . items ( ) )
build_steps . extend ( set ( make_commands ) )
build_steps . extend ( set (
jobset . JobSpec ( cmdline , environ = build_step_environ ( build_config ) , timeout_seconds = None )
@ -1083,13 +1085,13 @@ forever = args.forever
def _shut_down_legacy_server ( legacy_server_port ) :
try :
version = int ( urllib2 . urlopen (
version = int ( urllib . request . urlopen (
' http://localhost: %d /version_number ' % legacy_server_port ,
timeout = 10 ) . read ( ) )
except :
pass
else :
urllib2 . urlopen (
urllib . request . urlopen (
' http://localhost: %d /quitquitquit ' % legacy_server_port ) . read ( )
@ -1099,29 +1101,29 @@ def _start_port_server(port_server_port):
# if not running ==> start a new one
# otherwise, leave it up
try :
version = int ( urllib2 . urlopen (
version = int ( urllib . request . urlopen (
' http://localhost: %d /version_number ' % port_server_port ,
timeout = 10 ) . read ( ) )
print ' detected port server running version %d ' % version
print ( ' detected port server running version %d ' % version )
running = True
except Exception as e :
print ' failed to detect port server: %s ' % sys . exc_info ( ) [ 0 ]
print e . strerror
print ( ' failed to detect port server: %s ' % sys . exc_info ( ) [ 0 ] )
print ( e . strerror )
running = False
if running :
current_version = int ( subprocess . check_output (
[ sys . executable , os . path . abspath ( ' tools/run_tests/port_server.py ' ) ,
' dump_version ' ] ) )
print ' my port server is version %d ' % current_version
print ( ' my port server is version %d ' % current_version )
running = ( version > = current_version )
if not running :
print ' port_server version mismatch: killing the old one '
urllib2 . urlopen ( ' http://localhost: %d /quitquitquit ' % port_server_port ) . read ( )
print ( ' port_server version mismatch: killing the old one ' )
urllib . request . urlopen ( ' http://localhost: %d /quitquitquit ' % port_server_port ) . read ( )
time . sleep ( 1 )
if not running :
fd , logfile = tempfile . mkstemp ( )
os . close ( fd )
print ' starting port_server, with log file %s ' % logfile
print ( ' starting port_server, with log file %s ' % logfile )
args = [ sys . executable , os . path . abspath ( ' tools/run_tests/port_server.py ' ) ,
' -p ' , ' %d ' % port_server_port , ' -l ' , logfile ]
env = dict ( os . environ )
@ -1147,34 +1149,34 @@ def _start_port_server(port_server_port):
waits = 0
while True :
if waits > 10 :
print ' killing port server due to excessive start up waits '
print ( ' killing port server due to excessive start up waits ' )
port_server . kill ( )
if port_server . poll ( ) is not None :
print ' port_server failed to start '
print ( ' port_server failed to start ' )
# try one final time: maybe another build managed to start one
time . sleep ( 1 )
try :
urllib2 . urlopen ( ' http://localhost: %d /get ' % port_server_port ,
urllib . request . urlopen ( ' http://localhost: %d /get ' % port_server_port ,
timeout = 1 ) . read ( )
print ' last ditch attempt to contact port server succeeded '
print ( ' last ditch attempt to contact port server succeeded ' )
break
except :
traceback . print_exc ( )
port_log = open ( logfile , ' r ' ) . read ( )
print port_log
print ( port_log )
sys . exit ( 1 )
try :
urllib2 . urlopen ( ' http://localhost: %d /get ' % port_server_port ,
urllib . request . urlopen ( ' http://localhost: %d /get ' % port_server_port ,
timeout = 1 ) . read ( )
print ' port server is up and ready '
print ( ' port server is up and ready ' )
break
except socket . timeout :
print ' waiting for port_server: timeout '
print ( ' waiting for port_server: timeout ' )
traceback . print_exc ( ) ;
time . sleep ( 1 )
waits + = 1
except urllib2 . URLError :
print ' waiting for port_server: urlerror '
except urllib . error . URLError :
print ( ' waiting for port_server: urlerror ' )
traceback . print_exc ( ) ;
time . sleep ( 1 )
waits + = 1
siddharthshukla
9 years ago