diff --git a/tools/gce_setup/cloud_prod_runner.sh b/tools/gce_setup/cloud_prod_runner.sh index 520dfcd998a..3a9ae51b769 100755 --- a/tools/gce_setup/cloud_prod_runner.sh +++ b/tools/gce_setup/cloud_prod_runner.sh @@ -28,11 +28,14 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +thisfile=$(readlink -ne "${BASH_SOURCE[0]}") +current_time=$(date "+%Y-%m-%d-%H-%M-%S") +result_file_name=cloud_prod_result.$current_time.html +echo $result_file_name main() { source grpc_docker.sh - # temporarily remove ping_pong and cancel_after_first_response while investigating timeout - test_cases=(large_unary empty_unary client_streaming server_streaming cancel_after_begin) + test_cases=(large_unary empty_unary ping_pong client_streaming server_streaming cancel_after_begin cancel_after_first_response) auth_test_cases=(service_account_creds compute_engine_creds) clients=(cxx java go ruby node csharp_mono) for test_case in "${test_cases[@]}" @@ -41,9 +44,9 @@ main() { do if grpc_cloud_prod_test $test_case grpc-docker-testclients $client then - echo "$test_case $client $server passed" >> /tmp/cloud_prod_result.txt + echo " ['$test_case', '$client', 'prod', true]," >> /tmp/cloud_prod_result.txt else - echo "$test_case $client $server failed" >> /tmp/cloud_prod_result.txt + echo " ['$test_case', '$client', 'prod', false]," >> /tmp/cloud_prod_result.txt fi done done @@ -53,14 +56,20 @@ main() { do if grpc_cloud_prod_auth_test $test_case grpc-docker-testclients $client then - echo "$test_case $client $server passed" >> /tmp/cloud_prod_result.txt + echo " ['$test_case', '$client', 'prod', true]," >> /tmp/cloud_prod_result.txt else - echo "$test_case $client $server failed" >> /tmp/cloud_prod_result.txt + echo " ['$test_case', '$client', 'prod', false]," >> /tmp/cloud_prod_result.txt fi done done - gsutil cp /tmp/cloud_prod_result.txt gs://stoked-keyword-656-output/cloud_prod_result.txt - rm /tmp/cloud_prod_result.txt + if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + cat pre.html /tmp/cloud_prod_result.txt post.html > /tmp/cloud_prod_result.html + gsutil cp /tmp/cloud_prod_result.txt gs://stoked-keyword-656-output/cloud_prod_result.txt + gsutil cp /tmp/cloud_prod_result.html gs://stoked-keyword-656-output/cloud_prod_result.html + gsutil cp /tmp/cloud_prod_result.html gs://stoked-keyword-656-output/result_history/$result_file_name + rm /tmp/cloud_prod_result.txt + rm /tmp/cloud_prod_result.html + fi } set -x diff --git a/tools/gce_setup/grpc_docker.sh b/tools/gce_setup/grpc_docker.sh index 619eff56d62..318bfc8868f 100755 --- a/tools/gce_setup/grpc_docker.sh +++ b/tools/gce_setup/grpc_docker.sh @@ -731,6 +731,44 @@ grpc_launch_servers() { done } +# Runs a test command on a docker instance +# +# The test command is issued via gcloud compute +# +# There are 3 possible results: +# 1. successful return code and finished within 60 seconds +# 2. failure return code and finished within 60 seconds +# 3. command does not return within 60 seconds, in which case it will be killed. +test_runner() { + local project_opt="--project $grpc_project" + local zone_opt="--zone $grpc_zone" + local ssh_cmd="bash -l -c \"$cmd\"" + echo "will run:" + echo " $ssh_cmd" + echo "on $host" + [[ $dry_run == 1 ]] && return 0 # don't run the command on a dry run + gcloud compute $project_opt ssh $zone_opt $host --command "$cmd" & + PID=$! + echo "pid is $PID" + for x in {0..5} + do + if ps -p $PID + then + # test command has not returned and 60 seconds timeout has not reached + sleep 10 + else + # test command has returned, return the return code from the test command + wait $PID + local ret=$? + echo " test runner return $ret before timeout" + return $ret + fi + done + kill $PID + echo "test got killed by timeout return as failure" + return 1 +} + # Runs a test command on a docker instance. # # call-seq: @@ -790,14 +828,7 @@ grpc_interop_test() { cmd=$($grpc_gen_test_cmd $flags) [[ -n $cmd ]] || return 1 - local project_opt="--project $grpc_project" - local zone_opt="--zone $grpc_zone" - local ssh_cmd="bash -l -c \"$cmd\"" - echo "will run:" - echo " $ssh_cmd" - echo "on $host" - [[ $dry_run == 1 ]] && return 0 # don't run the command on a dry run - gcloud compute $project_opt ssh $zone_opt $host --command "$cmd" + test_runner } # Runs a test command on a docker instance. @@ -836,14 +867,7 @@ grpc_cloud_prod_test() { cmd=$($grpc_gen_test_cmd $test_case_flag) [[ -n $cmd ]] || return 1 - local project_opt="--project $grpc_project" - local zone_opt="--zone $grpc_zone" - local ssh_cmd="bash -l -c \"$cmd\"" - echo "will run:" - echo " $ssh_cmd" - echo "on $host" - [[ $dry_run == 1 ]] && return 0 # don't run the command on a dry run - gcloud compute $project_opt ssh $zone_opt $host --command "$cmd" + test_runner } # Runs a test command on a docker instance. @@ -882,14 +906,7 @@ grpc_cloud_prod_auth_test() { cmd=$($grpc_gen_test_cmd $test_case_flag) [[ -n $cmd ]] || return 1 - local project_opt="--project $grpc_project" - local zone_opt="--zone $grpc_zone" - local ssh_cmd="bash -l -c \"$cmd\"" - echo "will run:" - echo " $ssh_cmd" - echo "on $host" - [[ $dry_run == 1 ]] && return 0 # don't run the command on a dry run - gcloud compute $project_opt ssh $zone_opt $host --command "$cmd" + test_runner } # constructs the full dockerized ruby interop test cmd. diff --git a/tools/gce_setup/interop_test_runner.sh b/tools/gce_setup/interop_test_runner.sh index ebc631c1fd5..90a78b1264d 100755 --- a/tools/gce_setup/interop_test_runner.sh +++ b/tools/gce_setup/interop_test_runner.sh @@ -35,8 +35,7 @@ echo $result_file_name main() { source grpc_docker.sh - # temporarily remove ping_pong and cancel_after_first_response while investigating timeout - test_cases=(large_unary empty_unary client_streaming server_streaming cancel_after_begin) + test_cases=(large_unary empty_unary ping_pong client_streaming server_streaming cancel_after_begin cancel_after_first_response) clients=(cxx java go ruby node csharp_mono) servers=(cxx java go ruby node python) for test_case in "${test_cases[@]}"