Merge pull request #972 from donnadionne/add_timed_out_tests_back

Added timeout for tests that hang

tools/gce_setup/cloud_prod_runner.sh

@@ -28,11 +28,14 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+thisfile=$(readlink -ne "${BASH_SOURCE[0]}")
+current_time=$(date "+%Y-%m-%d-%H-%M-%S")
+result_file_name=cloud_prod_result.$current_time.html
+echo $result_file_name
 
 main() {
   source grpc_docker.sh
-  # temporarily remove ping_pong and cancel_after_first_response while investigating timeout
+  test_cases=(large_unary empty_unary ping_pong client_streaming server_streaming cancel_after_begin cancel_after_first_response)
-  test_cases=(large_unary empty_unary client_streaming server_streaming cancel_after_begin)
   auth_test_cases=(service_account_creds compute_engine_creds)
   clients=(cxx java go ruby node csharp_mono)
   for test_case in "${test_cases[@]}"
@@ -41,9 +44,9 @@ main() {
     do
       if grpc_cloud_prod_test $test_case grpc-docker-testclients $client
       then
-        echo "$test_case $client $server passed" >> /tmp/cloud_prod_result.txt
+        echo "          ['$test_case', '$client', 'prod', true]," >> /tmp/cloud_prod_result.txt
       else
-        echo "$test_case $client $server failed" >> /tmp/cloud_prod_result.txt
+        echo "          ['$test_case', '$client', 'prod', false]," >> /tmp/cloud_prod_result.txt
       fi
     done
   done
@@ -53,14 +56,20 @@ main() {
     do
       if grpc_cloud_prod_auth_test $test_case grpc-docker-testclients $client
       then
-        echo "$test_case $client $server passed" >> /tmp/cloud_prod_result.txt
+        echo "          ['$test_case', '$client', 'prod', true]," >> /tmp/cloud_prod_result.txt
       else
-        echo "$test_case $client $server failed" >> /tmp/cloud_prod_result.txt
+        echo "          ['$test_case', '$client', 'prod', false]," >> /tmp/cloud_prod_result.txt
       fi
     done
   done
-  gsutil cp /tmp/cloud_prod_result.txt gs://stoked-keyword-656-output/cloud_prod_result.txt
+  if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
-  rm /tmp/cloud_prod_result.txt
+    cat pre.html /tmp/cloud_prod_result.txt post.html > /tmp/cloud_prod_result.html
+    gsutil cp /tmp/cloud_prod_result.txt gs://stoked-keyword-656-output/cloud_prod_result.txt
+    gsutil cp /tmp/cloud_prod_result.html gs://stoked-keyword-656-output/cloud_prod_result.html
+    gsutil cp /tmp/cloud_prod_result.html gs://stoked-keyword-656-output/result_history/$result_file_name
+    rm /tmp/cloud_prod_result.txt
+    rm /tmp/cloud_prod_result.html
+  fi
 }
 
 set -x

tools/gce_setup/grpc_docker.sh

@@ -731,6 +731,44 @@ grpc_launch_servers() {
   done
 }
 
+# Runs a test command on a docker instance
+#
+# The test command is issued via gcloud compute
+#
+# There are 3 possible results:
+# 1. successful return code and finished within 60 seconds
+# 2. failure return code and finished within 60 seconds
+# 3. command does not return within 60 seconds, in which case it will be killed.
+test_runner() {
+  local project_opt="--project $grpc_project"
+  local zone_opt="--zone $grpc_zone"
+  local ssh_cmd="bash -l -c \"$cmd\""
+  echo "will run:"
+  echo "  $ssh_cmd"
+  echo "on $host"
+  [[ $dry_run == 1 ]] && return 0  # don't run the command on a dry run
+  gcloud compute $project_opt ssh $zone_opt $host --command "$cmd" &
+  PID=$!
+  echo "pid is $PID"
+  for x in {0..5}
+  do
+    if ps -p $PID
+    then
+      # test command has not returned and 60 seconds timeout has not reached
+      sleep 10
+    else
+      # test command has returned, return the return code from the test command
+      wait $PID
+      local ret=$?
+      echo " test runner return $ret before timeout"
+      return $ret
+    fi
+  done
+  kill $PID
+  echo "test got killed by timeout return as failure"
+  return 1
+}
+
 # Runs a test command on a docker instance.
 #
 # call-seq:
@@ -790,14 +828,7 @@ grpc_interop_test() {
   cmd=$($grpc_gen_test_cmd $flags)
   [[ -n $cmd ]] || return 1
 
-  local project_opt="--project $grpc_project"
+  test_runner
-  local zone_opt="--zone $grpc_zone"
-  local ssh_cmd="bash -l -c \"$cmd\""
-  echo "will run:"
-  echo "  $ssh_cmd"
-  echo "on $host"
-  [[ $dry_run == 1 ]] && return 0  # don't run the command on a dry run
-  gcloud compute $project_opt ssh $zone_opt $host --command "$cmd" 
 }
 
 # Runs a test command on a docker instance.
@@ -836,14 +867,7 @@ grpc_cloud_prod_test() {
   cmd=$($grpc_gen_test_cmd $test_case_flag)
   [[ -n $cmd ]] || return 1
 
-  local project_opt="--project $grpc_project"
+  test_runner
-  local zone_opt="--zone $grpc_zone"
-  local ssh_cmd="bash -l -c \"$cmd\""
-  echo "will run:"
-  echo "  $ssh_cmd"
-  echo "on $host"
-  [[ $dry_run == 1 ]] && return 0  # don't run the command on a dry run
-  gcloud compute $project_opt ssh $zone_opt $host --command "$cmd"
 }
 
 # Runs a test command on a docker instance.
@@ -882,14 +906,7 @@ grpc_cloud_prod_auth_test() {
   cmd=$($grpc_gen_test_cmd $test_case_flag)
   [[ -n $cmd ]] || return 1
 
-  local project_opt="--project $grpc_project"
+  test_runner
-  local zone_opt="--zone $grpc_zone"
-  local ssh_cmd="bash -l -c \"$cmd\""
-  echo "will run:"
-  echo "  $ssh_cmd"
-  echo "on $host"
-  [[ $dry_run == 1 ]] && return 0  # don't run the command on a dry run
-  gcloud compute $project_opt ssh $zone_opt $host --command "$cmd"
 }
 
 # constructs the full dockerized ruby interop test cmd.

tools/gce_setup/interop_test_runner.sh

@@ -35,8 +35,7 @@ echo $result_file_name
 
 main() {
   source grpc_docker.sh
-  # temporarily remove ping_pong and cancel_after_first_response while investigating timeout
+  test_cases=(large_unary empty_unary ping_pong client_streaming server_streaming cancel_after_begin cancel_after_first_response)
-  test_cases=(large_unary empty_unary client_streaming server_streaming cancel_after_begin)
   clients=(cxx java go ruby node csharp_mono)
   servers=(cxx java go ruby node python)
   for test_case in "${test_cases[@]}"