xDS interop: enable pod log collection in the buildscripts (#30735)

- Enables pod log collection in all PSM interop jobs implemented in https://github.com/grpc/grpc/pull/30594.
- Associate test suite runs with their own log file, so it's displayed on "Target Log" tab

tools/internal_ci/linux/grpc_xds_k8s_lb.cfg

@@ -20,7 +20,7 @@ timeout_mins: 180
 action {
   define_artifacts {
     regex: "artifacts/**/*sponge_log.xml"
-    regex: "artifacts/**/*sponge_log.log"
+    regex: "artifacts/**/*.log"
     strip_prefix: "artifacts"
   }
 }

tools/internal_ci/linux/grpc_xds_k8s_lb.sh

@@ -101,6 +101,8 @@ run_test() {
   # Test driver usage:
   # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage
   local test_name="${1:?Usage: run_test test_name}"
+  local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}"
+  mkdir -pv "${out_dir}"
   set -x
   python3 -m "tests.${test_name}" \
     --flagfile="${TEST_DRIVER_FLAGFILE}" \
@@ -109,8 +111,12 @@ run_test() {
     --server_image="${SERVER_IMAGE_NAME}:${GIT_COMMIT}" \
     --client_image="${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" \
     --testing_version="${TESTING_VERSION}" \
-    --xml_output_file="${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml" \
-    ${@:2}
+    --force_cleanup \
+    --collect_app_logs \
+    --log_dir="${out_dir}" \
+    --xml_output_file="${out_dir}/sponge_log.xml" \
+    ${@:2} \
+    |& tee "${out_dir}/sponge_log.log"
 }
 
 run_alpha_test() {
@@ -171,7 +177,6 @@ main() {
   if (( failed_tests > 0 )); then
     exit 1
   fi
-
 }
 
 main "$@"

tools/internal_ci/linux/grpc_xds_k8s_lb_python.cfg

@@ -20,7 +20,7 @@ timeout_mins: 120
 action {
   define_artifacts {
     regex: "artifacts/**/*sponge_log.xml"
-    regex: "artifacts/**/*sponge_log.log"
+    regex: "artifacts/**/*.log"
     strip_prefix: "artifacts"
   }
 }

tools/internal_ci/linux/grpc_xds_k8s_lb_python.sh

@@ -114,6 +114,8 @@ run_test() {
   # Test driver usage:
   # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage
   local test_name="${1:?Usage: run_test test_name}"
+  local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}"
+  mkdir -pv "${out_dir}"
   set -x
   python3 -m "tests.${test_name}" \
     --flagfile="${TEST_DRIVER_FLAGFILE}" \
@@ -122,7 +124,11 @@ run_test() {
     --client_image="${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" \
     --server_image="${SERVER_IMAGE_NAME}:${GIT_COMMIT}" \
     --testing_version="${TESTING_VERSION}" \
-    --xml_output_file="${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml"
+    --force_cleanup \
+    --collect_app_logs \
+    --log_dir="${out_dir}" \
+    --xml_output_file="${out_dir}/sponge_log.xml" \
+    |& tee "${out_dir}/sponge_log.log"
 }
 
 #######################################

tools/internal_ci/linux/grpc_xds_k8s_xlang.cfg

@@ -20,7 +20,7 @@ timeout_mins: 540
 action {
   define_artifacts {
     regex: "artifacts/**/*sponge_log.xml"
-    regex: "artifacts/**/*sponge_log.log"
+    regex: "artifacts/**/*.log"
     strip_prefix: "artifacts"
   }
 }

tools/internal_ci/linux/grpc_xds_k8s_xlang.sh

@@ -50,6 +50,8 @@ run_test() {
   local server_image_name="${IMAGE_REPO}/${slang}-server:${tag}"
   local client_image_name="${IMAGE_REPO}/${clang}-client:${tag}"
   # TODO(sanjaypujare): skip test if image not found (by using gcloud_gcr_list_image_tags)
+  local out_dir="${TEST_XML_OUTPUT_DIR}/${tag}/${clang}-${slang}"
+  mkdir -pv "${out_dir}"
   set -x
   python -m "tests.security_test" \
     --flagfile="${TEST_DRIVER_FLAGFILE}" \
@@ -57,9 +59,12 @@ run_test() {
     --server_image="${server_image_name}" \
     --client_image="${client_image_name}" \
     --testing_version="${TESTING_VERSION}" \
-    --xml_output_file="${TEST_XML_OUTPUT_DIR}/${tag}/${clang}-${slang}/sponge_log.xml" \
+    --nocheck_local_certs \
     --force_cleanup \
-    --nocheck_local_certs
+    --collect_app_logs \
+    --log_dir="${out_dir}" \
+    --xml_output_file="${out_dir}/sponge_log.xml" \
+    |& tee "${out_dir}/sponge_log.log"
 }
 
 #######################################

tools/internal_ci/linux/grpc_xds_url_map.cfg

@@ -20,7 +20,7 @@ timeout_mins: 90
 action {
   define_artifacts {
     regex: "artifacts/**/*sponge_log.xml"
-    regex: "artifacts/**/*sponge_log.log"
+    regex: "artifacts/**/*.log"
     strip_prefix: "artifacts"
   }
 }

tools/internal_ci/linux/grpc_xds_url_map.sh

@@ -89,14 +89,19 @@ run_test() {
   # Test driver usage:
   # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage
   local test_name="${1:?Usage: run_test test_name}"
+  local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}"
+  mkdir -pv "${out_dir}"
   set -x
   python3 -m "tests.${test_name}" \
     --flagfile="${TEST_DRIVER_FLAGFILE}" \
+    --flagfile="config/url-map.cfg" \
     --kube_context="${KUBE_CONTEXT}" \
     --client_image="${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" \
     --testing_version="${TESTING_VERSION}" \
-    --xml_output_file="${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml" \
-    --flagfile="config/url-map.cfg"
+    --collect_app_logs \
+    --log_dir="${out_dir}" \
+    --xml_output_file="${out_dir}/sponge_log.xml" \
+    |& tee "${out_dir}/sponge_log.log"
   set +x
 }
 

tools/internal_ci/linux/grpc_xds_url_map_python.cfg

@@ -20,7 +20,7 @@ timeout_mins: 90
 action {
   define_artifacts {
     regex: "artifacts/**/*sponge_log.xml"
-    regex: "artifacts/**/*sponge_log.log"
+    regex: "artifacts/**/*.log"
     strip_prefix: "artifacts"
   }
 }

tools/internal_ci/linux/grpc_xds_url_map_python.sh

@@ -99,14 +99,19 @@ run_test() {
   # Test driver usage:
   # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage
   local test_name="${1:?Usage: run_test test_name}"
+  local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}"
+  mkdir -pv "${out_dir}"
   set -x
   python3 -m "tests.${test_name}" \
     --flagfile="${TEST_DRIVER_FLAGFILE}" \
+    --flagfile="config/url-map.cfg" \
     --kube_context="${KUBE_CONTEXT}" \
     --client_image="${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" \
     --testing_version="${TESTING_VERSION}" \
-    --xml_output_file="${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml" \
-    --flagfile="config/url-map.cfg"
+    --collect_app_logs \
+    --log_dir="${out_dir}" \
+    --xml_output_file="${out_dir}/sponge_log.xml" \
+    |& tee "${out_dir}/sponge_log.log"
   set +x
 }
 

tools/internal_ci/linux/psm-security-python.cfg

@@ -20,7 +20,7 @@ timeout_mins: 240
 action {
   define_artifacts {
     regex: "artifacts/**/*sponge_log.xml"
-    regex: "artifacts/**/*sponge_log.log"
+    regex: "artifacts/**/*.log"
     strip_prefix: "artifacts"
   }
 }

tools/internal_ci/linux/psm-security-python.sh

@@ -116,6 +116,8 @@ run_test() {
   # Test driver usage:
   # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage
   local test_name="${1:?Usage: run_test test_name}"
+  local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}"
+  mkdir -pv "${out_dir}"
   set -x
   python3 -m "tests.${test_name}" \
     --flagfile="${TEST_DRIVER_FLAGFILE}" \
@@ -123,9 +125,12 @@ run_test() {
     --server_image="${SERVER_IMAGE_NAME}:${GIT_COMMIT}" \
     --client_image="${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" \
     --testing_version="${TESTING_VERSION}" \
-    --xml_output_file="${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml" \
+    --nocheck_local_certs \
     --force_cleanup \
-    --nocheck_local_certs
+    --collect_app_logs \
+    --log_dir="${out_dir}" \
+    --xml_output_file="${out_dir}/sponge_log.xml" \
+    |& tee "${out_dir}/sponge_log.log"
   set +x
 }
 
@@ -168,9 +173,15 @@ main() {
   build_docker_images_if_needed
   # Run tests
   cd "${TEST_DRIVER_FULL_DIR}"
-  run_test baseline_test
-  run_test security_test
-  run_test authz_test
+  local failed_tests=0
+  test_suites=("baseline_test" "security_test" "authz_test")
+  for test in "${test_suites[@]}"; do
+    run_test $test || (( failed_tests++ ))
+  done
+  echo "Failed test suites: ${failed_tests}"
+  if (( failed_tests > 0 )); then
+    exit 1
+  fi
 }
 
 main "$@"

tools/internal_ci/linux/psm-security.cfg

@@ -20,7 +20,7 @@ timeout_mins: 240
 action {
   define_artifacts {
     regex: "artifacts/**/*sponge_log.xml"
-    regex: "artifacts/**/*sponge_log.log"
+    regex: "artifacts/**/*.log"
     strip_prefix: "artifacts"
   }
 }

tools/internal_ci/linux/psm-security.sh

@@ -101,6 +101,8 @@ run_test() {
   # Test driver usage:
   # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage
   local test_name="${1:?Usage: run_test test_name}"
+  local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}"
+  mkdir -pv "${out_dir}"
   set -x
   python3 -m "tests.${test_name}" \
     --flagfile="${TEST_DRIVER_FLAGFILE}" \
@@ -108,9 +110,12 @@ run_test() {
     --server_image="${SERVER_IMAGE_NAME}:${GIT_COMMIT}" \
     --client_image="${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" \
     --testing_version="${TESTING_VERSION}" \
-    --xml_output_file="${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml" \
+    --nocheck_local_certs \
     --force_cleanup \
-    --nocheck_local_certs
+    --collect_app_logs \
+    --log_dir="${out_dir}" \
+    --xml_output_file="${out_dir}/sponge_log.xml" \
+    |& tee "${out_dir}/sponge_log.log"
   set +x
 }
 
@@ -153,9 +158,15 @@ main() {
   build_docker_images_if_needed
   # Run tests
   cd "${TEST_DRIVER_FULL_DIR}"
-  run_test baseline_test
-  run_test security_test
-  run_test authz_test
+  local failed_tests=0
+  test_suites=("baseline_test" "security_test" "authz_test")
+  for test in "${test_suites[@]}"; do
+    run_test $test || (( failed_tests++ ))
+  done
+  echo "Failed test suites: ${failed_tests}"
+  if (( failed_tests > 0 )); then
+    exit 1
+  fi
 }
 
 main "$@"