xds testing: check serving status is set correctly

And print what status was set. This helps debugging when the test fails

tools/internal_ci/linux/grpc_xds_bazel_python_test_in_docker.sh

@@ -46,6 +46,18 @@ touch "$TOOLS_DIR"/src/proto/grpc/testing/__init__.py
     "$PROTO_SOURCE_DIR"/messages.proto \
     "$PROTO_SOURCE_DIR"/empty.proto
 
+HEALTH_PROTO_SOURCE_DIR=src/proto/grpc/health/v1
+HEALTH_PROTO_DEST_DIR=${TOOLS_DIR}/${HEALTH_PROTO_SOURCE_DIR}
+mkdir -p ${HEALTH_PROTO_DEST_DIR}
+touch "$TOOLS_DIR"/src/proto/grpc/health/__init__.py
+touch "$TOOLS_DIR"/src/proto/grpc/health/v1/__init__.py
+
+"$PYTHON" -m grpc_tools.protoc \
+    --proto_path=. \
+    --python_out=${TOOLS_DIR} \
+    --grpc_python_out=${TOOLS_DIR} \
+    ${HEALTH_PROTO_SOURCE_DIR}/health.proto
+
 bazel build //src/python/grpcio_tests/tests_py3_only/interop:xds_interop_client
 
 # Test cases "path_matching" and "header_matching" are not included in "all",

tools/internal_ci/linux/grpc_xds_bazel_test_in_docker.sh

@@ -46,6 +46,18 @@ touch "$TOOLS_DIR"/src/proto/grpc/testing/__init__.py
     "$PROTO_SOURCE_DIR"/messages.proto \
     "$PROTO_SOURCE_DIR"/empty.proto
 
+HEALTH_PROTO_SOURCE_DIR=src/proto/grpc/health/v1
+HEALTH_PROTO_DEST_DIR=${TOOLS_DIR}/${HEALTH_PROTO_SOURCE_DIR}
+mkdir -p ${HEALTH_PROTO_DEST_DIR}
+touch "$TOOLS_DIR"/src/proto/grpc/health/__init__.py
+touch "$TOOLS_DIR"/src/proto/grpc/health/v1/__init__.py
+
+"$PYTHON" -m grpc_tools.protoc \
+    --proto_path=. \
+    --python_out=${TOOLS_DIR} \
+    --grpc_python_out=${TOOLS_DIR} \
+    ${HEALTH_PROTO_SOURCE_DIR}/health.proto
+
 bazel build test/cpp/interop:xds_interop_client
 
 # Test cases "path_matching" and "header_matching" are not included in "all",

tools/internal_ci/linux/grpc_xds_csharp_test_in_docker.sh

@@ -46,6 +46,18 @@ touch "$TOOLS_DIR"/src/proto/grpc/testing/__init__.py
     "$PROTO_SOURCE_DIR"/messages.proto \
     "$PROTO_SOURCE_DIR"/empty.proto
 
+HEALTH_PROTO_SOURCE_DIR=src/proto/grpc/health/v1
+HEALTH_PROTO_DEST_DIR=${TOOLS_DIR}/${HEALTH_PROTO_SOURCE_DIR}
+mkdir -p ${HEALTH_PROTO_DEST_DIR}
+touch "$TOOLS_DIR"/src/proto/grpc/health/__init__.py
+touch "$TOOLS_DIR"/src/proto/grpc/health/v1/__init__.py
+
+"$PYTHON" -m grpc_tools.protoc \
+    --proto_path=. \
+    --python_out=${TOOLS_DIR} \
+    --grpc_python_out=${TOOLS_DIR} \
+    ${HEALTH_PROTO_SOURCE_DIR}/health.proto
+
 python tools/run_tests/run_tests.py -l csharp -c opt --build_only
 
 # Test cases "path_matching" and "header_matching" are not included in "all",

tools/internal_ci/linux/grpc_xds_php_test_in_docker.sh

@@ -46,6 +46,18 @@ touch "$TOOLS_DIR"/src/proto/grpc/testing/__init__.py
     "$PROTO_SOURCE_DIR"/messages.proto \
     "$PROTO_SOURCE_DIR"/empty.proto
 
+HEALTH_PROTO_SOURCE_DIR=src/proto/grpc/health/v1
+HEALTH_PROTO_DEST_DIR=${TOOLS_DIR}/${HEALTH_PROTO_SOURCE_DIR}
+mkdir -p ${HEALTH_PROTO_DEST_DIR}
+touch "$TOOLS_DIR"/src/proto/grpc/health/__init__.py
+touch "$TOOLS_DIR"/src/proto/grpc/health/v1/__init__.py
+
+"$PYTHON" -m grpc_tools.protoc \
+    --proto_path=. \
+    --python_out=${TOOLS_DIR} \
+    --grpc_python_out=${TOOLS_DIR} \
+    ${HEALTH_PROTO_SOURCE_DIR}/health.proto
+
 # Generate and compile the PHP extension.
 (pear package && \
   find . -name grpc-*.tgz | xargs -I{} pecl install {})

tools/internal_ci/linux/grpc_xds_ruby_test_in_docker.sh

@@ -46,6 +46,18 @@ touch "$TOOLS_DIR"/src/proto/grpc/testing/__init__.py
     "$PROTO_SOURCE_DIR"/messages.proto \
     "$PROTO_SOURCE_DIR"/empty.proto
 
+HEALTH_PROTO_SOURCE_DIR=src/proto/grpc/health/v1
+HEALTH_PROTO_DEST_DIR=${TOOLS_DIR}/${HEALTH_PROTO_SOURCE_DIR}
+mkdir -p ${HEALTH_PROTO_DEST_DIR}
+touch "$TOOLS_DIR"/src/proto/grpc/health/__init__.py
+touch "$TOOLS_DIR"/src/proto/grpc/health/v1/__init__.py
+
+"$PYTHON" -m grpc_tools.protoc \
+    --proto_path=. \
+    --python_out=${TOOLS_DIR} \
+    --grpc_python_out=${TOOLS_DIR} \
+    ${HEALTH_PROTO_SOURCE_DIR}/health.proto
+
 (cd src/ruby && bundle && rake compile)
 
 GRPC_VERBOSITY=debug GRPC_TRACE=xds_client,xds_resolver,xds_routing_lb,cds_lb,eds_lb,priority_lb,weighted_target_lb,lrs_lb "$PYTHON" \

tools/run_tests/helper_scripts/prep_xds.sh

@@ -35,3 +35,13 @@ python3 -m grpc_tools.protoc \
     ${PROTO_SOURCE_DIR}/test.proto \
     ${PROTO_SOURCE_DIR}/messages.proto \
     ${PROTO_SOURCE_DIR}/empty.proto
+
+HEALTH_PROTO_SOURCE_DIR=src/proto/grpc/health/v1
+HEALTH_PROTO_DEST_DIR=${TOOLS_DIR}/${HEALTH_PROTO_SOURCE_DIR}
+mkdir -p ${HEALTH_PROTO_DEST_DIR}
+
+python3 -m grpc_tools.protoc \
+    --proto_path=. \
+    --python_out=${TOOLS_DIR} \
+    --grpc_python_out=${TOOLS_DIR} \
+    ${HEALTH_PROTO_SOURCE_DIR}/health.proto

tools/run_tests/run_xds_tests.py

@@ -33,6 +33,8 @@ from oauth2client.client import GoogleCredentials
 import python_utils.jobset as jobset
 import python_utils.report_utils as report_utils
 
+from src.proto.grpc.health.v1 import health_pb2
+from src.proto.grpc.health.v1 import health_pb2_grpc
 from src.proto.grpc.testing import empty_pb2
 from src.proto.grpc.testing import messages_pb2
 from src.proto.grpc.testing import test_pb2_grpc
@@ -1002,14 +1004,30 @@ def test_header_matching(gcp, original_backend_service, instance_group,
 
 
 def set_serving_status(instances, service_port, serving):
+    logger.info('setting %s serving status to %s', instances, serving)
     for instance in instances:
         with grpc.insecure_channel('%s:%d' %
                                    (instance, service_port)) as channel:
+            logger.info('setting %s serving status to %s', instance, serving)
             stub = test_pb2_grpc.XdsUpdateHealthServiceStub(channel)
-            if serving:
-                stub.SetServing(empty_pb2.Empty())
-            else:
-                stub.SetNotServing(empty_pb2.Empty())
+            health_stub = health_pb2_grpc.HealthStub(channel)
+
+            retry_count = 5
+            for i in range(5):
+                if serving:
+                    stub.SetServing(empty_pb2.Empty())
+                else:
+                    stub.SetNotServing(empty_pb2.Empty())
+                serving_status = health_stub.Check(
+                    health_pb2.HealthCheckRequest())
+                logger.info('got instance service status %s', serving_status)
+                want_status = health_pb2.HealthCheckResponse.SERVING if serving else health_pb2.HealthCheckResponse.NOT_SERVING
+                if serving_status.status == want_status:
+                    break
+                if i == retry_count - 1:
+                    raise Exception(
+                        'failed to set instance service status after %d retries'
+                        % retry_count)
 
 
 def is_primary_instance_group(gcp, instance_group):