Chiebot-Mirror
/
grpc
mirror of https://github.com/grpc/grpc.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #24757 from voidzcy/bugfix/create_extra_backend_services_for_circuit_breaking_test

Browse Source 
The backend service API is not able to unset backend service's circuit breakers configuration, which causes trouble for restoring target proxy's validate_for_proxyless flag. So we create two dedicated backend services (still using existing instance groups) for performing the circuit breaking test. Existing backend services, which are shared among other test cases will not be polluted.
pull/24774/head
Chengyuan Zhang 4 years ago committed by GitHub
parent 20c350fd6c 972978c11e
commit 4fb1d56a4a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 81 additions and 37 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 118
      tools/run_tests/run_xds_tests.py

118
tools/run_tests/run_xds_tests.py
Unescape View File

@ -1077,8 +1077,35 @@ def test_header_matching(gcp, original_backend_service, instance_group,
patch_backend_service(gcp, alternate_backend_service, [])
def test_circuit_breaking(gcp, original_backend_service, instance_group,
alternate_backend_service, same_zone_instance_group):
def test_circuit_breaking(gcp, original_backend_service, extra_backend_service,
more_extra_backend_service, instance_group,
same_zone_instance_group):
'''
Since backend service circuit_breakers configuration cannot be unset,
which causes trouble for restoring validate_for_proxy flag in target
proxy/global forwarding rule. This test uses dedicated backend sevices.
The url_map and backend services undergoes the following state changes:
Before test:
original_backend_service -> [instance_group]
extra_backend_service -> []
more_extra_backend_service -> []
url_map -> [original_backend_service]
In test:
extra_backend_service (with circuit_breakers) -> [instance_group]
more_extra_backend_service (with circuit_breakers) -> [same_zone_instance_group]
url_map -> [extra_backend_service, more_extra_backend_service]
After test:
original_backend_service -> [instance_group]
extra_backend_service (with circuit_breakers) -> []
more_extra_backend_service (with circuit_breakers) -> []
url_map -> [original_backend_service]
'''
logger.info('Running test_circuit_breaking')
# The config validation for proxyless doesn't allow setting
# circuit_breakers. Disable validate validate_for_proxyless
@ -1086,42 +1113,43 @@ def test_circuit_breaking(gcp, original_backend_service, instance_group,
# accepts circuit_breakers.
logger.info('disabling validate_for_proxyless in target proxy')
set_validate_for_proxyless(gcp, False)
original_backend_service_max_requests = 500
alternate_backend_service_max_requests = 1000
extra_backend_service_max_requests = 500
more_extra_backend_service_max_requests = 1000
patch_backend_service(
gcp,
original_backend_service, [instance_group],
circuit_breakers={'maxRequests': original_backend_service_max_requests})
logger.info('Waiting for original backends to become healthy')
wait_for_healthy_backends(gcp, original_backend_service, instance_group)
extra_backend_service, [instance_group],
circuit_breakers={'maxRequests': extra_backend_service_max_requests})
logger.info('Waiting for extra backends to become healthy')
wait_for_healthy_backends(gcp, extra_backend_service, instance_group)
patch_backend_service(gcp,
alternate_backend_service, [same_zone_instance_group],
more_extra_backend_service,
[same_zone_instance_group],
circuit_breakers={
'maxRequests':
alternate_backend_service_max_requests
more_extra_backend_service_max_requests
})
logger.info('Waiting for alternate to become healthy')
wait_for_healthy_backends(gcp, alternate_backend_service,
logger.info('Waiting for more extra backend to become healthy')
wait_for_healthy_backends(gcp, more_extra_backend_service,
same_zone_instance_group)
original_backend_instances = get_instance_names(gcp, instance_group)
alternate_backend_instances = get_instance_names(gcp,
same_zone_instance_group)
extra_backend_instances = get_instance_names(gcp, instance_group)
more_extra_backend_instances = get_instance_names(gcp,
same_zone_instance_group)
route_rules = [
{
'priority': 0,
# UnaryCall -> original_backend_service
# UnaryCall -> extra_backend_service
'matchRules': [{
'fullPathMatch': '/grpc.testing.TestService/UnaryCall'
}],
'service': original_backend_service.url
'service': extra_backend_service.url
},
{
'priority': 1,
# EmptyCall -> alternate_backend_service
# EmptyCall -> more_extra_backend_service
'matchRules': [{
'fullPathMatch': '/grpc.testing.TestService/EmptyCall'
}],
'service': alternate_backend_service.url
'service': more_extra_backend_service.url
},
]
try:
@ -1132,11 +1160,11 @@ def test_circuit_breaking(gcp, original_backend_service, instance_group,
], [])
logger.info('Patching url map with %s', route_rules)
patch_url_map_backend_service(gcp,
original_backend_service,
extra_backend_service,
route_rules=route_rules)
logger.info('Waiting for traffic to go to all backends')
wait_until_all_rpcs_go_to_given_backends(
original_backend_instances + alternate_backend_instances,
extra_backend_instances + more_extra_backend_instances,
_WAIT_FOR_STATS_SEC)
# Make all calls keep-open.
@ -1148,33 +1176,32 @@ def test_circuit_breaking(gcp, original_backend_service, instance_group,
(messages_pb2.ClientConfigureRequest.RpcType.EMPTY_CALL,
'rpc-behavior', 'keep-open')])
wait_until_rpcs_in_flight(
'UNARY_CALL',
(_WAIT_FOR_BACKEND_SEC +
int(original_backend_service_max_requests / args.qps)),
original_backend_service_max_requests, 1)
'UNARY_CALL', (_WAIT_FOR_BACKEND_SEC +
int(extra_backend_service_max_requests / args.qps)),
extra_backend_service_max_requests, 1)
wait_until_rpcs_in_flight(
'EMPTY_CALL',
(_WAIT_FOR_BACKEND_SEC +
int(alternate_backend_service_max_requests / args.qps)),
alternate_backend_service_max_requests, 1)
int(more_extra_backend_service_max_requests / args.qps)),
more_extra_backend_service_max_requests, 1)
# Increment circuit breakers max_requests threshold.
original_backend_service_max_requests = 800
extra_backend_service_max_requests = 800
patch_backend_service(gcp,
original_backend_service, [instance_group],
extra_backend_service, [instance_group],
circuit_breakers={
'maxRequests':
original_backend_service_max_requests
extra_backend_service_max_requests
})
wait_until_rpcs_in_flight(
'UNARY_CALL',
(_WAIT_FOR_BACKEND_SEC +
int(original_backend_service_max_requests / args.qps)),
original_backend_service_max_requests, 1)
'UNARY_CALL', (_WAIT_FOR_BACKEND_SEC +
int(extra_backend_service_max_requests / args.qps)),
extra_backend_service_max_requests, 1)
finally:
patch_url_map_backend_service(gcp, original_backend_service)
patch_backend_service(gcp, original_backend_service, [instance_group])
patch_backend_service(gcp, alternate_backend_service, [])
patch_backend_service(gcp, extra_backend_service, [])
patch_backend_service(gcp, extra_backend_service, [])
set_validate_for_proxyless(gcp, True)
@ -1908,6 +1935,13 @@ try:
firewall_name = _BASE_FIREWALL_RULE_NAME + gcp_suffix
backend_service_name = _BASE_BACKEND_SERVICE_NAME + gcp_suffix
alternate_backend_service_name = _BASE_BACKEND_SERVICE_NAME + '-alternate' + gcp_suffix
# TODO(chengyuanzhang): Dedicated backend services created for circuit
# breaking test. Other tests should avoid using them. Once the issue
# for unsetting backend service circuit breakers is resolved or
# configuring backend service circuit breakers is enabled for config
# validation, these dedicated backend services can be eliminated.
extra_backend_service_name = _BASE_BACKEND_SERVICE_NAME + '-extra' + gcp_suffix
more_extra_backend_service_name = _BASE_BACKEND_SERVICE_NAME + '-more-extra' + gcp_suffix
url_map_name = _BASE_URL_MAP_NAME + gcp_suffix
service_host_name = _BASE_SERVICE_HOST + gcp_suffix
target_proxy_name = _BASE_TARGET_PROXY_NAME + gcp_suffix
@ -1929,6 +1963,10 @@ try:
backend_service = get_backend_service(gcp, backend_service_name)
alternate_backend_service = get_backend_service(
gcp, alternate_backend_service_name)
extra_backend_service = get_backend_service(gcp,
extra_backend_service_name)
more_extra_backend_service = get_backend_service(
gcp, more_extra_backend_service_name)
get_url_map(gcp, url_map_name)
get_target_proxy(gcp, target_proxy_name)
get_global_forwarding_rule(gcp, forwarding_rule_name)
@ -1943,6 +1981,10 @@ try:
backend_service = add_backend_service(gcp, backend_service_name)
alternate_backend_service = add_backend_service(
gcp, alternate_backend_service_name)
extra_backend_service = add_backend_service(gcp,
extra_backend_service_name)
more_extra_backend_service = add_backend_service(
gcp, more_extra_backend_service_name)
create_url_map(gcp, url_map_name, backend_service, service_host_name)
create_target_proxy(gcp, target_proxy_name)
potential_service_ports = list(args.service_port_range)
@ -2089,8 +2131,10 @@ try:
alternate_backend_service,
same_zone_instance_group)
elif test_case == 'circuit_breaking':
test_circuit_breaking(gcp, backend_service, instance_group,
alternate_backend_service,
test_circuit_breaking(gcp, backend_service,
extra_backend_service,
more_extra_backend_service,
instance_group,
same_zone_instance_group)
else:
logger.error('Unknown test case: %s', test_case)

Write Preview
Loading…
Cancel
Save