|
|
|
@ -400,21 +400,6 @@ def wait_until_all_rpcs_go_to_given_backends(backends, |
|
|
|
|
num_rpcs, |
|
|
|
|
allow_failures=False) |
|
|
|
|
|
|
|
|
|
def wait_until_all_rpcs_fail(timeout_sec, num_rpcs): |
|
|
|
|
start_time = time.time() |
|
|
|
|
error_msg = None |
|
|
|
|
logger.debug('Waiting for %d sec until all of next %d RPCs fail' % |
|
|
|
|
(timeout_sec, num_rpcs)) |
|
|
|
|
while time.time() - start_time <= timeout_sec: |
|
|
|
|
error_msg = None |
|
|
|
|
stats = get_client_stats(num_rpcs, timeout_sec) |
|
|
|
|
diff = num_rpcs - stats.num_failures |
|
|
|
|
if not diff : |
|
|
|
|
error_msg = 'Unexpected completion for %d RPCs' % diff |
|
|
|
|
time.sleep(2) |
|
|
|
|
else: |
|
|
|
|
return |
|
|
|
|
raise RpcDistributionError(error_msg) |
|
|
|
|
|
|
|
|
|
def wait_until_rpcs_in_flight(timeout_sec, num_rpcs): |
|
|
|
|
start_time = time.time() |
|
|
|
@ -434,6 +419,7 @@ def wait_until_rpcs_in_flight(timeout_sec, num_rpcs): |
|
|
|
|
return |
|
|
|
|
raise RpcDistributionError(error_msg) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def compare_distributions(actual_distribution, expected_distribution, |
|
|
|
|
threshold): |
|
|
|
|
"""Compare if two distributions are similar. |
|
|
|
@ -1075,8 +1061,7 @@ def test_circuit_breaking(gcp, |
|
|
|
|
configure_client([messages_pb2.ClientConfigureRequest.RpcType.UNARY_CALL], |
|
|
|
|
[(messages_pb2.ClientConfigureRequest.RpcType.UNARY_CALL, |
|
|
|
|
'rpc-behavior', 'keep-open')]) |
|
|
|
|
wait_until_all_rpcs_fail(int(_WAIT_FOR_STATS_SEC + _NUM_TEST_RPCS / args.qps), |
|
|
|
|
_NUM_TEST_RPCS) |
|
|
|
|
wait_until_all_rpcs_go_to_given_backends_or_fail([], _WAIT_FOR_BACKEND_SEC) |
|
|
|
|
_assert_rpcs_in_flight(max_requests) |
|
|
|
|
|
|
|
|
|
# Increment circuit breakers max_requests threshold. |
|
|
|
@ -1086,8 +1071,7 @@ def test_circuit_breaking(gcp, |
|
|
|
|
circuit_breakers={'maxRequests': max_requests}) |
|
|
|
|
wait_until_rpcs_in_flight(int(_WAIT_FOR_STATS_SEC + max_requests / args.qps), |
|
|
|
|
max_requests) |
|
|
|
|
wait_until_all_rpcs_fail(int(_WAIT_FOR_STATS_SEC + _NUM_TEST_RPCS / args.qps), |
|
|
|
|
_NUM_TEST_RPCS) |
|
|
|
|
wait_until_all_rpcs_go_to_given_backends_or_fail([], _WAIT_FOR_BACKEND_SEC) |
|
|
|
|
_assert_rpcs_in_flight(max_requests) |
|
|
|
|
finally: |
|
|
|
|
patch_url_map_backend_service(gcp, original_backend_service) |
|
|
|
|