Merge remote-tracking branch 'upstream/master' into plugin_credentials_api_fix

pull/12374/head
Mark D. Roth 8 years ago
commit 6456e494a8
  1. 6
      include/grpc++/support/channel_arguments.h
  2. 6
      include/grpc/impl/codegen/grpc_types.h
  3. 1
      include/grpc/impl/codegen/port_platform.h
  4. 28
      src/core/ext/filters/client_channel/http_proxy.c
  5. 422
      src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c
  6. 2
      src/core/ext/filters/client_channel/lb_policy_factory.c
  7. 2
      src/core/ext/filters/client_channel/lb_policy_factory.h
  8. 62
      src/core/lib/debug/stats_data.c
  9. 26
      src/core/lib/debug/stats_data.h
  10. 8
      src/core/lib/debug/stats_data.yaml
  11. 2
      src/core/lib/debug/stats_data_bq_schema.sql
  12. 69
      src/core/lib/iomgr/executor.c
  13. 14
      src/core/lib/security/credentials/composite/composite_credentials.c
  14. 4
      src/cpp/common/channel_arguments.cc
  15. 10
      src/python/grpcio/support.py
  16. 9
      src/ruby/lib/grpc/google_rpc_status_utils.rb
  17. 77
      src/ruby/spec/google_rpc_status_utils_spec.rb
  18. 4
      templates/test/cpp/naming/create_private_dns_zone.sh.template
  19. 32
      templates/test/cpp/naming/create_private_dns_zone_defs.include
  20. 4
      templates/test/cpp/naming/private_dns_zone_init.sh.template
  21. 40
      templates/test/cpp/naming/private_dns_zone_init_defs.include
  22. 64
      templates/test/cpp/naming/resolver_gce_integration_tests_defs.include
  23. 4
      templates/test/cpp/naming/resolver_gce_integration_tests_runner.sh.template
  24. 11
      test/core/bad_client/bad_client.c
  25. 1
      test/core/bad_client/bad_client.h
  26. 3
      test/core/bad_client/tests/window_overflow.c
  27. 6
      test/core/iomgr/pollset_set_test.c
  28. 29
      test/core/tsi/transport_security_test_lib.c
  29. 19
      test/core/tsi/transport_security_test_lib.h
  30. 180
      test/cpp/end2end/grpclb_end2end_test.cc
  31. 43
      test/cpp/naming/README.md
  32. 27
      test/cpp/naming/create_private_dns_zone.sh
  33. 110
      test/cpp/naming/gen_build_yaml.py
  34. 215
      test/cpp/naming/private_dns_zone_init.sh
  35. 26
      test/cpp/naming/resolver_component_tests_runner.sh
  36. 359
      test/cpp/naming/resolver_gce_integration_tests_runner.sh
  37. 2
      test/cpp/naming/resolver_test_record_groups.yaml
  38. 2
      test/cpp/naming/test_dns_server.py
  39. 4
      tools/codegen/core/gen_stats_data.py
  40. 163
      tools/github/pr_latency.py
  41. 2
      tools/internal_ci/linux/grpc_performance_profile_daily.sh
  42. 2
      tools/jenkins/run_performance_profile_daily.sh
  43. 8
      tools/run_tests/performance/massage_qps_stats.py
  44. 70
      tools/run_tests/performance/scenario_result_schema.json
  45. 5
      tools/run_tests/run_tests.py

@ -64,6 +64,12 @@ class ChannelArguments {
/// Set the compression algorithm for the channel.
void SetCompressionAlgorithm(grpc_compression_algorithm algorithm);
/// Set the grpclb fallback timeout (in ms) for the channel. If this amount
/// of time has passed but we have not gotten any non-empty \a serverlist from
/// the balancer, we will fall back to use the backend address(es) returned by
/// the resolver.
void SetGrpclbFallbackTimeout(int fallback_timeout);
/// Set the socket mutator for the channel.
void SetSocketMutator(grpc_socket_mutator* mutator);

@ -288,7 +288,11 @@ typedef struct {
"grpc.experimental.tcp_max_read_chunk_size"
/* Timeout in milliseconds to use for calls to the grpclb load balancer.
If 0 or unset, the balancer calls will have no deadline. */
#define GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS "grpc.grpclb_timeout_ms"
#define GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS "grpc.grpclb_call_timeout_ms"
/* Timeout in milliseconds to wait for the serverlist from the grpclb load
balancer before using fallback backend addresses from the resolver.
If 0, fallback will never be used. */
#define GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS "grpc.grpclb_fallback_timeout_ms"
/** If non-zero, grpc server's cronet compression workaround will be enabled */
#define GRPC_ARG_WORKAROUND_CRONET_COMPRESSION \
"grpc.workaround.cronet_compression"

@ -183,7 +183,6 @@
#define _BSD_SOURCE
#endif
#if TARGET_OS_IPHONE
#define GPR_FORBID_UNREACHABLE_CODE 1
#define GPR_PLATFORM_STRING "ios"
#define GPR_CPU_IPHONE 1
#define GPR_PTHREAD_TLS 1

@ -91,6 +91,7 @@ static bool proxy_mapper_map_name(grpc_exec_ctx* exec_ctx,
char* user_cred = NULL;
*name_to_resolve = get_http_proxy_server(exec_ctx, &user_cred);
if (*name_to_resolve == NULL) return false;
char* no_proxy_str = NULL;
grpc_uri* uri =
grpc_uri_parse(exec_ctx, server_uri, false /* suppress_errors */);
if (uri == NULL || uri->path[0] == '\0') {
@ -98,20 +99,14 @@ static bool proxy_mapper_map_name(grpc_exec_ctx* exec_ctx,
"'http_proxy' environment variable set, but cannot "
"parse server URI '%s' -- not using proxy",
server_uri);
if (uri != NULL) {
gpr_free(user_cred);
grpc_uri_destroy(uri);
}
return false;
goto no_use_proxy;
}
if (strcmp(uri->scheme, "unix") == 0) {
gpr_log(GPR_INFO, "not using proxy for Unix domain socket '%s'",
server_uri);
gpr_free(user_cred);
grpc_uri_destroy(uri);
return false;
goto no_use_proxy;
}
char* no_proxy_str = gpr_getenv("no_proxy");
no_proxy_str = gpr_getenv("no_proxy");
if (no_proxy_str != NULL) {
static const char* NO_PROXY_SEPARATOR = ",";
bool use_proxy = true;
@ -147,12 +142,7 @@ static bool proxy_mapper_map_name(grpc_exec_ctx* exec_ctx,
gpr_free(no_proxy_hosts);
gpr_free(server_host);
gpr_free(server_port);
if (!use_proxy) {
grpc_uri_destroy(uri);
gpr_free(*name_to_resolve);
*name_to_resolve = NULL;
return false;
}
if (!use_proxy) goto no_use_proxy;
}
}
grpc_arg args_to_add[2];
@ -173,9 +163,15 @@ static bool proxy_mapper_map_name(grpc_exec_ctx* exec_ctx,
} else {
*new_args = grpc_channel_args_copy_and_add(args, args_to_add, 1);
}
gpr_free(user_cred);
grpc_uri_destroy(uri);
gpr_free(user_cred);
return true;
no_use_proxy:
if (uri != NULL) grpc_uri_destroy(uri);
gpr_free(*name_to_resolve);
*name_to_resolve = NULL;
gpr_free(user_cred);
return false;
}
static bool proxy_mapper_map_address(grpc_exec_ctx* exec_ctx,

@ -123,6 +123,7 @@
#define GRPC_GRPCLB_RECONNECT_BACKOFF_MULTIPLIER 1.6
#define GRPC_GRPCLB_RECONNECT_MAX_BACKOFF_SECONDS 120
#define GRPC_GRPCLB_RECONNECT_JITTER 0.2
#define GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS 10000
grpc_tracer_flag grpc_lb_glb_trace = GRPC_TRACER_INITIALIZER(false, "glb");
@ -299,6 +300,10 @@ typedef struct glb_lb_policy {
/** timeout in milliseconds for the LB call. 0 means no deadline. */
int lb_call_timeout_ms;
/** timeout in milliseconds for before using fallback backend addresses.
* 0 means not using fallback. */
int lb_fallback_timeout_ms;
/** for communicating with the LB server */
grpc_channel *lb_channel;
@ -325,6 +330,9 @@ typedef struct glb_lb_policy {
* Otherwise, we delegate to the RR policy. */
size_t serverlist_index;
/** stores the backend addresses from the resolver */
grpc_lb_addresses *fallback_backend_addresses;
/** list of picks that are waiting on RR's policy connectivity */
pending_pick *pending_picks;
@ -345,6 +353,9 @@ typedef struct glb_lb_policy {
/** is \a lb_call_retry_timer active? */
bool retry_timer_active;
/** is \a lb_fallback_timer active? */
bool fallback_timer_active;
/** called upon changes to the LB channel's connectivity. */
grpc_closure lb_channel_on_connectivity_changed;
@ -354,9 +365,6 @@ typedef struct glb_lb_policy {
/************************************************************/
/* client data associated with the LB server communication */
/************************************************************/
/* Finished sending initial request. */
grpc_closure lb_on_sent_initial_request;
/* Status from the LB server has been received. This signals the end of the LB
* call. */
grpc_closure lb_on_server_status_received;
@ -367,6 +375,9 @@ typedef struct glb_lb_policy {
/* LB call retry timer callback. */
grpc_closure lb_on_call_retry;
/* LB fallback timer callback. */
grpc_closure lb_on_fallback;
grpc_call *lb_call; /* streaming call to the LB server, */
grpc_metadata_array lb_initial_metadata_recv; /* initial MD from LB server */
@ -390,7 +401,9 @@ typedef struct glb_lb_policy {
/** LB call retry timer */
grpc_timer lb_call_retry_timer;
bool initial_request_sent;
/** LB fallback timer */
grpc_timer lb_fallback_timer;
bool seen_initial_response;
/* Stats for client-side load reporting. Should be unreffed and
@ -536,6 +549,32 @@ static grpc_lb_addresses *process_serverlist_locked(
return lb_addresses;
}
/* Returns the backend addresses extracted from the given addresses */
static grpc_lb_addresses *extract_backend_addresses_locked(
grpc_exec_ctx *exec_ctx, const grpc_lb_addresses *addresses) {
/* first pass: count the number of backend addresses */
size_t num_backends = 0;
for (size_t i = 0; i < addresses->num_addresses; ++i) {
if (!addresses->addresses[i].is_balancer) {
++num_backends;
}
}
/* second pass: actually populate the addresses and (empty) LB tokens */
grpc_lb_addresses *backend_addresses =
grpc_lb_addresses_create(num_backends, &lb_token_vtable);
size_t num_copied = 0;
for (size_t i = 0; i < addresses->num_addresses; ++i) {
if (addresses->addresses[i].is_balancer) continue;
const grpc_resolved_address *addr = &addresses->addresses[i].address;
grpc_lb_addresses_set_address(backend_addresses, num_copied, &addr->addr,
addr->len, false /* is_balancer */,
NULL /* balancer_name */,
(void *)GRPC_MDELEM_LB_TOKEN_EMPTY.payload);
++num_copied;
}
return backend_addresses;
}
static void update_lb_connectivity_status_locked(
grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy,
grpc_connectivity_state rr_state, grpc_error *rr_state_error) {
@ -603,35 +642,38 @@ static bool pick_from_internal_rr_locked(
grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy,
const grpc_lb_policy_pick_args *pick_args, bool force_async,
grpc_connected_subchannel **target, wrapped_rr_closure_arg *wc_arg) {
// Look at the index into the serverlist to see if we should drop this call.
grpc_grpclb_server *server =
glb_policy->serverlist->servers[glb_policy->serverlist_index++];
if (glb_policy->serverlist_index == glb_policy->serverlist->num_servers) {
glb_policy->serverlist_index = 0; // Wrap-around.
}
if (server->drop) {
// Not using the RR policy, so unref it.
if (GRPC_TRACER_ON(grpc_lb_glb_trace)) {
gpr_log(GPR_INFO, "Unreffing RR for drop (0x%" PRIxPTR ")",
(intptr_t)wc_arg->rr_policy);
// Check for drops if we are not using fallback backend addresses.
if (glb_policy->serverlist != NULL) {
// Look at the index into the serverlist to see if we should drop this call.
grpc_grpclb_server *server =
glb_policy->serverlist->servers[glb_policy->serverlist_index++];
if (glb_policy->serverlist_index == glb_policy->serverlist->num_servers) {
glb_policy->serverlist_index = 0; // Wrap-around.
}
GRPC_LB_POLICY_UNREF(exec_ctx, wc_arg->rr_policy, "glb_pick_sync");
// Update client load reporting stats to indicate the number of
// dropped calls. Note that we have to do this here instead of in
// the client_load_reporting filter, because we do not create a
// subchannel call (and therefore no client_load_reporting filter)
// for dropped calls.
grpc_grpclb_client_stats_add_call_dropped_locked(server->load_balance_token,
wc_arg->client_stats);
grpc_grpclb_client_stats_unref(wc_arg->client_stats);
if (force_async) {
GPR_ASSERT(wc_arg->wrapped_closure != NULL);
GRPC_CLOSURE_SCHED(exec_ctx, wc_arg->wrapped_closure, GRPC_ERROR_NONE);
if (server->drop) {
// Not using the RR policy, so unref it.
if (GRPC_TRACER_ON(grpc_lb_glb_trace)) {
gpr_log(GPR_INFO, "Unreffing RR for drop (0x%" PRIxPTR ")",
(intptr_t)wc_arg->rr_policy);
}
GRPC_LB_POLICY_UNREF(exec_ctx, wc_arg->rr_policy, "glb_pick_sync");
// Update client load reporting stats to indicate the number of
// dropped calls. Note that we have to do this here instead of in
// the client_load_reporting filter, because we do not create a
// subchannel call (and therefore no client_load_reporting filter)
// for dropped calls.
grpc_grpclb_client_stats_add_call_dropped_locked(
server->load_balance_token, wc_arg->client_stats);
grpc_grpclb_client_stats_unref(wc_arg->client_stats);
if (force_async) {
GPR_ASSERT(wc_arg->wrapped_closure != NULL);
GRPC_CLOSURE_SCHED(exec_ctx, wc_arg->wrapped_closure, GRPC_ERROR_NONE);
gpr_free(wc_arg->free_when_done);
return false;
}
gpr_free(wc_arg->free_when_done);
return false;
return true;
}
gpr_free(wc_arg->free_when_done);
return true;
}
// Pick via the RR policy.
const bool pick_done = grpc_lb_policy_pick_locked(
@ -669,8 +711,18 @@ static bool pick_from_internal_rr_locked(
static grpc_lb_policy_args *lb_policy_args_create(grpc_exec_ctx *exec_ctx,
glb_lb_policy *glb_policy) {
grpc_lb_addresses *addresses =
process_serverlist_locked(exec_ctx, glb_policy->serverlist);
grpc_lb_addresses *addresses;
if (glb_policy->serverlist != NULL) {
GPR_ASSERT(glb_policy->serverlist->num_servers > 0);
addresses = process_serverlist_locked(exec_ctx, glb_policy->serverlist);
} else {
// If rr_handover_locked() is invoked when we haven't received any
// serverlist from the balancer, we use the fallback backends returned by
// the resolver. Note that the fallback backend list may be empty, in which
// case the new round_robin policy will keep the requested picks pending.
GPR_ASSERT(glb_policy->fallback_backend_addresses != NULL);
addresses = grpc_lb_addresses_copy(glb_policy->fallback_backend_addresses);
}
GPR_ASSERT(addresses != NULL);
grpc_lb_policy_args *args = (grpc_lb_policy_args *)gpr_zalloc(sizeof(*args));
args->client_channel_factory = glb_policy->cc_factory;
@ -776,8 +828,6 @@ static void create_rr_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy,
/* glb_policy->rr_policy may be NULL (initial handover) */
static void rr_handover_locked(grpc_exec_ctx *exec_ctx,
glb_lb_policy *glb_policy) {
GPR_ASSERT(glb_policy->serverlist != NULL &&
glb_policy->serverlist->num_servers > 0);
if (glb_policy->shutting_down) return;
grpc_lb_policy_args *args = lb_policy_args_create(exec_ctx, glb_policy);
GPR_ASSERT(args != NULL);
@ -926,6 +976,9 @@ static void glb_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) {
if (glb_policy->serverlist != NULL) {
grpc_grpclb_destroy_serverlist(glb_policy->serverlist);
}
if (glb_policy->fallback_backend_addresses != NULL) {
grpc_lb_addresses_destroy(exec_ctx, glb_policy->fallback_backend_addresses);
}
grpc_fake_resolver_response_generator_unref(glb_policy->response_generator);
grpc_subchannel_index_unref();
if (glb_policy->pending_update_args != NULL) {
@ -1067,10 +1120,28 @@ static void glb_cancel_picks_locked(grpc_exec_ctx *exec_ctx,
GRPC_ERROR_UNREF(error);
}
static void lb_on_fallback_timer_locked(grpc_exec_ctx *exec_ctx, void *arg,
grpc_error *error);
static void query_for_backends_locked(grpc_exec_ctx *exec_ctx,
glb_lb_policy *glb_policy);
static void start_picking_locked(grpc_exec_ctx *exec_ctx,
glb_lb_policy *glb_policy) {
/* start a timer to fall back */
if (glb_policy->lb_fallback_timeout_ms > 0 &&
glb_policy->serverlist == NULL && !glb_policy->fallback_timer_active) {
gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC);
gpr_timespec deadline = gpr_time_add(
now,
gpr_time_from_millis(glb_policy->lb_fallback_timeout_ms, GPR_TIMESPAN));
GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "grpclb_fallback_timer");
GRPC_CLOSURE_INIT(&glb_policy->lb_on_fallback, lb_on_fallback_timer_locked,
glb_policy,
grpc_combiner_scheduler(glb_policy->base.combiner));
glb_policy->fallback_timer_active = true;
grpc_timer_init(exec_ctx, &glb_policy->lb_fallback_timer, deadline,
&glb_policy->lb_on_fallback, now);
}
glb_policy->started_picking = true;
gpr_backoff_reset(&glb_policy->lb_call_backoff_state);
query_for_backends_locked(exec_ctx, glb_policy);
@ -1173,6 +1244,58 @@ static void glb_notify_on_state_change_locked(grpc_exec_ctx *exec_ctx,
exec_ctx, &glb_policy->state_tracker, current, notify);
}
static void lb_call_on_retry_timer_locked(grpc_exec_ctx *exec_ctx, void *arg,
grpc_error *error) {
glb_lb_policy *glb_policy = (glb_lb_policy *)arg;
glb_policy->retry_timer_active = false;
if (!glb_policy->shutting_down && error == GRPC_ERROR_NONE) {
if (GRPC_TRACER_ON(grpc_lb_glb_trace)) {
gpr_log(GPR_INFO, "Restaring call to LB server (grpclb %p)",
(void *)glb_policy);
}
GPR_ASSERT(glb_policy->lb_call == NULL);
query_for_backends_locked(exec_ctx, glb_policy);
}
GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base, "grpclb_retry_timer");
}
static void maybe_restart_lb_call(grpc_exec_ctx *exec_ctx,
glb_lb_policy *glb_policy) {
if (glb_policy->started_picking && glb_policy->updating_lb_call) {
if (glb_policy->retry_timer_active) {
grpc_timer_cancel(exec_ctx, &glb_policy->lb_call_retry_timer);
}
if (!glb_policy->shutting_down) start_picking_locked(exec_ctx, glb_policy);
glb_policy->updating_lb_call = false;
} else if (!glb_policy->shutting_down) {
/* if we aren't shutting down, restart the LB client call after some time */
gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC);
gpr_timespec next_try =
gpr_backoff_step(&glb_policy->lb_call_backoff_state, now);
if (GRPC_TRACER_ON(grpc_lb_glb_trace)) {
gpr_log(GPR_DEBUG, "Connection to LB server lost (grpclb: %p)...",
(void *)glb_policy);
gpr_timespec timeout = gpr_time_sub(next_try, now);
if (gpr_time_cmp(timeout, gpr_time_0(timeout.clock_type)) > 0) {
gpr_log(GPR_DEBUG,
"... retry_timer_active in %" PRId64 ".%09d seconds.",
timeout.tv_sec, timeout.tv_nsec);
} else {
gpr_log(GPR_DEBUG, "... retry_timer_active immediately.");
}
}
GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "grpclb_retry_timer");
GRPC_CLOSURE_INIT(&glb_policy->lb_on_call_retry,
lb_call_on_retry_timer_locked, glb_policy,
grpc_combiner_scheduler(glb_policy->base.combiner));
glb_policy->retry_timer_active = true;
grpc_timer_init(exec_ctx, &glb_policy->lb_call_retry_timer, next_try,
&glb_policy->lb_on_call_retry, now);
}
GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base,
"lb_on_server_status_received_locked");
}
static void send_client_load_report_locked(grpc_exec_ctx *exec_ctx, void *arg,
grpc_error *error);
@ -1203,21 +1326,6 @@ static void client_load_report_done_locked(grpc_exec_ctx *exec_ctx, void *arg,
schedule_next_client_load_report(exec_ctx, glb_policy);
}
static void do_send_client_load_report_locked(grpc_exec_ctx *exec_ctx,
glb_lb_policy *glb_policy) {
grpc_op op;
memset(&op, 0, sizeof(op));
op.op = GRPC_OP_SEND_MESSAGE;
op.data.send_message.send_message = glb_policy->client_load_report_payload;
GRPC_CLOSURE_INIT(&glb_policy->client_load_report_closure,
client_load_report_done_locked, glb_policy,
grpc_combiner_scheduler(glb_policy->base.combiner));
grpc_call_error call_error = grpc_call_start_batch_and_execute(
exec_ctx, glb_policy->lb_call, &op, 1,
&glb_policy->client_load_report_closure);
GPR_ASSERT(GRPC_CALL_OK == call_error);
}
static bool load_report_counters_are_zero(grpc_grpclb_request *request) {
grpc_grpclb_dropped_call_counts *drop_entries =
(grpc_grpclb_dropped_call_counts *)
@ -1237,6 +1345,9 @@ static void send_client_load_report_locked(grpc_exec_ctx *exec_ctx, void *arg,
glb_policy->client_load_report_timer_pending = false;
GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base,
"client_load_report");
if (glb_policy->lb_call == NULL) {
maybe_restart_lb_call(exec_ctx, glb_policy);
}
return;
}
// Construct message payload.
@ -1260,17 +1371,23 @@ static void send_client_load_report_locked(grpc_exec_ctx *exec_ctx, void *arg,
grpc_raw_byte_buffer_create(&request_payload_slice, 1);
grpc_slice_unref_internal(exec_ctx, request_payload_slice);
grpc_grpclb_request_destroy(request);
// If we've already sent the initial request, then we can go ahead and
// sent the load report. Otherwise, we need to wait until the initial
// request has been sent to send this
// (see lb_on_sent_initial_request_locked() below).
if (glb_policy->initial_request_sent) {
do_send_client_load_report_locked(exec_ctx, glb_policy);
// Send load report message.
grpc_op op;
memset(&op, 0, sizeof(op));
op.op = GRPC_OP_SEND_MESSAGE;
op.data.send_message.send_message = glb_policy->client_load_report_payload;
GRPC_CLOSURE_INIT(&glb_policy->client_load_report_closure,
client_load_report_done_locked, glb_policy,
grpc_combiner_scheduler(glb_policy->base.combiner));
grpc_call_error call_error = grpc_call_start_batch_and_execute(
exec_ctx, glb_policy->lb_call, &op, 1,
&glb_policy->client_load_report_closure);
if (call_error != GRPC_CALL_OK) {
gpr_log(GPR_ERROR, "call_error=%d", call_error);
GPR_ASSERT(GRPC_CALL_OK == call_error);
}
}
static void lb_on_sent_initial_request_locked(grpc_exec_ctx *exec_ctx,
void *arg, grpc_error *error);
static void lb_on_server_status_received_locked(grpc_exec_ctx *exec_ctx,
void *arg, grpc_error *error);
static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg,
@ -1315,9 +1432,6 @@ static void lb_call_init_locked(grpc_exec_ctx *exec_ctx,
grpc_slice_unref_internal(exec_ctx, request_payload_slice);
grpc_grpclb_request_destroy(request);
GRPC_CLOSURE_INIT(&glb_policy->lb_on_sent_initial_request,
lb_on_sent_initial_request_locked, glb_policy,
grpc_combiner_scheduler(glb_policy->base.combiner));
GRPC_CLOSURE_INIT(&glb_policy->lb_on_server_status_received,
lb_on_server_status_received_locked, glb_policy,
grpc_combiner_scheduler(glb_policy->base.combiner));
@ -1332,7 +1446,6 @@ static void lb_call_init_locked(grpc_exec_ctx *exec_ctx,
GRPC_GRPCLB_MIN_CONNECT_TIMEOUT_SECONDS * 1000,
GRPC_GRPCLB_RECONNECT_MAX_BACKOFF_SECONDS * 1000);
glb_policy->initial_request_sent = false;
glb_policy->seen_initial_response = false;
glb_policy->last_client_load_report_counters_were_zero = false;
}
@ -1349,7 +1462,7 @@ static void lb_call_destroy_locked(grpc_exec_ctx *exec_ctx,
grpc_byte_buffer_destroy(glb_policy->lb_request_payload);
grpc_slice_unref_internal(exec_ctx, glb_policy->lb_call_status_details);
if (!glb_policy->client_load_report_timer_pending) {
if (glb_policy->client_load_report_timer_pending) {
grpc_timer_cancel(exec_ctx, &glb_policy->client_load_report_timer);
}
}
@ -1373,7 +1486,7 @@ static void query_for_backends_locked(grpc_exec_ctx *exec_ctx,
GPR_ASSERT(glb_policy->lb_call != NULL);
grpc_call_error call_error;
grpc_op ops[4];
grpc_op ops[3];
memset(ops, 0, sizeof(ops));
grpc_op *op = ops;
@ -1394,13 +1507,8 @@ static void query_for_backends_locked(grpc_exec_ctx *exec_ctx,
op->flags = 0;
op->reserved = NULL;
op++;
/* take a weak ref (won't prevent calling of \a glb_shutdown if the strong ref
* count goes to zero) to be unref'd in lb_on_sent_initial_request_locked() */
GRPC_LB_POLICY_WEAK_REF(&glb_policy->base,
"lb_on_sent_initial_request_locked");
call_error = grpc_call_start_batch_and_execute(
exec_ctx, glb_policy->lb_call, ops, (size_t)(op - ops),
&glb_policy->lb_on_sent_initial_request);
call_error = grpc_call_start_batch_and_execute(exec_ctx, glb_policy->lb_call,
ops, (size_t)(op - ops), NULL);
GPR_ASSERT(GRPC_CALL_OK == call_error);
op = ops;
@ -1437,19 +1545,6 @@ static void query_for_backends_locked(grpc_exec_ctx *exec_ctx,
GPR_ASSERT(GRPC_CALL_OK == call_error);
}
static void lb_on_sent_initial_request_locked(grpc_exec_ctx *exec_ctx,
void *arg, grpc_error *error) {
glb_lb_policy *glb_policy = (glb_lb_policy *)arg;
glb_policy->initial_request_sent = true;
// If we attempted to send a client load report before the initial
// request was sent, send the load report now.
if (glb_policy->client_load_report_payload != NULL) {
do_send_client_load_report_locked(exec_ctx, glb_policy);
}
GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base,
"lb_on_sent_initial_request_locked");
}
static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg,
grpc_error *error) {
glb_lb_policy *glb_policy = (glb_lb_policy *)arg;
@ -1525,6 +1620,15 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg,
if (glb_policy->serverlist != NULL) {
/* dispose of the old serverlist */
grpc_grpclb_destroy_serverlist(glb_policy->serverlist);
} else {
/* or dispose of the fallback */
grpc_lb_addresses_destroy(exec_ctx,
glb_policy->fallback_backend_addresses);
glb_policy->fallback_backend_addresses = NULL;
if (glb_policy->fallback_timer_active) {
grpc_timer_cancel(exec_ctx, &glb_policy->lb_fallback_timer);
glb_policy->fallback_timer_active = false;
}
}
/* and update the copy in the glb_lb_policy instance. This
* serverlist instance will be destroyed either upon the next
@ -1535,9 +1639,7 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg,
}
} else {
if (GRPC_TRACER_ON(grpc_lb_glb_trace)) {
gpr_log(GPR_INFO,
"Received empty server list. Picks will stay pending until "
"a response with > 0 servers is received");
gpr_log(GPR_INFO, "Received empty server list, ignoring.");
}
grpc_grpclb_destroy_serverlist(serverlist);
}
@ -1572,19 +1674,25 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg,
}
}
static void lb_call_on_retry_timer_locked(grpc_exec_ctx *exec_ctx, void *arg,
grpc_error *error) {
static void lb_on_fallback_timer_locked(grpc_exec_ctx *exec_ctx, void *arg,
grpc_error *error) {
glb_lb_policy *glb_policy = (glb_lb_policy *)arg;
glb_policy->retry_timer_active = false;
if (!glb_policy->shutting_down && error == GRPC_ERROR_NONE) {
if (GRPC_TRACER_ON(grpc_lb_glb_trace)) {
gpr_log(GPR_INFO, "Restaring call to LB server (grpclb %p)",
(void *)glb_policy);
glb_policy->fallback_timer_active = false;
/* If we receive a serverlist after the timer fires but before this callback
* actually runs, don't fall back. */
if (glb_policy->serverlist == NULL) {
if (!glb_policy->shutting_down && error == GRPC_ERROR_NONE) {
if (GRPC_TRACER_ON(grpc_lb_glb_trace)) {
gpr_log(GPR_INFO,
"Falling back to use backends from resolver (grpclb %p)",
(void *)glb_policy);
}
GPR_ASSERT(glb_policy->fallback_backend_addresses != NULL);
rr_handover_locked(exec_ctx, glb_policy);
}
GPR_ASSERT(glb_policy->lb_call == NULL);
query_for_backends_locked(exec_ctx, glb_policy);
}
GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base, "grpclb_retry_timer");
GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base,
"grpclb_fallback_timer");
}
static void lb_on_server_status_received_locked(grpc_exec_ctx *exec_ctx,
@ -1603,66 +1711,30 @@ static void lb_on_server_status_received_locked(grpc_exec_ctx *exec_ctx,
}
/* We need to perform cleanups no matter what. */
lb_call_destroy_locked(exec_ctx, glb_policy);
if (glb_policy->started_picking && glb_policy->updating_lb_call) {
if (glb_policy->retry_timer_active) {
grpc_timer_cancel(exec_ctx, &glb_policy->lb_call_retry_timer);
}
if (!glb_policy->shutting_down) start_picking_locked(exec_ctx, glb_policy);
glb_policy->updating_lb_call = false;
} else if (!glb_policy->shutting_down) {
/* if we aren't shutting down, restart the LB client call after some time */
gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC);
gpr_timespec next_try =
gpr_backoff_step(&glb_policy->lb_call_backoff_state, now);
if (GRPC_TRACER_ON(grpc_lb_glb_trace)) {
gpr_log(GPR_DEBUG, "Connection to LB server lost (grpclb: %p)...",
(void *)glb_policy);
gpr_timespec timeout = gpr_time_sub(next_try, now);
if (gpr_time_cmp(timeout, gpr_time_0(timeout.clock_type)) > 0) {
gpr_log(GPR_DEBUG,
"... retry_timer_active in %" PRId64 ".%09d seconds.",
timeout.tv_sec, timeout.tv_nsec);
} else {
gpr_log(GPR_DEBUG, "... retry_timer_active immediately.");
}
}
GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "grpclb_retry_timer");
GRPC_CLOSURE_INIT(&glb_policy->lb_on_call_retry,
lb_call_on_retry_timer_locked, glb_policy,
grpc_combiner_scheduler(glb_policy->base.combiner));
glb_policy->retry_timer_active = true;
grpc_timer_init(exec_ctx, &glb_policy->lb_call_retry_timer, next_try,
&glb_policy->lb_on_call_retry, now);
// If the load report timer is still pending, we wait for it to be
// called before restarting the call. Otherwise, we restart the call
// here.
if (!glb_policy->client_load_report_timer_pending) {
maybe_restart_lb_call(exec_ctx, glb_policy);
}
}
static void fallback_update_locked(grpc_exec_ctx *exec_ctx,
glb_lb_policy *glb_policy,
const grpc_lb_addresses *addresses) {
GPR_ASSERT(glb_policy->fallback_backend_addresses != NULL);
grpc_lb_addresses_destroy(exec_ctx, glb_policy->fallback_backend_addresses);
glb_policy->fallback_backend_addresses =
extract_backend_addresses_locked(exec_ctx, addresses);
if (glb_policy->lb_fallback_timeout_ms > 0 &&
!glb_policy->fallback_timer_active) {
rr_handover_locked(exec_ctx, glb_policy);
}
GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base,
"lb_on_server_status_received_locked");
}
static void glb_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
const grpc_lb_policy_args *args) {
glb_lb_policy *glb_policy = (glb_lb_policy *)policy;
if (glb_policy->updating_lb_channel) {
if (GRPC_TRACER_ON(grpc_lb_glb_trace)) {
gpr_log(GPR_INFO,
"Update already in progress for grpclb %p. Deferring update.",
(void *)glb_policy);
}
if (glb_policy->pending_update_args != NULL) {
grpc_channel_args_destroy(exec_ctx,
glb_policy->pending_update_args->args);
gpr_free(glb_policy->pending_update_args);
}
glb_policy->pending_update_args = (grpc_lb_policy_args *)gpr_zalloc(
sizeof(*glb_policy->pending_update_args));
glb_policy->pending_update_args->client_channel_factory =
args->client_channel_factory;
glb_policy->pending_update_args->args = grpc_channel_args_copy(args->args);
glb_policy->pending_update_args->combiner = args->combiner;
return;
}
glb_policy->updating_lb_channel = true;
// Propagate update to lb_channel (pick first).
const grpc_arg *arg =
grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES);
if (arg == NULL || arg->type != GRPC_ARG_POINTER) {
@ -1680,13 +1752,43 @@ static void glb_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy,
"ignoring.",
(void *)glb_policy);
}
return;
}
const grpc_lb_addresses *addresses =
(const grpc_lb_addresses *)arg->value.pointer.p;
if (glb_policy->serverlist == NULL) {
// If a non-empty serverlist hasn't been received from the balancer,
// propagate the update to fallback_backend_addresses.
fallback_update_locked(exec_ctx, glb_policy, addresses);
} else if (glb_policy->updating_lb_channel) {
// If we have recieved serverlist from the balancer, we need to defer update
// when there is an in-progress one.
if (GRPC_TRACER_ON(grpc_lb_glb_trace)) {
gpr_log(GPR_INFO,
"Update already in progress for grpclb %p. Deferring update.",
(void *)glb_policy);
}
if (glb_policy->pending_update_args != NULL) {
grpc_channel_args_destroy(exec_ctx,
glb_policy->pending_update_args->args);
gpr_free(glb_policy->pending_update_args);
}
glb_policy->pending_update_args = (grpc_lb_policy_args *)gpr_zalloc(
sizeof(*glb_policy->pending_update_args));
glb_policy->pending_update_args->client_channel_factory =
args->client_channel_factory;
glb_policy->pending_update_args->args = grpc_channel_args_copy(args->args);
glb_policy->pending_update_args->combiner = args->combiner;
return;
}
glb_policy->updating_lb_channel = true;
GPR_ASSERT(glb_policy->lb_channel != NULL);
grpc_channel_args *lb_channel_args = build_lb_channel_args(
exec_ctx, addresses, glb_policy->response_generator, args->args);
/* Propagate updates to the LB channel through the fake resolver */
/* Propagate updates to the LB channel (pick first) through the fake resolver
*/
grpc_fake_resolver_response_generator_set_response(
exec_ctx, glb_policy->response_generator, lb_channel_args);
grpc_channel_args_destroy(exec_ctx, lb_channel_args);
@ -1789,13 +1891,7 @@ static const grpc_lb_policy_vtable glb_lb_policy_vtable = {
static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx,
grpc_lb_policy_factory *factory,
grpc_lb_policy_args *args) {
/* Count the number of gRPC-LB addresses. There must be at least one.
* TODO(roth): For now, we ignore non-balancer addresses, but in the
* future, we may change the behavior such that we fall back to using
* the non-balancer addresses if we cannot reach any balancers. In the
* fallback case, we should use the LB policy indicated by
* GRPC_ARG_LB_POLICY_NAME (although if that specifies grpclb or is
* unset, we should default to pick_first). */
/* Count the number of gRPC-LB addresses. There must be at least one. */
const grpc_arg *arg =
grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES);
if (arg == NULL || arg->type != GRPC_ARG_POINTER) {
@ -1831,6 +1927,11 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx,
glb_policy->lb_call_timeout_ms =
grpc_channel_arg_get_integer(arg, (grpc_integer_options){0, 0, INT_MAX});
arg = grpc_channel_args_find(args->args, GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS);
glb_policy->lb_fallback_timeout_ms = grpc_channel_arg_get_integer(
arg, (grpc_integer_options){GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS, 0,
INT_MAX});
// Make sure that GRPC_ARG_LB_POLICY_NAME is set in channel args,
// since we use this to trigger the client_load_reporting filter.
grpc_arg new_arg = grpc_channel_arg_string_create(
@ -1839,6 +1940,11 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx,
glb_policy->args = grpc_channel_args_copy_and_add_and_remove(
args->args, args_to_remove, GPR_ARRAY_SIZE(args_to_remove), &new_arg, 1);
/* Extract the backend addresses (may be empty) from the resolver for
* fallback. */
glb_policy->fallback_backend_addresses =
extract_backend_addresses_locked(exec_ctx, addresses);
/* Create a client channel over them to communicate with a LB service */
glb_policy->response_generator =
grpc_fake_resolver_response_generator_create();

@ -56,7 +56,7 @@ grpc_lb_addresses* grpc_lb_addresses_copy(const grpc_lb_addresses* addresses) {
}
void grpc_lb_addresses_set_address(grpc_lb_addresses* addresses, size_t index,
void* address, size_t address_len,
const void* address, size_t address_len,
bool is_balancer, const char* balancer_name,
void* user_data) {
GPR_ASSERT(index < addresses->num_addresses);

@ -73,7 +73,7 @@ grpc_lb_addresses *grpc_lb_addresses_copy(const grpc_lb_addresses *addresses);
* \a address is a socket address of length \a address_len.
* Takes ownership of \a balancer_name. */
void grpc_lb_addresses_set_address(grpc_lb_addresses *addresses, size_t index,
void *address, size_t address_len,
const void *address, size_t address_len,
bool is_balancer, const char *balancer_name,
void *user_data);

@ -109,8 +109,6 @@ const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = {
"executor_wakeup_initiated",
"executor_queue_drained",
"executor_push_retries",
"executor_threads_created",
"executor_threads_used",
"server_requested_calls",
"server_slowpath_requests_queued",
};
@ -219,8 +217,6 @@ const char *grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = {
"Number of times an executor queue was drained",
"Number of times we raced and were forced to retry pushing a closure to "
"the executor",
"Size of the backing thread pool for overflow gRPC Core work",
"How many executor threads actually got used",
"How many calls were requested (not necessarily received) by the server",
"How many times was the server slow path taken (indicates too few "
"outstanding requests)",
@ -238,7 +234,6 @@ const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT] = {
"http2_send_message_per_write",
"http2_send_trailing_metadata_per_write",
"http2_send_flowctl_per_write",
"executor_closures_per_wakeup",
"server_cqs_checked",
};
const char *grpc_stats_histogram_doc[GRPC_STATS_HISTOGRAM_COUNT] = {
@ -254,7 +249,6 @@ const char *grpc_stats_histogram_doc[GRPC_STATS_HISTOGRAM_COUNT] = {
"Number of streams whose payload was written per TCP write",
"Number of streams terminated per TCP write",
"Number of flow control updates written per TCP write",
"Number of closures executed each time an executor wakes up",
"How many completion queues were checked looking for a CQ that had "
"requested the incoming call",
};
@ -326,7 +320,6 @@ const uint8_t grpc_stats_table_7[102] = {
const int grpc_stats_table_8[9] = {0, 1, 2, 4, 7, 13, 23, 39, 64};
const uint8_t grpc_stats_table_9[9] = {0, 0, 1, 2, 2, 3, 4, 4, 5};
void grpc_stats_inc_call_initial_size(grpc_exec_ctx *exec_ctx, int value) {
/* Automatically generated by tools/codegen/core/gen_stats_data.py */
value = GPR_CLAMP(value, 0, 262144);
if (value < 6) {
GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE,
@ -352,7 +345,6 @@ void grpc_stats_inc_call_initial_size(grpc_exec_ctx *exec_ctx, int value) {
(exec_ctx), value, grpc_stats_table_0, 64));
}
void grpc_stats_inc_poll_events_returned(grpc_exec_ctx *exec_ctx, int value) {
/* Automatically generated by tools/codegen/core/gen_stats_data.py */
value = GPR_CLAMP(value, 0, 1024);
if (value < 29) {
GRPC_STATS_INC_HISTOGRAM((exec_ctx),
@ -379,7 +371,6 @@ void grpc_stats_inc_poll_events_returned(grpc_exec_ctx *exec_ctx, int value) {
(exec_ctx), value, grpc_stats_table_2, 128));
}
void grpc_stats_inc_tcp_write_size(grpc_exec_ctx *exec_ctx, int value) {
/* Automatically generated by tools/codegen/core/gen_stats_data.py */
value = GPR_CLAMP(value, 0, 16777216);
if (value < 5) {
GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE,
@ -405,7 +396,6 @@ void grpc_stats_inc_tcp_write_size(grpc_exec_ctx *exec_ctx, int value) {
(exec_ctx), value, grpc_stats_table_4, 64));
}
void grpc_stats_inc_tcp_write_iov_size(grpc_exec_ctx *exec_ctx, int value) {
/* Automatically generated by tools/codegen/core/gen_stats_data.py */
value = GPR_CLAMP(value, 0, 1024);
if (value < 13) {
GRPC_STATS_INC_HISTOGRAM((exec_ctx),
@ -431,7 +421,6 @@ void grpc_stats_inc_tcp_write_iov_size(grpc_exec_ctx *exec_ctx, int value) {
(exec_ctx), value, grpc_stats_table_6, 64));
}
void grpc_stats_inc_tcp_read_size(grpc_exec_ctx *exec_ctx, int value) {
/* Automatically generated by tools/codegen/core/gen_stats_data.py */
value = GPR_CLAMP(value, 0, 16777216);
if (value < 5) {
GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_SIZE,
@ -457,7 +446,6 @@ void grpc_stats_inc_tcp_read_size(grpc_exec_ctx *exec_ctx, int value) {
(exec_ctx), value, grpc_stats_table_4, 64));
}
void grpc_stats_inc_tcp_read_offer(grpc_exec_ctx *exec_ctx, int value) {
/* Automatically generated by tools/codegen/core/gen_stats_data.py */
value = GPR_CLAMP(value, 0, 16777216);
if (value < 5) {
GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_OFFER,
@ -484,7 +472,6 @@ void grpc_stats_inc_tcp_read_offer(grpc_exec_ctx *exec_ctx, int value) {
}
void grpc_stats_inc_tcp_read_offer_iov_size(grpc_exec_ctx *exec_ctx,
int value) {
/* Automatically generated by tools/codegen/core/gen_stats_data.py */
value = GPR_CLAMP(value, 0, 1024);
if (value < 13) {
GRPC_STATS_INC_HISTOGRAM(
@ -512,7 +499,6 @@ void grpc_stats_inc_tcp_read_offer_iov_size(grpc_exec_ctx *exec_ctx,
}
void grpc_stats_inc_http2_send_message_size(grpc_exec_ctx *exec_ctx,
int value) {
/* Automatically generated by tools/codegen/core/gen_stats_data.py */
value = GPR_CLAMP(value, 0, 16777216);
if (value < 5) {
GRPC_STATS_INC_HISTOGRAM(
@ -540,7 +526,6 @@ void grpc_stats_inc_http2_send_message_size(grpc_exec_ctx *exec_ctx,
}
void grpc_stats_inc_http2_send_initial_metadata_per_write(
grpc_exec_ctx *exec_ctx, int value) {
/* Automatically generated by tools/codegen/core/gen_stats_data.py */
value = GPR_CLAMP(value, 0, 1024);
if (value < 13) {
GRPC_STATS_INC_HISTOGRAM(
@ -570,7 +555,6 @@ void grpc_stats_inc_http2_send_initial_metadata_per_write(
}
void grpc_stats_inc_http2_send_message_per_write(grpc_exec_ctx *exec_ctx,
int value) {
/* Automatically generated by tools/codegen/core/gen_stats_data.py */
value = GPR_CLAMP(value, 0, 1024);
if (value < 13) {
GRPC_STATS_INC_HISTOGRAM(
@ -598,7 +582,6 @@ void grpc_stats_inc_http2_send_message_per_write(grpc_exec_ctx *exec_ctx,
}
void grpc_stats_inc_http2_send_trailing_metadata_per_write(
grpc_exec_ctx *exec_ctx, int value) {
/* Automatically generated by tools/codegen/core/gen_stats_data.py */
value = GPR_CLAMP(value, 0, 1024);
if (value < 13) {
GRPC_STATS_INC_HISTOGRAM(
@ -628,7 +611,6 @@ void grpc_stats_inc_http2_send_trailing_metadata_per_write(
}
void grpc_stats_inc_http2_send_flowctl_per_write(grpc_exec_ctx *exec_ctx,
int value) {
/* Automatically generated by tools/codegen/core/gen_stats_data.py */
value = GPR_CLAMP(value, 0, 1024);
if (value < 13) {
GRPC_STATS_INC_HISTOGRAM(
@ -654,36 +636,7 @@ void grpc_stats_inc_http2_send_flowctl_per_write(grpc_exec_ctx *exec_ctx,
grpc_stats_histo_find_bucket_slow(
(exec_ctx), value, grpc_stats_table_6, 64));
}
void grpc_stats_inc_executor_closures_per_wakeup(grpc_exec_ctx *exec_ctx,
int value) {
/* Automatically generated by tools/codegen/core/gen_stats_data.py */
value = GPR_CLAMP(value, 0, 1024);
if (value < 13) {
GRPC_STATS_INC_HISTOGRAM(
(exec_ctx), GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP, value);
return;
}
union {
double dbl;
uint64_t uint;
} _val, _bkt;
_val.dbl = value;
if (_val.uint < 4637863191261478912ull) {
int bucket =
grpc_stats_table_7[((_val.uint - 4623507967449235456ull) >> 48)] + 13;
_bkt.dbl = grpc_stats_table_6[bucket];
bucket -= (_val.uint < _bkt.uint);
GRPC_STATS_INC_HISTOGRAM(
(exec_ctx), GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP, bucket);
return;
}
GRPC_STATS_INC_HISTOGRAM((exec_ctx),
GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP,
grpc_stats_histo_find_bucket_slow(
(exec_ctx), value, grpc_stats_table_6, 64));
}
void grpc_stats_inc_server_cqs_checked(grpc_exec_ctx *exec_ctx, int value) {
/* Automatically generated by tools/codegen/core/gen_stats_data.py */
value = GPR_CLAMP(value, 0, 64);
if (value < 3) {
GRPC_STATS_INC_HISTOGRAM((exec_ctx),
@ -708,17 +661,17 @@ void grpc_stats_inc_server_cqs_checked(grpc_exec_ctx *exec_ctx, int value) {
grpc_stats_histo_find_bucket_slow(
(exec_ctx), value, grpc_stats_table_8, 8));
}
const int grpc_stats_histo_buckets[14] = {64, 128, 64, 64, 64, 64, 64,
64, 64, 64, 64, 64, 64, 8};
const int grpc_stats_histo_start[14] = {0, 64, 192, 256, 320, 384, 448,
512, 576, 640, 704, 768, 832, 896};
const int *const grpc_stats_histo_bucket_boundaries[14] = {
const int grpc_stats_histo_buckets[13] = {64, 128, 64, 64, 64, 64, 64,
64, 64, 64, 64, 64, 8};
const int grpc_stats_histo_start[13] = {0, 64, 192, 256, 320, 384, 448,
512, 576, 640, 704, 768, 832};
const int *const grpc_stats_histo_bucket_boundaries[13] = {
grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_4,
grpc_stats_table_6, grpc_stats_table_4, grpc_stats_table_4,
grpc_stats_table_6, grpc_stats_table_4, grpc_stats_table_6,
grpc_stats_table_6, grpc_stats_table_6, grpc_stats_table_6,
grpc_stats_table_6, grpc_stats_table_8};
void (*const grpc_stats_inc_histogram[14])(grpc_exec_ctx *exec_ctx, int x) = {
grpc_stats_table_8};
void (*const grpc_stats_inc_histogram[13])(grpc_exec_ctx *exec_ctx, int x) = {
grpc_stats_inc_call_initial_size,
grpc_stats_inc_poll_events_returned,
grpc_stats_inc_tcp_write_size,
@ -731,5 +684,4 @@ void (*const grpc_stats_inc_histogram[14])(grpc_exec_ctx *exec_ctx, int x) = {
grpc_stats_inc_http2_send_message_per_write,
grpc_stats_inc_http2_send_trailing_metadata_per_write,
grpc_stats_inc_http2_send_flowctl_per_write,
grpc_stats_inc_executor_closures_per_wakeup,
grpc_stats_inc_server_cqs_checked};

@ -111,8 +111,6 @@ typedef enum {
GRPC_STATS_COUNTER_EXECUTOR_WAKEUP_INITIATED,
GRPC_STATS_COUNTER_EXECUTOR_QUEUE_DRAINED,
GRPC_STATS_COUNTER_EXECUTOR_PUSH_RETRIES,
GRPC_STATS_COUNTER_EXECUTOR_THREADS_CREATED,
GRPC_STATS_COUNTER_EXECUTOR_THREADS_USED,
GRPC_STATS_COUNTER_SERVER_REQUESTED_CALLS,
GRPC_STATS_COUNTER_SERVER_SLOWPATH_REQUESTS_QUEUED,
GRPC_STATS_COUNTER_COUNT
@ -132,7 +130,6 @@ typedef enum {
GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_PER_WRITE,
GRPC_STATS_HISTOGRAM_HTTP2_SEND_TRAILING_METADATA_PER_WRITE,
GRPC_STATS_HISTOGRAM_HTTP2_SEND_FLOWCTL_PER_WRITE,
GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP,
GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED,
GRPC_STATS_HISTOGRAM_COUNT
} grpc_stats_histograms;
@ -163,11 +160,9 @@ typedef enum {
GRPC_STATS_HISTOGRAM_HTTP2_SEND_TRAILING_METADATA_PER_WRITE_BUCKETS = 64,
GRPC_STATS_HISTOGRAM_HTTP2_SEND_FLOWCTL_PER_WRITE_FIRST_SLOT = 768,
GRPC_STATS_HISTOGRAM_HTTP2_SEND_FLOWCTL_PER_WRITE_BUCKETS = 64,
GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP_FIRST_SLOT = 832,
GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP_BUCKETS = 64,
GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED_FIRST_SLOT = 896,
GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED_FIRST_SLOT = 832,
GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED_BUCKETS = 8,
GRPC_STATS_HISTOGRAM_BUCKETS = 904
GRPC_STATS_HISTOGRAM_BUCKETS = 840
} grpc_stats_histogram_constants;
#define GRPC_STATS_INC_CLIENT_CALLS_CREATED(exec_ctx) \
GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED)
@ -417,11 +412,6 @@ typedef enum {
GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_EXECUTOR_QUEUE_DRAINED)
#define GRPC_STATS_INC_EXECUTOR_PUSH_RETRIES(exec_ctx) \
GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_EXECUTOR_PUSH_RETRIES)
#define GRPC_STATS_INC_EXECUTOR_THREADS_CREATED(exec_ctx) \
GRPC_STATS_INC_COUNTER((exec_ctx), \
GRPC_STATS_COUNTER_EXECUTOR_THREADS_CREATED)
#define GRPC_STATS_INC_EXECUTOR_THREADS_USED(exec_ctx) \
GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_EXECUTOR_THREADS_USED)
#define GRPC_STATS_INC_SERVER_REQUESTED_CALLS(exec_ctx) \
GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SERVER_REQUESTED_CALLS)
#define GRPC_STATS_INC_SERVER_SLOWPATH_REQUESTS_QUEUED(exec_ctx) \
@ -468,17 +458,13 @@ void grpc_stats_inc_http2_send_trailing_metadata_per_write(
grpc_stats_inc_http2_send_flowctl_per_write((exec_ctx), (int)(value))
void grpc_stats_inc_http2_send_flowctl_per_write(grpc_exec_ctx *exec_ctx,
int x);
#define GRPC_STATS_INC_EXECUTOR_CLOSURES_PER_WAKEUP(exec_ctx, value) \
grpc_stats_inc_executor_closures_per_wakeup((exec_ctx), (int)(value))
void grpc_stats_inc_executor_closures_per_wakeup(grpc_exec_ctx *exec_ctx,
int x);
#define GRPC_STATS_INC_SERVER_CQS_CHECKED(exec_ctx, value) \
grpc_stats_inc_server_cqs_checked((exec_ctx), (int)(value))
void grpc_stats_inc_server_cqs_checked(grpc_exec_ctx *exec_ctx, int x);
extern const int grpc_stats_histo_buckets[14];
extern const int grpc_stats_histo_start[14];
extern const int *const grpc_stats_histo_bucket_boundaries[14];
extern void (*const grpc_stats_inc_histogram[14])(grpc_exec_ctx *exec_ctx,
extern const int grpc_stats_histo_buckets[13];
extern const int grpc_stats_histo_start[13];
extern const int *const grpc_stats_histo_bucket_boundaries[13];
extern void (*const grpc_stats_inc_histogram[13])(grpc_exec_ctx *exec_ctx,
int x);
#endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */

@ -259,14 +259,6 @@
- counter: executor_push_retries
doc: Number of times we raced and were forced to retry pushing a closure to
the executor
- counter: executor_threads_created
doc: Size of the backing thread pool for overflow gRPC Core work
- counter: executor_threads_used
doc: How many executor threads actually got used
- histogram: executor_closures_per_wakeup
max: 1024
buckets: 64
doc: Number of closures executed each time an executor wakes up
# server
- counter: server_requested_calls
doc: How many calls were requested (not necessarily received) by the server

@ -84,7 +84,5 @@ executor_scheduled_to_self_per_iteration:FLOAT,
executor_wakeup_initiated_per_iteration:FLOAT,
executor_queue_drained_per_iteration:FLOAT,
executor_push_retries_per_iteration:FLOAT,
executor_threads_created_per_iteration:FLOAT,
executor_threads_used_per_iteration:FLOAT,
server_requested_calls_per_iteration:FLOAT,
server_slowpath_requests_queued_per_iteration:FLOAT

@ -32,14 +32,16 @@
#include "src/core/lib/iomgr/exec_ctx.h"
#include "src/core/lib/support/spinlock.h"
#define MAX_DEPTH 2
typedef struct {
gpr_mu mu;
gpr_cv cv;
grpc_closure_list elems;
size_t depth;
bool shutdown;
bool queued_long_job;
gpr_thd_id id;
grpc_closure_list local_elems;
} thread_state;
static thread_state *g_thread_state;
@ -54,35 +56,32 @@ static grpc_tracer_flag executor_trace =
static void executor_thread(void *arg);
static void run_closures(grpc_exec_ctx *exec_ctx, grpc_closure_list *list) {
int n = 0; // number of closures executed
static size_t run_closures(grpc_exec_ctx *exec_ctx, grpc_closure_list list) {
size_t n = 0;
while (!grpc_closure_list_empty(*list)) {
grpc_closure *c = list->head;
grpc_closure_list_init(list);
while (c != NULL) {
grpc_closure *next = c->next_data.next;
grpc_error *error = c->error_data.error;
if (GRPC_TRACER_ON(executor_trace)) {
grpc_closure *c = list.head;
while (c != NULL) {
grpc_closure *next = c->next_data.next;
grpc_error *error = c->error_data.error;
if (GRPC_TRACER_ON(executor_trace)) {
#ifndef NDEBUG
gpr_log(GPR_DEBUG, "EXECUTOR: run %p [created by %s:%d]", c,
c->file_created, c->line_created);
gpr_log(GPR_DEBUG, "EXECUTOR: run %p [created by %s:%d]", c,
c->file_created, c->line_created);
#else
gpr_log(GPR_DEBUG, "EXECUTOR: run %p", c);
gpr_log(GPR_DEBUG, "EXECUTOR: run %p", c);
#endif
}
}
#ifndef NDEBUG
c->scheduled = false;
c->scheduled = false;
#endif
n++;
c->cb(exec_ctx, c->cb_arg, error);
GRPC_ERROR_UNREF(error);
c = next;
grpc_exec_ctx_flush(exec_ctx);
}
c->cb(exec_ctx, c->cb_arg, error);
GRPC_ERROR_UNREF(error);
c = next;
n++;
grpc_exec_ctx_flush(exec_ctx);
}
GRPC_STATS_INC_EXECUTOR_CLOSURES_PER_WAKEUP(exec_ctx, n);
return n;
}
bool grpc_executor_is_threaded() {
@ -127,7 +126,7 @@ void grpc_executor_set_threading(grpc_exec_ctx *exec_ctx, bool threading) {
for (size_t i = 0; i < g_max_threads; i++) {
gpr_mu_destroy(&g_thread_state[i].mu);
gpr_cv_destroy(&g_thread_state[i].cv);
run_closures(exec_ctx, &g_thread_state[i].elems);
run_closures(exec_ctx, g_thread_state[i].elems);
}
gpr_free(g_thread_state);
gpr_tls_destroy(&g_this_thread_state);
@ -151,14 +150,14 @@ static void executor_thread(void *arg) {
grpc_exec_ctx exec_ctx =
GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL);
GRPC_STATS_INC_EXECUTOR_THREADS_CREATED(&exec_ctx);
bool used = false;
size_t subtract_depth = 0;
for (;;) {
if (GRPC_TRACER_ON(executor_trace)) {
gpr_log(GPR_DEBUG, "EXECUTOR[%d]: step", (int)(ts - g_thread_state));
gpr_log(GPR_DEBUG, "EXECUTOR[%d]: step (sub_depth=%" PRIdPTR ")",
(int)(ts - g_thread_state), subtract_depth);
}
gpr_mu_lock(&ts->mu);
ts->depth -= subtract_depth;
while (grpc_closure_list_empty(ts->elems) && !ts->shutdown) {
ts->queued_long_job = false;
gpr_cv_wait(&ts->cv, &ts->mu, gpr_inf_future(GPR_CLOCK_REALTIME));
@ -171,20 +170,15 @@ static void executor_thread(void *arg) {
gpr_mu_unlock(&ts->mu);
break;
}
if (!used) {
GRPC_STATS_INC_EXECUTOR_THREADS_USED(&exec_ctx);
used = true;
}
GRPC_STATS_INC_EXECUTOR_QUEUE_DRAINED(&exec_ctx);
GPR_ASSERT(grpc_closure_list_empty(ts->local_elems));
ts->local_elems = ts->elems;
grpc_closure_list exec = ts->elems;
ts->elems = (grpc_closure_list)GRPC_CLOSURE_LIST_INIT;
gpr_mu_unlock(&ts->mu);
if (GRPC_TRACER_ON(executor_trace)) {
gpr_log(GPR_DEBUG, "EXECUTOR[%d]: execute", (int)(ts - g_thread_state));
}
run_closures(&exec_ctx, &ts->local_elems);
subtract_depth = run_closures(&exec_ctx, exec);
}
grpc_exec_ctx_finish(&exec_ctx);
}
@ -217,10 +211,6 @@ static void executor_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
ts = &g_thread_state[GPR_HASH_POINTER(exec_ctx, cur_thread_count)];
} else {
GRPC_STATS_INC_EXECUTOR_SCHEDULED_TO_SELF(exec_ctx);
if (is_short) {
grpc_closure_list_append(&ts->local_elems, closure, error);
return;
}
}
thread_state *orig_ts = ts;
@ -260,7 +250,8 @@ static void executor_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure,
gpr_cv_signal(&ts->cv);
}
grpc_closure_list_append(&ts->elems, closure, error);
try_new_thread = ts->elems.head != closure &&
ts->depth++;
try_new_thread = ts->depth > MAX_DEPTH &&
cur_thread_count < g_max_threads && !ts->shutdown;
if (!is_short) ts->queued_long_job = true;
gpr_mu_unlock(&ts->mu);

@ -87,6 +87,7 @@ static bool composite_call_get_request_metadata(
ctx->on_request_metadata = on_request_metadata;
GRPC_CLOSURE_INIT(&ctx->internal_on_request_metadata,
composite_call_metadata_cb, ctx, grpc_schedule_on_exec_ctx);
bool synchronous = true;
while (ctx->creds_index < ctx->composite_creds->inner.num_creds) {
grpc_call_credentials *inner_creds =
ctx->composite_creds->inner.creds_array[ctx->creds_index++];
@ -95,19 +96,12 @@ static bool composite_call_get_request_metadata(
ctx->md_array, &ctx->internal_on_request_metadata, error)) {
if (*error != GRPC_ERROR_NONE) break;
} else {
synchronous = false; // Async return.
break;
}
}
// If we got through all creds synchronously or we got a synchronous
// error on one of them, return synchronously.
if (ctx->creds_index == ctx->composite_creds->inner.num_creds ||
*error != GRPC_ERROR_NONE) {
gpr_free(ctx);
return true;
}
// At least one inner cred is returning asynchronously, so we'll
// return asynchronously as well.
return false;
if (synchronous) gpr_free(ctx);
return synchronous;
}
static void composite_call_cancel_get_request_metadata(

@ -86,6 +86,10 @@ void ChannelArguments::SetCompressionAlgorithm(
SetInt(GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM, algorithm);
}
void ChannelArguments::SetGrpclbFallbackTimeout(int fallback_timeout) {
SetInt(GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS, fallback_timeout);
}
void ChannelArguments::SetSocketMutator(grpc_socket_mutator* mutator) {
if (!mutator) {
return;

@ -94,7 +94,7 @@ def diagnose_attribute_error(build_ext, error):
_ERROR_DIAGNOSES = {
errors.CompileError: diagnose_compile_error,
AttributeError: diagnose_attribute_error
AttributeError: diagnose_attribute_error,
}
@ -102,8 +102,10 @@ def diagnose_build_ext_error(build_ext, error, formatted):
diagnostic = _ERROR_DIAGNOSES.get(type(error))
if diagnostic is None:
raise commands.CommandError(
"\n\nWe could not diagnose your build failure. Please file an issue at "
"http://www.github.com/grpc/grpc with `[Python install]` in the title."
"\n\n{}".format(formatted))
"\n\nWe could not diagnose your build failure. If you are unable to "
"proceed, please file an issue at http://www.github.com/grpc/grpc "
"with `[Python install]` in the title; please attach the whole log "
"(including everything that may have appeared above the Python "
"backtrace).\n\n{}".format(formatted))
else:
diagnostic(build_ext, error)

@ -19,10 +19,17 @@ require 'google/rpc/status_pb'
module GRPC
# GoogleRpcStatusUtils provides utilities to convert between a
# GRPC::Core::Status and a deserialized Google::Rpc::Status proto
# Returns nil if the grpc-status-details-bin trailer could not be
# converted to a GoogleRpcStatus due to the server not providing
# the necessary trailers.
# Raises an error if the server did provide the necessary trailers
# but they fail to deseriliaze into a GoogleRpcStatus protobuf.
class GoogleRpcStatusUtils
def self.extract_google_rpc_status(status)
fail ArgumentError, 'bad type' unless status.is_a? Struct::Status
Google::Rpc::Status.decode(status.metadata['grpc-status-details-bin'])
grpc_status_details_bin_trailer = 'grpc-status-details-bin'
return nil if status.metadata[grpc_status_details_bin_trailer].nil?
Google::Rpc::Status.decode(status.metadata[grpc_status_details_bin_trailer])
end
end
end

@ -31,12 +31,11 @@ describe 'conversion from a status struct to a google protobuf status' do
expect(exception.message.include?('bad type')).to be true
end
it 'fails with some error if the header key is missing' do
it 'returns nil if the header key is missing' do
status = Struct::Status.new(1, 'details', key: 'val')
expect(status.metadata.nil?).to be false
expect do
GRPC::GoogleRpcStatusUtils.extract_google_rpc_status(status)
end.to raise_error(StandardError)
expect(GRPC::GoogleRpcStatusUtils.extract_google_rpc_status(
status)).to be(nil)
end
it 'fails with some error if the header key fails to deserialize' do
@ -221,3 +220,73 @@ describe 'receving a google rpc status from a remote endpoint' do
status_from_exception)).to eq(rpc_status)
end
end
# A test service that fails without explicitly setting the
# grpc-status-details-bin trailer. Tests assumptions about value
# of grpc-status-details-bin on the client side when the trailer wasn't
# set explicitly.
class NoStatusDetailsBinTestService
include GRPC::GenericService
rpc :an_rpc, EchoMsg, EchoMsg
def an_rpc(_, _)
fail GRPC::Unknown
end
end
NoStatusDetailsBinTestServiceStub = NoStatusDetailsBinTestService.rpc_stub_class
describe 'when the endpoint doesnt send grpc-status-details-bin' do
def start_server
@srv = GRPC::RpcServer.new(pool_size: 1)
@server_port = @srv.add_http2_port('localhost:0',
:this_port_is_insecure)
@srv.handle(NoStatusDetailsBinTestService)
@server_thd = Thread.new { @srv.run }
@srv.wait_till_running
end
def stop_server
expect(@srv.stopped?).to be(false)
@srv.stop
@server_thd.join
expect(@srv.stopped?).to be(true)
end
before(:each) do
start_server
end
after(:each) do
stop_server
end
it 'should receive nil when we extract try to extract a google '\
'rpc status from a BadStatus exception that didnt have it' do
stub = NoStatusDetailsBinTestServiceStub.new("localhost:#{@server_port}",
:this_channel_is_insecure)
begin
stub.an_rpc(EchoMsg.new)
rescue GRPC::Unknown => e
rpc_status = GRPC::GoogleRpcStatusUtils.extract_google_rpc_status(
e.to_status)
end
expect(rpc_status).to be(nil)
end
it 'should receive nil when we extract try to extract a google '\
'rpc status from an op views status object that didnt have it' do
stub = NoStatusDetailsBinTestServiceStub.new("localhost:#{@server_port}",
:this_channel_is_insecure)
op = stub.an_rpc(EchoMsg.new, return_op: true)
begin
op.execute
rescue GRPC::Unknown => e
status_from_exception = e.to_status
end
expect(GRPC::GoogleRpcStatusUtils.extract_google_rpc_status(
status_from_exception)).to be(nil)
expect(GRPC::GoogleRpcStatusUtils.extract_google_rpc_status(
op.status)).to be nil
end
end

@ -0,0 +1,4 @@
%YAML 1.2
--- |
<%namespace file="create_private_dns_zone_defs.include" import="*"/>\
${create_private_dns_zone(resolver_gce_integration_tests_zone_id, resolver_tests_common_zone_name)}

@ -0,0 +1,32 @@
<%def name="create_private_dns_zone(resolver_gce_integration_tests_zone_id, resolver_tests_common_zone_name)">#!/bin/bash
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This file is auto-generated
set -ex
cd $(dirname $0)/../../..
gcloud alpha dns managed-zones create \\
${resolver_gce_integration_tests_zone_id} \\
--dns-name=${resolver_tests_common_zone_name} \\
--description="GCE-DNS-private-zone-for-GRPC-testing" \\
--visibility=private \\
--networks=default</%def>

@ -0,0 +1,4 @@
%YAML 1.2
--- |
<%namespace file="private_dns_zone_init_defs.include" import="*"/>\
${private_dns_zone_init(all_integration_test_records, resolver_gce_integration_tests_zone_id, resolver_tests_common_zone_name)}

@ -0,0 +1,40 @@
<%def name="private_dns_zone_init(records,resolver_gce_integration_tests_zone_id,resolver_tests_common_zone_name)">#!/bin/bash
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This file is auto-generated
set -ex
cd $(dirname $0)/../../..
gcloud dns record-sets transaction start -z=${resolver_gce_integration_tests_zone_id}
% for r in records:
gcloud dns record-sets transaction add \\
-z=${resolver_gce_integration_tests_zone_id} \\
--name=${r['name']}.${resolver_tests_common_zone_name} \\
--type=${r['type']} \\
--ttl=${r['ttl']} \\
${r['data']}
% endfor
gcloud dns record-sets transaction describe -z=${resolver_gce_integration_tests_zone_id}
gcloud dns record-sets transaction execute -z=${resolver_gce_integration_tests_zone_id}
gcloud dns record-sets list -z=${resolver_gce_integration_tests_zone_id}</%def>

@ -0,0 +1,64 @@
<%def name="resolver_gce_integration_tests(tests, records, resolver_tests_common_zone_name)">#!/bin/bash
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This file is auto-generated
set -ex
if [[ "$GRPC_DNS_RESOLVER" == "" ]]; then
export GRPC_DNS_RESOLVER=ares
elif [[ "$GRPC_DNS_RESOLVER" != ares ]]; then
echo "Unexpected: GRPC_DNS_RESOLVER=$GRPC_DNS_RESOLVER. This test only works with c-ares resolver"
exit 1
fi
cd $(dirname $0)/../../..
if [[ "$CONFIG" == "" ]]; then
export CONFIG=opt
fi
make resolver_component_test
echo "Sanity check DNS records are resolveable with dig:"
EXIT_CODE=0
% for r in records:
ONE_FAILED=0
dig ${r['type']} ${r['name']}.${resolver_tests_common_zone_name} | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig ${r['type']} ${r['name']}.${resolver_tests_common_zone_name} FAILED"
exit 1
fi
% endfor
echo "Sanity check PASSED. Run resolver tests:"
% for test in tests:
ONE_FAILED=0
bins/$CONFIG/resolver_component_test \\
--target_name='${test['target_name']}' \\
--expected_addrs='${test['expected_addrs']}' \\
--expected_chosen_service_config='${test['expected_chosen_service_config']}' \\
--expected_lb_policy='${test['expected_lb_policy']}' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Test based on target record: ${test['target_name']} FAILED"
EXIT_CODE=1
fi
% endfor
exit $EXIT_CODE</%def>

@ -0,0 +1,4 @@
%YAML 1.2
--- |
<%namespace file="resolver_gce_integration_tests_defs.include" import="*"/>\
${resolver_gce_integration_tests(resolver_gce_integration_test_cases, all_integration_test_records, resolver_tests_common_zone_name)}

@ -134,9 +134,12 @@ void grpc_run_bad_client_test(
grpc_endpoint_write(&exec_ctx, sfd.client, &outgoing, &done_write_closure);
grpc_exec_ctx_finish(&exec_ctx);
/* Await completion */
GPR_ASSERT(
gpr_event_wait(&a.done_write, grpc_timeout_seconds_to_deadline(5)));
/* Await completion, unless the request is large and write may not finish
* before the peer shuts down. */
if (!(flags & GRPC_BAD_CLIENT_LARGE_REQUEST)) {
GPR_ASSERT(
gpr_event_wait(&a.done_write, grpc_timeout_seconds_to_deadline(5)));
}
if (flags & GRPC_BAD_CLIENT_DISCONNECT) {
grpc_endpoint_shutdown(
@ -186,6 +189,8 @@ void grpc_run_bad_client_test(
grpc_exec_ctx_finish(&exec_ctx);
}
GPR_ASSERT(
gpr_event_wait(&a.done_write, grpc_timeout_seconds_to_deadline(1)));
shutdown_cq = grpc_completion_queue_create_for_pluck(NULL);
grpc_server_shutdown_and_notify(a.server, shutdown_cq, NULL);
GPR_ASSERT(grpc_completion_queue_pluck(

@ -37,6 +37,7 @@ typedef bool (*grpc_bad_client_client_stream_validator)(
grpc_slice_buffer *incoming);
#define GRPC_BAD_CLIENT_DISCONNECT 1
#define GRPC_BAD_CLIENT_LARGE_REQUEST 2
/* Test runner.

@ -90,7 +90,8 @@ int main(int argc, char **argv) {
addbuf(message, sizeof(message));
}
}
grpc_run_bad_client_test(verifier, NULL, g_buffer, g_count, 0);
grpc_run_bad_client_test(verifier, NULL, g_buffer, g_count,
GRPC_BAD_CLIENT_LARGE_REQUEST);
gpr_free(g_buffer);
return 0;

@ -24,7 +24,6 @@
#include <string.h>
#include <unistd.h>
#include <grpc/grpc.h>
#include <grpc/support/alloc.h>
#include <grpc/support/log.h>
#include <grpc/support/useful.h>
@ -434,7 +433,8 @@ int main(int argc, char **argv) {
const char *poll_strategy = grpc_get_poll_strategy_name();
grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
grpc_test_init(argc, argv);
grpc_init();
grpc_iomgr_init(&exec_ctx);
grpc_iomgr_start(&exec_ctx);
if (poll_strategy != NULL &&
(strcmp(poll_strategy, "epoll") == 0 ||
@ -449,8 +449,8 @@ int main(int argc, char **argv) {
poll_strategy);
}
grpc_iomgr_shutdown(&exec_ctx);
grpc_exec_ctx_finish(&exec_ctx);
grpc_shutdown();
return 0;
}
#else /* defined(GRPC_LINUX_EPOLL) */

@ -23,9 +23,26 @@
#include <grpc/grpc.h>
#include <grpc/support/alloc.h>
#include <grpc/support/log.h>
#include <grpc/support/thd.h>
#include "src/core/lib/security/transport/tsi_error.h"
#include "test/core/tsi/transport_security_test_lib.h"
static void notification_signal(tsi_test_fixture *fixture) {
gpr_mu_lock(&fixture->mu);
fixture->notified = true;
gpr_cv_signal(&fixture->cv);
gpr_mu_unlock(&fixture->mu);
}
static void notification_wait(tsi_test_fixture *fixture) {
gpr_mu_lock(&fixture->mu);
while (!fixture->notified) {
gpr_cv_wait(&fixture->cv, &fixture->mu, gpr_inf_future(GPR_CLOCK_REALTIME));
}
fixture->notified = false;
gpr_mu_unlock(&fixture->mu);
}
typedef struct handshaker_args {
tsi_test_fixture *fixture;
unsigned char *handshake_buffer;
@ -273,9 +290,11 @@ grpc_error *on_handshake_next_done(tsi_result result, void *user_data,
/* Read more data if we need to. */
if (result == TSI_INCOMPLETE_DATA) {
GPR_ASSERT(bytes_to_send_size == 0);
notification_signal(fixture);
return error;
}
if (result != TSI_OK) {
notification_signal(fixture);
return grpc_set_tsi_error_result(
GRPC_ERROR_CREATE_FROM_STATIC_STRING("Handshake failed"), result);
}
@ -295,6 +314,7 @@ grpc_error *on_handshake_next_done(tsi_result result, void *user_data,
if (handshaker_result != NULL) {
maybe_append_unused_bytes(args);
}
notification_signal(fixture);
return error;
}
@ -345,7 +365,11 @@ static void do_handshaker_next(handshaker_args *args) {
if (result != TSI_ASYNC) {
args->error = on_handshake_next_done(result, args, bytes_to_send,
bytes_to_send_size, handshaker_result);
if (args->error != GRPC_ERROR_NONE) {
return;
}
}
notification_wait(fixture);
}
void tsi_test_do_handshake(tsi_test_fixture *fixture) {
@ -532,6 +556,9 @@ void tsi_test_fixture_init(tsi_test_fixture *fixture) {
fixture->bytes_read_from_server_channel = 0;
fixture->test_unused_bytes = true;
fixture->has_client_finished_first = false;
gpr_mu_init(&fixture->mu);
gpr_cv_init(&fixture->cv);
fixture->notified = false;
}
void tsi_test_fixture_destroy(tsi_test_fixture *fixture) {
@ -546,5 +573,7 @@ void tsi_test_fixture_destroy(tsi_test_fixture *fixture) {
GPR_ASSERT(fixture->vtable != NULL);
GPR_ASSERT(fixture->vtable->destruct != NULL);
fixture->vtable->destruct(fixture);
gpr_mu_destroy(&fixture->mu);
gpr_cv_destroy(&fixture->cv);
gpr_free(fixture);
}

@ -21,6 +21,10 @@
#include "src/core/tsi/transport_security_interface.h"
#ifdef __cplusplus
extern "C" {
#endif
#define TSI_TEST_TINY_HANDSHAKE_BUFFER_SIZE 32
#define TSI_TEST_SMALL_HANDSHAKE_BUFFER_SIZE 128
#define TSI_TEST_SMALL_READ_BUFFER_ALLOCATED_SIZE 41
@ -56,10 +60,10 @@ typedef struct tsi_test_fixture_vtable {
void (*setup_handshakers)(tsi_test_fixture *fixture);
void (*check_handshaker_peers)(tsi_test_fixture *fixture);
void (*destruct)(tsi_test_fixture *fixture);
} tranport_security_test_vtable;
} tsi_test_fixture_vtable;
struct tsi_test_fixture {
const struct tsi_test_fixture_vtable *vtable;
const tsi_test_fixture_vtable *vtable;
/* client/server TSI handshaker used to perform TSI handshakes, and will get
instantiated during the call to setup_handshakers. */
tsi_handshaker *client_handshaker;
@ -95,6 +99,13 @@ struct tsi_test_fixture {
(https://github.com/grpc/grpc/issues/12164).
*/
bool test_unused_bytes;
/* These objects will be used coordinate client/server handshakers with TSI
thread to perform TSI handshakes in an asynchronous manner (for GTS TSI
implementations).
*/
gpr_cv cv;
gpr_mu mu;
bool notified;
};
struct tsi_test_frame_protector_config {
@ -162,4 +173,8 @@ void tsi_test_do_handshake(tsi_test_fixture *fixture);
the client and server switching its role. */
void tsi_test_do_round_trip(tsi_test_fixture *fixture);
#ifdef __cplusplus
}
#endif
#endif // GRPC_TEST_CORE_TSI_TRANSPORT_SECURITY_TEST_LIB_H_

@ -368,8 +368,9 @@ class GrpclbEnd2endTest : public ::testing::Test {
grpc_fake_resolver_response_generator_unref(response_generator_);
}
void ResetStub() {
void ResetStub(int fallback_timeout = 0) {
ChannelArguments args;
args.SetGrpclbFallbackTimeout(fallback_timeout);
args.SetPointer(GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR,
response_generator_);
std::ostringstream uri;
@ -470,10 +471,10 @@ class GrpclbEnd2endTest : public ::testing::Test {
grpc_exec_ctx_finish(&exec_ctx);
}
const std::vector<int> GetBackendPorts() const {
const std::vector<int> GetBackendPorts(const size_t start_index = 0) const {
std::vector<int> backend_ports;
for (const auto& bs : backend_servers_) {
backend_ports.push_back(bs.port_);
for (size_t i = start_index; i < backend_servers_.size(); ++i) {
backend_ports.push_back(backend_servers_[i].port_);
}
return backend_ports;
}
@ -642,6 +643,177 @@ TEST_F(SingleBalancerTest, InitiallyEmptyServerlist) {
EXPECT_EQ("grpclb", channel_->GetLoadBalancingPolicyName());
}
TEST_F(SingleBalancerTest, Fallback) {
const int kFallbackTimeoutMs = 200 * grpc_test_slowdown_factor();
const int kServerlistDelayMs = 500 * grpc_test_slowdown_factor();
const size_t kNumBackendInResolution = backends_.size() / 2;
ResetStub(kFallbackTimeoutMs);
std::vector<AddressData> addresses;
addresses.emplace_back(AddressData{balancer_servers_[0].port_, true, ""});
for (size_t i = 0; i < kNumBackendInResolution; ++i) {
addresses.emplace_back(AddressData{backend_servers_[i].port_, false, ""});
}
SetNextResolution(addresses);
// Send non-empty serverlist only after kServerlistDelayMs.
ScheduleResponseForBalancer(
0, BalancerServiceImpl::BuildResponseForBackends(
GetBackendPorts(kNumBackendInResolution /* start_index */), {}),
kServerlistDelayMs);
// Wait until all the fallback backends are reachable.
for (size_t i = 0; i < kNumBackendInResolution; ++i) {
WaitForBackend(i);
}
// The first request.
gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH ==========");
CheckRpcSendOk(kNumBackendInResolution);
gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH ==========");
// Fallback is used: each backend returned by the resolver should have
// gotten one request.
for (size_t i = 0; i < kNumBackendInResolution; ++i) {
EXPECT_EQ(1U, backend_servers_[i].service_->request_count());
}
for (size_t i = kNumBackendInResolution; i < backends_.size(); ++i) {
EXPECT_EQ(0U, backend_servers_[i].service_->request_count());
}
// Wait until the serverlist reception has been processed and all backends
// in the serverlist are reachable.
for (size_t i = kNumBackendInResolution; i < backends_.size(); ++i) {
WaitForBackend(i);
}
// Send out the second request.
gpr_log(GPR_INFO, "========= BEFORE SECOND BATCH ==========");
CheckRpcSendOk(backends_.size() - kNumBackendInResolution);
gpr_log(GPR_INFO, "========= DONE WITH SECOND BATCH ==========");
// Serverlist is used: each backend returned by the balancer should
// have gotten one request.
for (size_t i = 0; i < kNumBackendInResolution; ++i) {
EXPECT_EQ(0U, backend_servers_[i].service_->request_count());
}
for (size_t i = kNumBackendInResolution; i < backends_.size(); ++i) {
EXPECT_EQ(1U, backend_servers_[i].service_->request_count());
}
balancers_[0]->NotifyDoneWithServerlists();
// The balancer got a single request.
EXPECT_EQ(1U, balancer_servers_[0].service_->request_count());
// and sent a single response.
EXPECT_EQ(1U, balancer_servers_[0].service_->response_count());
}
TEST_F(SingleBalancerTest, FallbackUpdate) {
const int kFallbackTimeoutMs = 200 * grpc_test_slowdown_factor();
const int kServerlistDelayMs = 500 * grpc_test_slowdown_factor();
const size_t kNumBackendInResolution = backends_.size() / 3;
const size_t kNumBackendInResolutionUpdate = backends_.size() / 3;
ResetStub(kFallbackTimeoutMs);
std::vector<AddressData> addresses;
addresses.emplace_back(AddressData{balancer_servers_[0].port_, true, ""});
for (size_t i = 0; i < kNumBackendInResolution; ++i) {
addresses.emplace_back(AddressData{backend_servers_[i].port_, false, ""});
}
SetNextResolution(addresses);
// Send non-empty serverlist only after kServerlistDelayMs.
ScheduleResponseForBalancer(
0, BalancerServiceImpl::BuildResponseForBackends(
GetBackendPorts(kNumBackendInResolution +
kNumBackendInResolutionUpdate /* start_index */),
{}),
kServerlistDelayMs);
// Wait until all the fallback backends are reachable.
for (size_t i = 0; i < kNumBackendInResolution; ++i) {
WaitForBackend(i);
}
// The first request.
gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH ==========");
CheckRpcSendOk(kNumBackendInResolution);
gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH ==========");
// Fallback is used: each backend returned by the resolver should have
// gotten one request.
for (size_t i = 0; i < kNumBackendInResolution; ++i) {
EXPECT_EQ(1U, backend_servers_[i].service_->request_count());
}
for (size_t i = kNumBackendInResolution; i < backends_.size(); ++i) {
EXPECT_EQ(0U, backend_servers_[i].service_->request_count());
}
addresses.clear();
addresses.emplace_back(AddressData{balancer_servers_[0].port_, true, ""});
for (size_t i = kNumBackendInResolution;
i < kNumBackendInResolution + kNumBackendInResolutionUpdate; ++i) {
addresses.emplace_back(AddressData{backend_servers_[i].port_, false, ""});
}
SetNextResolution(addresses);
// Wait until the resolution update has been processed and all the new
// fallback backends are reachable.
for (size_t i = kNumBackendInResolution;
i < kNumBackendInResolution + kNumBackendInResolutionUpdate; ++i) {
WaitForBackend(i);
}
// Send out the second request.
gpr_log(GPR_INFO, "========= BEFORE SECOND BATCH ==========");
CheckRpcSendOk(kNumBackendInResolutionUpdate);
gpr_log(GPR_INFO, "========= DONE WITH SECOND BATCH ==========");
// The resolution update is used: each backend in the resolution update should
// have gotten one request.
for (size_t i = 0; i < kNumBackendInResolution; ++i) {
EXPECT_EQ(0U, backend_servers_[i].service_->request_count());
}
for (size_t i = kNumBackendInResolution;
i < kNumBackendInResolution + kNumBackendInResolutionUpdate; ++i) {
EXPECT_EQ(1U, backend_servers_[i].service_->request_count());
}
for (size_t i = kNumBackendInResolution + kNumBackendInResolutionUpdate;
i < backends_.size(); ++i) {
EXPECT_EQ(0U, backend_servers_[i].service_->request_count());
}
// Wait until the serverlist reception has been processed and all backends
// in the serverlist are reachable.
for (size_t i = kNumBackendInResolution + kNumBackendInResolutionUpdate;
i < backends_.size(); ++i) {
WaitForBackend(i);
}
// Send out the third request.
gpr_log(GPR_INFO, "========= BEFORE THIRD BATCH ==========");
CheckRpcSendOk(backends_.size() - kNumBackendInResolution -
kNumBackendInResolutionUpdate);
gpr_log(GPR_INFO, "========= DONE WITH THIRD BATCH ==========");
// Serverlist is used: each backend returned by the balancer should
// have gotten one request.
for (size_t i = 0;
i < kNumBackendInResolution + kNumBackendInResolutionUpdate; ++i) {
EXPECT_EQ(0U, backend_servers_[i].service_->request_count());
}
for (size_t i = kNumBackendInResolution + kNumBackendInResolutionUpdate;
i < backends_.size(); ++i) {
EXPECT_EQ(1U, backend_servers_[i].service_->request_count());
}
balancers_[0]->NotifyDoneWithServerlists();
// The balancer got a single request.
EXPECT_EQ(1U, balancer_servers_[0].service_->request_count());
// and sent a single response.
EXPECT_EQ(1U, balancer_servers_[0].service_->response_count());
}
TEST_F(SingleBalancerTest, BackendsRestart) {
const size_t kNumRpcsPerAddress = 100;
ScheduleResponseForBalancer(

@ -0,0 +1,43 @@
# Resolver Tests
This directory has tests and infrastructure for unit tests and GCE
integration tests of gRPC resolver functionality.
There are two different tests here:
## Resolver unit tests (resolver "component" tests)
These tests run per-change, along with the rest of the grpc unit tests.
They query a local testing DNS server.
## GCE integration tests
These tests use the same test binary and the same test records
as the unit tests, but they run against GCE DNS (this is done by
running the test on a GCE instance and not specifying an authority
in uris). These tests run in a background job, which needs to be
actively monitored.
## Making changes to test records
After making a change to `resolver_test_record_groups.yaml`:
1. Increment the "version number" in the `resolver_tests_common_zone_name`
DNS zone (this is a yaml field at the top
of `resolver_test_record_groups.yaml`).
2. Regenerate projects.
3. From the repo root, run:
```
$ test/cpp/naming/create_dns_private_zone.sh
$ test/cpp/naming/private_dns_zone_init.sh
```
Note that these commands must be ran in environment that
has access to the grpc-testing GCE project.
If everything runs smoothly, then once the change is merged,
the GCE DNS integration testing job will transition to the
new records and continue passing.

@ -0,0 +1,27 @@
#!/bin/bash
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This file is auto-generated
set -ex
cd $(dirname $0)/../../..
gcloud alpha dns managed-zones create \
resolver-tests-version-1-grpctestingexp-zone-id \
--dns-name=resolver-tests-version-1.grpctestingexp. \
--description="GCE-DNS-private-zone-for-GRPC-testing" \
--visibility=private \
--networks=default

@ -24,6 +24,12 @@ import json
_LOCAL_DNS_SERVER_ADDRESS = '127.0.0.1:15353'
_TARGET_RECORDS_TO_SKIP_AGAINST_GCE = [
# TODO: enable this once able to upload the very large TXT record
# in this group to GCE DNS.
'ipv4-config-causing-fallback-to-tcp',
]
def _append_zone_name(name, zone_name):
return '%s.%s' % (name, zone_name)
@ -33,21 +39,107 @@ def _build_expected_addrs_cmd_arg(expected_addrs):
out.append('%s,%s' % (addr['address'], str(addr['is_balancer'])))
return ';'.join(out)
def _data_for_type(r_type, r_data, common_zone_name):
if r_type in ['A', 'AAAA']:
return ' '.join(map(lambda x: '\"%s\"' % x, r_data))
if r_type == 'SRV':
assert len(r_data) == 1
target = r_data[0].split(' ')[3]
uploadable_target = '%s.%s' % (target, common_zone_name)
uploadable = r_data[0].split(' ')
uploadable[3] = uploadable_target
return '\"%s\"' % ' '.join(uploadable)
if r_type == 'TXT':
assert len(r_data) == 1
chunks = []
all_data = r_data[0]
cur = 0
# Split TXT records that span more than 255 characters (the single
# string length-limit in DNS) into multiple strings. Each string
# needs to be wrapped with double-quotes, and all inner double-quotes
# are escaped. The wrapping double-quotes and inner backslashes can be
# counted towards the 255 character length limit (as observed with gcloud),
# so make sure all strings fit within that limit.
while len(all_data[cur:]) > 0:
next_chunk = '\"'
while len(next_chunk) < 254 and len(all_data[cur:]) > 0:
if all_data[cur] == '\"':
if len(next_chunk) < 253:
next_chunk += '\\\"'
else:
break
else:
next_chunk += all_data[cur]
cur += 1
next_chunk += '\"'
if len(next_chunk) > 255:
raise Exception('Bug: next chunk is too long.')
chunks.append(next_chunk)
# Wrap the whole record in single quotes to make sure all strings
# are associated with the same TXT record (to make it one bash token for
# gcloud)
return '\'%s\'' % ' '.join(chunks)
# Convert DNS records from their "within a test group" format
# of the yaml file to an easier form for the templates to use.
def _gcloud_uploadable_form(test_cases, common_zone_name):
out = []
for group in test_cases:
if group['record_to_resolve'] in _TARGET_RECORDS_TO_SKIP_AGAINST_GCE:
continue
for record_name in group['records'].keys():
r_ttl = None
all_r_data = {}
for r_data in group['records'][record_name]:
# enforce records have the same TTL only for simplicity
if r_ttl is None:
r_ttl = r_data['TTL']
assert r_ttl == r_data['TTL'], '%s and %s differ' % (r_ttl, r_data['TTL'])
r_type = r_data['type']
if all_r_data.get(r_type) is None:
all_r_data[r_type] = []
all_r_data[r_type].append(r_data['data'])
for r_type in all_r_data.keys():
for r in out:
assert r['name'] != record_name or r['type'] != r_type, 'attempt to add a duplicate record'
out.append({
'name': record_name,
'ttl': r_ttl,
'type': r_type,
'data': _data_for_type(r_type, all_r_data[r_type], common_zone_name)
})
return out
def _gce_dns_zone_id(resolver_component_data):
dns_name = resolver_component_data['resolver_tests_common_zone_name']
return dns_name.replace('.', '-') + 'zone-id'
def _resolver_test_cases(resolver_component_data, records_to_skip):
out = []
for test_case in resolver_component_data['resolver_component_tests']:
if test_case['record_to_resolve'] in records_to_skip:
continue
out.append({
'target_name': _append_zone_name(test_case['record_to_resolve'],
resolver_component_data['resolver_tests_common_zone_name']),
'expected_addrs': _build_expected_addrs_cmd_arg(test_case['expected_addrs']),
'expected_chosen_service_config': (test_case['expected_chosen_service_config'] or ''),
'expected_lb_policy': (test_case['expected_lb_policy'] or ''),
})
return out
def main():
resolver_component_data = ''
with open('test/cpp/naming/resolver_test_record_groups.yaml') as f:
resolver_component_data = yaml.load(f)
json = {
'resolver_component_test_cases': [
{
'target_name': _append_zone_name(test_case['record_to_resolve'],
resolver_component_data['resolver_component_tests_common_zone_name']),
'expected_addrs': _build_expected_addrs_cmd_arg(test_case['expected_addrs']),
'expected_chosen_service_config': (test_case['expected_chosen_service_config'] or ''),
'expected_lb_policy': (test_case['expected_lb_policy'] or ''),
} for test_case in resolver_component_data['resolver_component_tests']
],
'resolver_tests_common_zone_name': resolver_component_data['resolver_tests_common_zone_name'],
'resolver_gce_integration_tests_zone_id': _gce_dns_zone_id(resolver_component_data),
'all_integration_test_records': _gcloud_uploadable_form(resolver_component_data['resolver_component_tests'],
resolver_component_data['resolver_tests_common_zone_name']),
'resolver_gce_integration_test_cases': _resolver_test_cases(resolver_component_data, _TARGET_RECORDS_TO_SKIP_AGAINST_GCE),
'resolver_component_test_cases': _resolver_test_cases(resolver_component_data, []),
'targets': [
{
'name': 'resolver_component_test' + unsecure_build_config_suffix,

@ -0,0 +1,215 @@
#!/bin/bash
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This file is auto-generated
set -ex
cd $(dirname $0)/../../..
gcloud dns record-sets transaction start -z=resolver-tests-version-1-grpctestingexp-zone-id
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=_grpclb._tcp.srv-ipv4-single-target.resolver-tests-version-1.grpctestingexp. \
--type=SRV \
--ttl=2100 \
"0 0 1234 ipv4-single-target.resolver-tests-version-1.grpctestingexp."
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=ipv4-single-target.resolver-tests-version-1.grpctestingexp. \
--type=A \
--ttl=2100 \
"1.2.3.4"
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=_grpclb._tcp.srv-ipv4-multi-target.resolver-tests-version-1.grpctestingexp. \
--type=SRV \
--ttl=2100 \
"0 0 1234 ipv4-multi-target.resolver-tests-version-1.grpctestingexp."
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=ipv4-multi-target.resolver-tests-version-1.grpctestingexp. \
--type=A \
--ttl=2100 \
"1.2.3.5" "1.2.3.6" "1.2.3.7"
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=_grpclb._tcp.srv-ipv6-single-target.resolver-tests-version-1.grpctestingexp. \
--type=SRV \
--ttl=2100 \
"0 0 1234 ipv6-single-target.resolver-tests-version-1.grpctestingexp."
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=ipv6-single-target.resolver-tests-version-1.grpctestingexp. \
--type=AAAA \
--ttl=2100 \
"2607:f8b0:400a:801::1001"
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=_grpclb._tcp.srv-ipv6-multi-target.resolver-tests-version-1.grpctestingexp. \
--type=SRV \
--ttl=2100 \
"0 0 1234 ipv6-multi-target.resolver-tests-version-1.grpctestingexp."
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=ipv6-multi-target.resolver-tests-version-1.grpctestingexp. \
--type=AAAA \
--ttl=2100 \
"2607:f8b0:400a:801::1002" "2607:f8b0:400a:801::1003" "2607:f8b0:400a:801::1004"
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. \
--type=TXT \
--ttl=2100 \
'"grpc_config=[{\"serviceConfig\":{\"loadBalancingPolicy\":\"round_robin\",\"methodConfig\":[{\"name\":[{\"method\":\"Foo\",\"service\":\"SimpleService\",\"waitForReady\":true}]}]}}]"'
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. \
--type=A \
--ttl=2100 \
"1.2.3.4"
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=_grpclb._tcp.srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. \
--type=SRV \
--ttl=2100 \
"0 0 1234 ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp."
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp. \
--type=A \
--ttl=2100 \
"1.2.3.4"
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp. \
--type=TXT \
--ttl=2100 \
'"grpc_config=[{\"serviceConfig\":{\"loadBalancingPolicy\":\"round_robin\",\"methodConfig\":[{\"name\":[{\"method\":\"Foo\",\"service\":\"NoSrvSimpleService\",\"waitForReady\":true}]}]}}]"'
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp. \
--type=A \
--ttl=2100 \
"1.2.3.4"
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp. \
--type=TXT \
--ttl=2100 \
'"grpc_config=[{\"clientLanguage\":[\"python\"],\"serviceConfig\":{\"loadBalancingPolicy\":\"round_robin\",\"methodConfig\":[{\"name\":[{\"method\":\"Foo\",\"service\":\"PythonService\",\"waitForReady\":true}]}]}}]"'
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp. \
--type=A \
--ttl=2100 \
"1.2.3.4"
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp. \
--type=TXT \
--ttl=2100 \
'"grpc_config=[{\"percentage\":0,\"serviceConfig\":{\"loadBalancingPolicy\":\"round_robin\",\"methodConfig\":[{\"name\":[{\"method\":\"Foo\",\"service\":\"CppService\",\"waitForReady\":true}]}]}}]"'
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp. \
--type=A \
--ttl=2100 \
"1.2.3.4"
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp. \
--type=TXT \
--ttl=2100 \
'"grpc_config=[{\"clientLanguage\":[\"go\"],\"serviceConfig\":{\"loadBalancingPolicy\":\"round_robin\",\"methodConfig\":[{\"name\":[{\"method\":\"Foo\",\"service\":\"GoService\",\"waitForReady\":true}]}]}},{\"clientLanguage\":[\"c++\"],\"serviceConfig\":{" "\"loadBalancingPolicy\":\"round_robin\",\"methodConfig\":[{\"name\":[{\"method\":\"Foo\",\"service\":\"CppService\",\"waitForReady\":true}]}]}}]"'
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp. \
--type=A \
--ttl=2100 \
"1.2.3.4"
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp. \
--type=TXT \
--ttl=2100 \
'"grpc_config=[{\"percentage\":0,\"serviceConfig\":{\"loadBalancingPolicy\":\"round_robin\",\"methodConfig\":[{\"name\":[{\"method\":\"Foo\",\"service\":\"NeverPickedService\",\"waitForReady\":true}]}]}},{\"percentage\":100,\"serviceConfig\":{\"loadBalanc" "ingPolicy\":\"round_robin\",\"methodConfig\":[{\"name\":[{\"method\":\"Foo\",\"service\":\"AlwaysPickedService\",\"waitForReady\":true}]}]}}]"'
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=_grpclb._tcp.srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. \
--type=SRV \
--ttl=2100 \
"0 0 1234 balancer-for-ipv4-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp."
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=balancer-for-ipv4-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. \
--type=A \
--ttl=2100 \
"1.2.3.4"
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. \
--type=A \
--ttl=2100 \
"1.2.3.4"
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=_grpclb._tcp.srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. \
--type=SRV \
--ttl=2100 \
"0 0 1234 balancer-for-ipv6-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp."
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=balancer-for-ipv6-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. \
--type=AAAA \
--ttl=2100 \
"2607:f8b0:400a:801::1002"
gcloud dns record-sets transaction add \
-z=resolver-tests-version-1-grpctestingexp-zone-id \
--name=srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. \
--type=AAAA \
--ttl=2100 \
"2607:f8b0:400a:801::1002"
gcloud dns record-sets transaction describe -z=resolver-tests-version-1-grpctestingexp-zone-id
gcloud dns record-sets transaction execute -z=resolver-tests-version-1-grpctestingexp-zone-id
gcloud dns record-sets list -z=resolver-tests-version-1-grpctestingexp-zone-id

@ -73,7 +73,7 @@ EXIT_CODE=0
# in the resolver.
$FLAGS_test_bin_path \
--target_name='srv-ipv4-single-target.resolver-tests.grpctestingexp.' \
--target_name='srv-ipv4-single-target.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:1234,True' \
--expected_chosen_service_config='' \
--expected_lb_policy='' \
@ -81,7 +81,7 @@ $FLAGS_test_bin_path \
wait $! || EXIT_CODE=1
$FLAGS_test_bin_path \
--target_name='srv-ipv4-multi-target.resolver-tests.grpctestingexp.' \
--target_name='srv-ipv4-multi-target.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.5:1234,True;1.2.3.6:1234,True;1.2.3.7:1234,True' \
--expected_chosen_service_config='' \
--expected_lb_policy='' \
@ -89,7 +89,7 @@ $FLAGS_test_bin_path \
wait $! || EXIT_CODE=1
$FLAGS_test_bin_path \
--target_name='srv-ipv6-single-target.resolver-tests.grpctestingexp.' \
--target_name='srv-ipv6-single-target.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='[2607:f8b0:400a:801::1001]:1234,True' \
--expected_chosen_service_config='' \
--expected_lb_policy='' \
@ -97,7 +97,7 @@ $FLAGS_test_bin_path \
wait $! || EXIT_CODE=1
$FLAGS_test_bin_path \
--target_name='srv-ipv6-multi-target.resolver-tests.grpctestingexp.' \
--target_name='srv-ipv6-multi-target.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='[2607:f8b0:400a:801::1002]:1234,True;[2607:f8b0:400a:801::1003]:1234,True;[2607:f8b0:400a:801::1004]:1234,True' \
--expected_chosen_service_config='' \
--expected_lb_policy='' \
@ -105,7 +105,7 @@ $FLAGS_test_bin_path \
wait $! || EXIT_CODE=1
$FLAGS_test_bin_path \
--target_name='srv-ipv4-simple-service-config.resolver-tests.grpctestingexp.' \
--target_name='srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:1234,True' \
--expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]}]}' \
--expected_lb_policy='round_robin' \
@ -113,7 +113,7 @@ $FLAGS_test_bin_path \
wait $! || EXIT_CODE=1
$FLAGS_test_bin_path \
--target_name='ipv4-no-srv-simple-service-config.resolver-tests.grpctestingexp.' \
--target_name='ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:443,False' \
--expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"NoSrvSimpleService","waitForReady":true}]}]}' \
--expected_lb_policy='round_robin' \
@ -121,7 +121,7 @@ $FLAGS_test_bin_path \
wait $! || EXIT_CODE=1
$FLAGS_test_bin_path \
--target_name='ipv4-no-config-for-cpp.resolver-tests.grpctestingexp.' \
--target_name='ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:443,False' \
--expected_chosen_service_config='' \
--expected_lb_policy='' \
@ -129,7 +129,7 @@ $FLAGS_test_bin_path \
wait $! || EXIT_CODE=1
$FLAGS_test_bin_path \
--target_name='ipv4-cpp-config-has-zero-percentage.resolver-tests.grpctestingexp.' \
--target_name='ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:443,False' \
--expected_chosen_service_config='' \
--expected_lb_policy='' \
@ -137,7 +137,7 @@ $FLAGS_test_bin_path \
wait $! || EXIT_CODE=1
$FLAGS_test_bin_path \
--target_name='ipv4-second-language-is-cpp.resolver-tests.grpctestingexp.' \
--target_name='ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:443,False' \
--expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"CppService","waitForReady":true}]}]}' \
--expected_lb_policy='round_robin' \
@ -145,7 +145,7 @@ $FLAGS_test_bin_path \
wait $! || EXIT_CODE=1
$FLAGS_test_bin_path \
--target_name='ipv4-config-with-percentages.resolver-tests.grpctestingexp.' \
--target_name='ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:443,False' \
--expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"AlwaysPickedService","waitForReady":true}]}]}' \
--expected_lb_policy='round_robin' \
@ -153,7 +153,7 @@ $FLAGS_test_bin_path \
wait $! || EXIT_CODE=1
$FLAGS_test_bin_path \
--target_name='srv-ipv4-target-has-backend-and-balancer.resolver-tests.grpctestingexp.' \
--target_name='srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:1234,True;1.2.3.4:443,False' \
--expected_chosen_service_config='' \
--expected_lb_policy='' \
@ -161,7 +161,7 @@ $FLAGS_test_bin_path \
wait $! || EXIT_CODE=1
$FLAGS_test_bin_path \
--target_name='srv-ipv6-target-has-backend-and-balancer.resolver-tests.grpctestingexp.' \
--target_name='srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='[2607:f8b0:400a:801::1002]:1234,True;[2607:f8b0:400a:801::1002]:443,False' \
--expected_chosen_service_config='' \
--expected_lb_policy='' \
@ -169,7 +169,7 @@ $FLAGS_test_bin_path \
wait $! || EXIT_CODE=1
$FLAGS_test_bin_path \
--target_name='ipv4-config-causing-fallback-to-tcp.resolver-tests.grpctestingexp.' \
--target_name='ipv4-config-causing-fallback-to-tcp.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:443,False' \
--expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooThree","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFour","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFive","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSix","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSeven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEight","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooNine","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTen","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEleven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]}]}' \
--expected_lb_policy='' \

@ -0,0 +1,359 @@
#!/bin/bash
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This file is auto-generated
set -ex
if [[ "$GRPC_DNS_RESOLVER" == "" ]]; then
export GRPC_DNS_RESOLVER=ares
elif [[ "$GRPC_DNS_RESOLVER" != ares ]]; then
echo "Unexpected: GRPC_DNS_RESOLVER=$GRPC_DNS_RESOLVER. This test only works with c-ares resolver"
exit 1
fi
cd $(dirname $0)/../../..
if [[ "$CONFIG" == "" ]]; then
export CONFIG=opt
fi
make resolver_component_test
echo "Sanity check DNS records are resolveable with dig:"
EXIT_CODE=0
ONE_FAILED=0
dig SRV _grpclb._tcp.srv-ipv4-single-target.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig SRV _grpclb._tcp.srv-ipv4-single-target.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig A ipv4-single-target.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig A ipv4-single-target.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig SRV _grpclb._tcp.srv-ipv4-multi-target.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig SRV _grpclb._tcp.srv-ipv4-multi-target.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig A ipv4-multi-target.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig A ipv4-multi-target.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig SRV _grpclb._tcp.srv-ipv6-single-target.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig SRV _grpclb._tcp.srv-ipv6-single-target.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig AAAA ipv6-single-target.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig AAAA ipv6-single-target.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig SRV _grpclb._tcp.srv-ipv6-multi-target.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig SRV _grpclb._tcp.srv-ipv6-multi-target.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig AAAA ipv6-multi-target.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig AAAA ipv6-multi-target.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig TXT srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig TXT srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig A ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig A ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig SRV _grpclb._tcp.srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig SRV _grpclb._tcp.srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig A ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig A ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig TXT ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig TXT ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig A ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig A ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig TXT ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig TXT ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig A ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig A ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig TXT ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig TXT ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig A ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig A ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig TXT ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig TXT ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig A ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig A ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig TXT ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig TXT ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig SRV _grpclb._tcp.srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig SRV _grpclb._tcp.srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig A balancer-for-ipv4-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig A balancer-for-ipv4-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig A srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig A srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig SRV _grpclb._tcp.srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig SRV _grpclb._tcp.srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig AAAA balancer-for-ipv6-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig AAAA balancer-for-ipv6-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
ONE_FAILED=0
dig AAAA srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Sanity check: dig AAAA srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. FAILED"
exit 1
fi
echo "Sanity check PASSED. Run resolver tests:"
ONE_FAILED=0
bins/$CONFIG/resolver_component_test \
--target_name='srv-ipv4-single-target.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:1234,True' \
--expected_chosen_service_config='' \
--expected_lb_policy='' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Test based on target record: srv-ipv4-single-target.resolver-tests-version-1.grpctestingexp. FAILED"
EXIT_CODE=1
fi
ONE_FAILED=0
bins/$CONFIG/resolver_component_test \
--target_name='srv-ipv4-multi-target.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.5:1234,True;1.2.3.6:1234,True;1.2.3.7:1234,True' \
--expected_chosen_service_config='' \
--expected_lb_policy='' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Test based on target record: srv-ipv4-multi-target.resolver-tests-version-1.grpctestingexp. FAILED"
EXIT_CODE=1
fi
ONE_FAILED=0
bins/$CONFIG/resolver_component_test \
--target_name='srv-ipv6-single-target.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='[2607:f8b0:400a:801::1001]:1234,True' \
--expected_chosen_service_config='' \
--expected_lb_policy='' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Test based on target record: srv-ipv6-single-target.resolver-tests-version-1.grpctestingexp. FAILED"
EXIT_CODE=1
fi
ONE_FAILED=0
bins/$CONFIG/resolver_component_test \
--target_name='srv-ipv6-multi-target.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='[2607:f8b0:400a:801::1002]:1234,True;[2607:f8b0:400a:801::1003]:1234,True;[2607:f8b0:400a:801::1004]:1234,True' \
--expected_chosen_service_config='' \
--expected_lb_policy='' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Test based on target record: srv-ipv6-multi-target.resolver-tests-version-1.grpctestingexp. FAILED"
EXIT_CODE=1
fi
ONE_FAILED=0
bins/$CONFIG/resolver_component_test \
--target_name='srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:1234,True' \
--expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]}]}' \
--expected_lb_policy='round_robin' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Test based on target record: srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. FAILED"
EXIT_CODE=1
fi
ONE_FAILED=0
bins/$CONFIG/resolver_component_test \
--target_name='ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:443,False' \
--expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"NoSrvSimpleService","waitForReady":true}]}]}' \
--expected_lb_policy='round_robin' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Test based on target record: ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp. FAILED"
EXIT_CODE=1
fi
ONE_FAILED=0
bins/$CONFIG/resolver_component_test \
--target_name='ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:443,False' \
--expected_chosen_service_config='' \
--expected_lb_policy='' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Test based on target record: ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp. FAILED"
EXIT_CODE=1
fi
ONE_FAILED=0
bins/$CONFIG/resolver_component_test \
--target_name='ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:443,False' \
--expected_chosen_service_config='' \
--expected_lb_policy='' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Test based on target record: ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp. FAILED"
EXIT_CODE=1
fi
ONE_FAILED=0
bins/$CONFIG/resolver_component_test \
--target_name='ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:443,False' \
--expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"CppService","waitForReady":true}]}]}' \
--expected_lb_policy='round_robin' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Test based on target record: ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp. FAILED"
EXIT_CODE=1
fi
ONE_FAILED=0
bins/$CONFIG/resolver_component_test \
--target_name='ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:443,False' \
--expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"AlwaysPickedService","waitForReady":true}]}]}' \
--expected_lb_policy='round_robin' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Test based on target record: ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp. FAILED"
EXIT_CODE=1
fi
ONE_FAILED=0
bins/$CONFIG/resolver_component_test \
--target_name='srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='1.2.3.4:1234,True;1.2.3.4:443,False' \
--expected_chosen_service_config='' \
--expected_lb_policy='' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Test based on target record: srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. FAILED"
EXIT_CODE=1
fi
ONE_FAILED=0
bins/$CONFIG/resolver_component_test \
--target_name='srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp.' \
--expected_addrs='[2607:f8b0:400a:801::1002]:1234,True;[2607:f8b0:400a:801::1002]:443,False' \
--expected_chosen_service_config='' \
--expected_lb_policy='' || ONE_FAILED=1
if [[ "$ONE_FAILED" != 0 ]]; then
echo "Test based on target record: srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. FAILED"
EXIT_CODE=1
fi
exit $EXIT_CODE

@ -1,4 +1,4 @@
resolver_component_tests_common_zone_name: resolver-tests.grpctestingexp.
resolver_tests_common_zone_name: resolver-tests-version-1.grpctestingexp.
resolver_component_tests:
- expected_addrs:
- {address: '1.2.3.4:1234', is_balancer: true}

@ -66,7 +66,7 @@ def start_local_dns_server(args):
with open(args.records_config_path) as config:
test_records_config = yaml.load(config)
common_zone_name = test_records_config['resolver_component_tests_common_zone_name']
common_zone_name = test_records_config['resolver_tests_common_zone_name']
for group in test_records_config['resolver_component_tests']:
for name in group['records'].keys():
for record in group['records'][name]:

@ -147,8 +147,7 @@ def gen_bucket_code(histogram):
shift_data = find_ideal_shift(code_bounds[first_nontrivial:], 256 * histogram.buckets)
#print first_nontrivial, shift_data, bounds
#if shift_data is not None: print [hex(x >> shift_data[0]) for x in code_bounds[first_nontrivial:]]
code = '\n/* Automatically generated by tools/codegen/core/gen_stats_data.py */\n'
code += 'value = GPR_CLAMP(value, 0, %d);\n' % histogram.max
code = 'value = GPR_CLAMP(value, 0, %d);\n' % histogram.max
map_table = gen_map_table(code_bounds[first_nontrivial:], shift_data)
if first_nontrivial is None:
code += ('GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_%s, value);\n'
@ -408,3 +407,4 @@ with open('src/core/lib/debug/stats_data_bq_schema.sql', 'w') as S:
for counter in inst_map['Counter']:
columns.append(('%s_per_iteration' % counter.name, 'FLOAT'))
print >>S, ',\n'.join('%s:%s' % x for x in columns)

@ -0,0 +1,163 @@
#!/usr/bin/env python
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Measure the time between PR creation and completion of all tests.
You'll need a github API token to avoid being rate-limited. See
https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/
This script goes over the most recent 100 pull requests. For PRs with a single
commit, it uses the PR's creation as the initial time; othewise, it uses the
date of the last commit. This is somewhat fragile, and imposed by the fact that
GitHub reports a PR's updated timestamp for any event that modifies the PR (e.g.
comments), not just the addition of new commits.
In addition, it ignores latencies greater than five hours, as that's likely due
to a manual re-run of tests.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import json
import logging
import pprint
import urllib2
from datetime import datetime, timedelta
logging.basicConfig(format='%(asctime)s %(message)s')
PRS = 'https://api.github.com/repos/grpc/grpc/pulls?state=open&per_page=100'
COMMITS = 'https://api.github.com/repos/grpc/grpc/pulls/{pr_number}/commits'
def gh(url):
request = urllib2.Request(url)
if TOKEN:
request.add_header('Authorization', 'token {}'.format(TOKEN))
response = urllib2.urlopen(request)
return response.read()
def print_csv_header():
print('pr,base_time,test_time,latency_seconds,successes,failures,errors')
def output(pr, base_time, test_time, diff_time, successes, failures, errors, mode='human'):
if mode == 'human':
print("PR #{} base time: {} UTC, Tests completed at: {} UTC. Latency: {}."
"\n\tSuccesses: {}, Failures: {}, Errors: {}".format(
pr, base_time, test_time, diff_time, successes, failures, errors))
elif mode == 'csv':
print(','.join([str(pr), str(base_time),
str(test_time), str(int((test_time-base_time).total_seconds())),
str(successes), str(failures), str(errors)]))
def parse_timestamp(datetime_str):
return datetime.strptime(datetime_str, '%Y-%m-%dT%H:%M:%SZ')
def to_posix_timestamp(dt):
return str((dt - datetime(1970, 1, 1)).total_seconds())
def get_pr_data():
latest_prs = json.loads(gh(PRS))
res = [{'number': pr['number'],
'created_at': parse_timestamp(pr['created_at']),
'updated_at': parse_timestamp(pr['updated_at']),
'statuses_url': pr['statuses_url']}
for pr in latest_prs]
return res
def get_commits_data(pr_number):
commits = json.loads(gh(COMMITS.format(pr_number=pr_number)))
return {'num_commits': len(commits),
'most_recent_date': parse_timestamp(commits[-1]['commit']['author']['date'])}
def get_status_data(statuses_url, system):
status_url = statuses_url.replace('statuses', 'status')
statuses = json.loads(gh(status_url + '?per_page=100'))
successes = 0
failures = 0
errors = 0
latest_datetime = None
if not statuses: return None
if system == 'kokoro': string_in_target_url = 'kokoro'
elif system == 'jenkins': string_in_target_url = 'grpc-testing'
for status in statuses['statuses']:
if not status['target_url'] or string_in_target_url not in status['target_url']: continue # Ignore jenkins
if status['state'] == 'pending': return None
elif status['state'] == 'success': successes += 1
elif status['state'] == 'failure': failures += 1
elif status['state'] == 'error': errors += 1
if not latest_datetime:
latest_datetime = parse_timestamp(status['updated_at'])
else:
latest_datetime = max(latest_datetime, parse_timestamp(status['updated_at']))
# First status is the most recent one.
if any([successes, failures, errors]) and sum([successes, failures, errors]) > 15:
return {'latest_datetime': latest_datetime,
'successes': successes,
'failures': failures,
'errors': errors}
else: return None
def build_args_parser():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--format', type=str, choices=['human', 'csv'],
default='human',
help='Output format: are you a human or a machine?')
parser.add_argument('--system', type=str, choices=['jenkins', 'kokoro'],
required=True, help='Consider only the given CI system')
parser.add_argument('--token', type=str, default='',
help='GitHub token to use its API with a higher rate limit')
return parser
def main():
import sys
global TOKEN
args_parser = build_args_parser()
args = args_parser.parse_args()
TOKEN = args.token
if args.format == 'csv': print_csv_header()
for pr_data in get_pr_data():
commit_data = get_commits_data(pr_data['number'])
# PR with a single commit -> use the PRs creation time.
# else -> use the latest commit's date.
base_timestamp = pr_data['updated_at']
if commit_data['num_commits'] > 1:
base_timestamp = commit_data['most_recent_date']
else:
base_timestamp = pr_data['created_at']
last_status = get_status_data(pr_data['statuses_url'], args.system)
if last_status:
diff = last_status['latest_datetime'] - base_timestamp
if diff < timedelta(hours=5):
output(pr_data['number'], base_timestamp, last_status['latest_datetime'],
diff, last_status['successes'], last_status['failures'],
last_status['errors'], mode=args.format)
if __name__ == '__main__':
main()

@ -22,6 +22,8 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc
CPUS=`python -c 'import multiprocessing; print multiprocessing.cpu_count()'`
./tools/run_tests/start_port_server.py || true
make CONFIG=opt memory_profile_test memory_profile_client memory_profile_server -j $CPUS
bins/opt/memory_profile_test
bq load microbenchmarks.memory memory_usage.csv

@ -29,4 +29,6 @@ fi
BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata"
./tools/run_tests/start_port_server.py || true
$PYTHON tools/run_tests/run_microbenchmark.py --collect summary perf latency -b $BENCHMARKS_TO_RUN

@ -106,8 +106,6 @@ def massage_qps_stats(scenario_result):
stats["core_executor_wakeup_initiated"] = massage_qps_stats_helpers.counter(core_stats, "executor_wakeup_initiated")
stats["core_executor_queue_drained"] = massage_qps_stats_helpers.counter(core_stats, "executor_queue_drained")
stats["core_executor_push_retries"] = massage_qps_stats_helpers.counter(core_stats, "executor_push_retries")
stats["core_executor_threads_created"] = massage_qps_stats_helpers.counter(core_stats, "executor_threads_created")
stats["core_executor_threads_used"] = massage_qps_stats_helpers.counter(core_stats, "executor_threads_used")
stats["core_server_requested_calls"] = massage_qps_stats_helpers.counter(core_stats, "server_requested_calls")
stats["core_server_slowpath_requests_queued"] = massage_qps_stats_helpers.counter(core_stats, "server_slowpath_requests_queued")
h = massage_qps_stats_helpers.histogram(core_stats, "call_initial_size")
@ -182,12 +180,6 @@ def massage_qps_stats(scenario_result):
stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
h = massage_qps_stats_helpers.histogram(core_stats, "executor_closures_per_wakeup")
stats["core_executor_closures_per_wakeup"] = ",".join("%f" % x for x in h.buckets)
stats["core_executor_closures_per_wakeup_bkts"] = ",".join("%f" % x for x in h.boundaries)
stats["core_executor_closures_per_wakeup_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries)
stats["core_executor_closures_per_wakeup_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries)
stats["core_executor_closures_per_wakeup_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries)
h = massage_qps_stats_helpers.histogram(core_stats, "server_cqs_checked")
stats["core_server_cqs_checked"] = ",".join("%f" % x for x in h.buckets)
stats["core_server_cqs_checked_bkts"] = ",".join("%f" % x for x in h.boundaries)

@ -540,16 +540,6 @@
"name": "core_executor_push_retries",
"type": "INTEGER"
},
{
"mode": "NULLABLE",
"name": "core_executor_threads_created",
"type": "INTEGER"
},
{
"mode": "NULLABLE",
"name": "core_executor_threads_used",
"type": "INTEGER"
},
{
"mode": "NULLABLE",
"name": "core_server_requested_calls",
@ -860,31 +850,6 @@
"name": "core_http2_send_flowctl_per_write_99p",
"type": "FLOAT"
},
{
"mode": "NULLABLE",
"name": "core_executor_closures_per_wakeup",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "core_executor_closures_per_wakeup_bkts",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "core_executor_closures_per_wakeup_50p",
"type": "FLOAT"
},
{
"mode": "NULLABLE",
"name": "core_executor_closures_per_wakeup_95p",
"type": "FLOAT"
},
{
"mode": "NULLABLE",
"name": "core_executor_closures_per_wakeup_99p",
"type": "FLOAT"
},
{
"mode": "NULLABLE",
"name": "core_server_cqs_checked",
@ -1367,16 +1332,6 @@
"name": "core_executor_push_retries",
"type": "INTEGER"
},
{
"mode": "NULLABLE",
"name": "core_executor_threads_created",
"type": "INTEGER"
},
{
"mode": "NULLABLE",
"name": "core_executor_threads_used",
"type": "INTEGER"
},
{
"mode": "NULLABLE",
"name": "core_server_requested_calls",
@ -1687,31 +1642,6 @@
"name": "core_http2_send_flowctl_per_write_99p",
"type": "FLOAT"
},
{
"mode": "NULLABLE",
"name": "core_executor_closures_per_wakeup",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "core_executor_closures_per_wakeup_bkts",
"type": "STRING"
},
{
"mode": "NULLABLE",
"name": "core_executor_closures_per_wakeup_50p",
"type": "FLOAT"
},
{
"mode": "NULLABLE",
"name": "core_executor_closures_per_wakeup_95p",
"type": "FLOAT"
},
{
"mode": "NULLABLE",
"name": "core_executor_closures_per_wakeup_99p",
"type": "FLOAT"
},
{
"mode": "NULLABLE",
"name": "core_server_cqs_checked",

@ -717,6 +717,9 @@ class PythonLanguage(object):
return (pypy32_config,)
elif args.compiler == 'python_alpine':
return (python27_config,)
elif args.compiler == 'all_the_cpythons':
return (python27_config, python34_config, python35_config,
python36_config,)
else:
raise Exception('Compiler %s not supported.' % args.compiler)
@ -1214,7 +1217,7 @@ argp.add_argument('--compiler',
choices=['default',
'gcc4.4', 'gcc4.6', 'gcc4.8', 'gcc4.9', 'gcc5.3', 'gcc_musl',
'clang3.4', 'clang3.5', 'clang3.6', 'clang3.7',
'python2.7', 'python3.4', 'python3.5', 'python3.6', 'pypy', 'pypy3', 'python_alpine',
'python2.7', 'python3.4', 'python3.5', 'python3.6', 'pypy', 'pypy3', 'python_alpine', 'all_the_cpythons',
'node0.12', 'node4', 'node5', 'node6', 'node7', 'node8',
'electron1.3', 'electron1.6',
'coreclr',

Loading…
Cancel
Save