diff --git a/include/grpc++/support/channel_arguments.h b/include/grpc++/support/channel_arguments.h index 7b6befeaf1e..9dc505f0082 100644 --- a/include/grpc++/support/channel_arguments.h +++ b/include/grpc++/support/channel_arguments.h @@ -64,6 +64,12 @@ class ChannelArguments { /// Set the compression algorithm for the channel. void SetCompressionAlgorithm(grpc_compression_algorithm algorithm); + /// Set the grpclb fallback timeout (in ms) for the channel. If this amount + /// of time has passed but we have not gotten any non-empty \a serverlist from + /// the balancer, we will fall back to use the backend address(es) returned by + /// the resolver. + void SetGrpclbFallbackTimeout(int fallback_timeout); + /// Set the socket mutator for the channel. void SetSocketMutator(grpc_socket_mutator* mutator); diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index 90f03f49a3e..65463bb33b6 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -288,7 +288,11 @@ typedef struct { "grpc.experimental.tcp_max_read_chunk_size" /* Timeout in milliseconds to use for calls to the grpclb load balancer. If 0 or unset, the balancer calls will have no deadline. */ -#define GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS "grpc.grpclb_timeout_ms" +#define GRPC_ARG_GRPCLB_CALL_TIMEOUT_MS "grpc.grpclb_call_timeout_ms" +/* Timeout in milliseconds to wait for the serverlist from the grpclb load + balancer before using fallback backend addresses from the resolver. + If 0, fallback will never be used. */ +#define GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS "grpc.grpclb_fallback_timeout_ms" /** If non-zero, grpc server's cronet compression workaround will be enabled */ #define GRPC_ARG_WORKAROUND_CRONET_COMPRESSION \ "grpc.workaround.cronet_compression" diff --git a/include/grpc/impl/codegen/port_platform.h b/include/grpc/impl/codegen/port_platform.h index 1904c636f75..7e5f85253e1 100644 --- a/include/grpc/impl/codegen/port_platform.h +++ b/include/grpc/impl/codegen/port_platform.h @@ -183,7 +183,6 @@ #define _BSD_SOURCE #endif #if TARGET_OS_IPHONE -#define GPR_FORBID_UNREACHABLE_CODE 1 #define GPR_PLATFORM_STRING "ios" #define GPR_CPU_IPHONE 1 #define GPR_PTHREAD_TLS 1 diff --git a/src/core/ext/filters/client_channel/http_proxy.c b/src/core/ext/filters/client_channel/http_proxy.c index c507a2750e2..a16b44d3dc3 100644 --- a/src/core/ext/filters/client_channel/http_proxy.c +++ b/src/core/ext/filters/client_channel/http_proxy.c @@ -91,6 +91,7 @@ static bool proxy_mapper_map_name(grpc_exec_ctx* exec_ctx, char* user_cred = NULL; *name_to_resolve = get_http_proxy_server(exec_ctx, &user_cred); if (*name_to_resolve == NULL) return false; + char* no_proxy_str = NULL; grpc_uri* uri = grpc_uri_parse(exec_ctx, server_uri, false /* suppress_errors */); if (uri == NULL || uri->path[0] == '\0') { @@ -98,20 +99,14 @@ static bool proxy_mapper_map_name(grpc_exec_ctx* exec_ctx, "'http_proxy' environment variable set, but cannot " "parse server URI '%s' -- not using proxy", server_uri); - if (uri != NULL) { - gpr_free(user_cred); - grpc_uri_destroy(uri); - } - return false; + goto no_use_proxy; } if (strcmp(uri->scheme, "unix") == 0) { gpr_log(GPR_INFO, "not using proxy for Unix domain socket '%s'", server_uri); - gpr_free(user_cred); - grpc_uri_destroy(uri); - return false; + goto no_use_proxy; } - char* no_proxy_str = gpr_getenv("no_proxy"); + no_proxy_str = gpr_getenv("no_proxy"); if (no_proxy_str != NULL) { static const char* NO_PROXY_SEPARATOR = ","; bool use_proxy = true; @@ -147,12 +142,7 @@ static bool proxy_mapper_map_name(grpc_exec_ctx* exec_ctx, gpr_free(no_proxy_hosts); gpr_free(server_host); gpr_free(server_port); - if (!use_proxy) { - grpc_uri_destroy(uri); - gpr_free(*name_to_resolve); - *name_to_resolve = NULL; - return false; - } + if (!use_proxy) goto no_use_proxy; } } grpc_arg args_to_add[2]; @@ -173,9 +163,15 @@ static bool proxy_mapper_map_name(grpc_exec_ctx* exec_ctx, } else { *new_args = grpc_channel_args_copy_and_add(args, args_to_add, 1); } - gpr_free(user_cred); grpc_uri_destroy(uri); + gpr_free(user_cred); return true; +no_use_proxy: + if (uri != NULL) grpc_uri_destroy(uri); + gpr_free(*name_to_resolve); + *name_to_resolve = NULL; + gpr_free(user_cred); + return false; } static bool proxy_mapper_map_address(grpc_exec_ctx* exec_ctx, diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c index 85ef7894ea6..8dc81b46d13 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.c @@ -123,6 +123,7 @@ #define GRPC_GRPCLB_RECONNECT_BACKOFF_MULTIPLIER 1.6 #define GRPC_GRPCLB_RECONNECT_MAX_BACKOFF_SECONDS 120 #define GRPC_GRPCLB_RECONNECT_JITTER 0.2 +#define GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS 10000 grpc_tracer_flag grpc_lb_glb_trace = GRPC_TRACER_INITIALIZER(false, "glb"); @@ -299,6 +300,10 @@ typedef struct glb_lb_policy { /** timeout in milliseconds for the LB call. 0 means no deadline. */ int lb_call_timeout_ms; + /** timeout in milliseconds for before using fallback backend addresses. + * 0 means not using fallback. */ + int lb_fallback_timeout_ms; + /** for communicating with the LB server */ grpc_channel *lb_channel; @@ -325,6 +330,9 @@ typedef struct glb_lb_policy { * Otherwise, we delegate to the RR policy. */ size_t serverlist_index; + /** stores the backend addresses from the resolver */ + grpc_lb_addresses *fallback_backend_addresses; + /** list of picks that are waiting on RR's policy connectivity */ pending_pick *pending_picks; @@ -345,6 +353,9 @@ typedef struct glb_lb_policy { /** is \a lb_call_retry_timer active? */ bool retry_timer_active; + /** is \a lb_fallback_timer active? */ + bool fallback_timer_active; + /** called upon changes to the LB channel's connectivity. */ grpc_closure lb_channel_on_connectivity_changed; @@ -354,9 +365,6 @@ typedef struct glb_lb_policy { /************************************************************/ /* client data associated with the LB server communication */ /************************************************************/ - /* Finished sending initial request. */ - grpc_closure lb_on_sent_initial_request; - /* Status from the LB server has been received. This signals the end of the LB * call. */ grpc_closure lb_on_server_status_received; @@ -367,6 +375,9 @@ typedef struct glb_lb_policy { /* LB call retry timer callback. */ grpc_closure lb_on_call_retry; + /* LB fallback timer callback. */ + grpc_closure lb_on_fallback; + grpc_call *lb_call; /* streaming call to the LB server, */ grpc_metadata_array lb_initial_metadata_recv; /* initial MD from LB server */ @@ -390,7 +401,9 @@ typedef struct glb_lb_policy { /** LB call retry timer */ grpc_timer lb_call_retry_timer; - bool initial_request_sent; + /** LB fallback timer */ + grpc_timer lb_fallback_timer; + bool seen_initial_response; /* Stats for client-side load reporting. Should be unreffed and @@ -536,6 +549,32 @@ static grpc_lb_addresses *process_serverlist_locked( return lb_addresses; } +/* Returns the backend addresses extracted from the given addresses */ +static grpc_lb_addresses *extract_backend_addresses_locked( + grpc_exec_ctx *exec_ctx, const grpc_lb_addresses *addresses) { + /* first pass: count the number of backend addresses */ + size_t num_backends = 0; + for (size_t i = 0; i < addresses->num_addresses; ++i) { + if (!addresses->addresses[i].is_balancer) { + ++num_backends; + } + } + /* second pass: actually populate the addresses and (empty) LB tokens */ + grpc_lb_addresses *backend_addresses = + grpc_lb_addresses_create(num_backends, &lb_token_vtable); + size_t num_copied = 0; + for (size_t i = 0; i < addresses->num_addresses; ++i) { + if (addresses->addresses[i].is_balancer) continue; + const grpc_resolved_address *addr = &addresses->addresses[i].address; + grpc_lb_addresses_set_address(backend_addresses, num_copied, &addr->addr, + addr->len, false /* is_balancer */, + NULL /* balancer_name */, + (void *)GRPC_MDELEM_LB_TOKEN_EMPTY.payload); + ++num_copied; + } + return backend_addresses; +} + static void update_lb_connectivity_status_locked( grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, grpc_connectivity_state rr_state, grpc_error *rr_state_error) { @@ -603,35 +642,38 @@ static bool pick_from_internal_rr_locked( grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, const grpc_lb_policy_pick_args *pick_args, bool force_async, grpc_connected_subchannel **target, wrapped_rr_closure_arg *wc_arg) { - // Look at the index into the serverlist to see if we should drop this call. - grpc_grpclb_server *server = - glb_policy->serverlist->servers[glb_policy->serverlist_index++]; - if (glb_policy->serverlist_index == glb_policy->serverlist->num_servers) { - glb_policy->serverlist_index = 0; // Wrap-around. - } - if (server->drop) { - // Not using the RR policy, so unref it. - if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { - gpr_log(GPR_INFO, "Unreffing RR for drop (0x%" PRIxPTR ")", - (intptr_t)wc_arg->rr_policy); + // Check for drops if we are not using fallback backend addresses. + if (glb_policy->serverlist != NULL) { + // Look at the index into the serverlist to see if we should drop this call. + grpc_grpclb_server *server = + glb_policy->serverlist->servers[glb_policy->serverlist_index++]; + if (glb_policy->serverlist_index == glb_policy->serverlist->num_servers) { + glb_policy->serverlist_index = 0; // Wrap-around. } - GRPC_LB_POLICY_UNREF(exec_ctx, wc_arg->rr_policy, "glb_pick_sync"); - // Update client load reporting stats to indicate the number of - // dropped calls. Note that we have to do this here instead of in - // the client_load_reporting filter, because we do not create a - // subchannel call (and therefore no client_load_reporting filter) - // for dropped calls. - grpc_grpclb_client_stats_add_call_dropped_locked(server->load_balance_token, - wc_arg->client_stats); - grpc_grpclb_client_stats_unref(wc_arg->client_stats); - if (force_async) { - GPR_ASSERT(wc_arg->wrapped_closure != NULL); - GRPC_CLOSURE_SCHED(exec_ctx, wc_arg->wrapped_closure, GRPC_ERROR_NONE); + if (server->drop) { + // Not using the RR policy, so unref it. + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { + gpr_log(GPR_INFO, "Unreffing RR for drop (0x%" PRIxPTR ")", + (intptr_t)wc_arg->rr_policy); + } + GRPC_LB_POLICY_UNREF(exec_ctx, wc_arg->rr_policy, "glb_pick_sync"); + // Update client load reporting stats to indicate the number of + // dropped calls. Note that we have to do this here instead of in + // the client_load_reporting filter, because we do not create a + // subchannel call (and therefore no client_load_reporting filter) + // for dropped calls. + grpc_grpclb_client_stats_add_call_dropped_locked( + server->load_balance_token, wc_arg->client_stats); + grpc_grpclb_client_stats_unref(wc_arg->client_stats); + if (force_async) { + GPR_ASSERT(wc_arg->wrapped_closure != NULL); + GRPC_CLOSURE_SCHED(exec_ctx, wc_arg->wrapped_closure, GRPC_ERROR_NONE); + gpr_free(wc_arg->free_when_done); + return false; + } gpr_free(wc_arg->free_when_done); - return false; + return true; } - gpr_free(wc_arg->free_when_done); - return true; } // Pick via the RR policy. const bool pick_done = grpc_lb_policy_pick_locked( @@ -669,8 +711,18 @@ static bool pick_from_internal_rr_locked( static grpc_lb_policy_args *lb_policy_args_create(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy) { - grpc_lb_addresses *addresses = - process_serverlist_locked(exec_ctx, glb_policy->serverlist); + grpc_lb_addresses *addresses; + if (glb_policy->serverlist != NULL) { + GPR_ASSERT(glb_policy->serverlist->num_servers > 0); + addresses = process_serverlist_locked(exec_ctx, glb_policy->serverlist); + } else { + // If rr_handover_locked() is invoked when we haven't received any + // serverlist from the balancer, we use the fallback backends returned by + // the resolver. Note that the fallback backend list may be empty, in which + // case the new round_robin policy will keep the requested picks pending. + GPR_ASSERT(glb_policy->fallback_backend_addresses != NULL); + addresses = grpc_lb_addresses_copy(glb_policy->fallback_backend_addresses); + } GPR_ASSERT(addresses != NULL); grpc_lb_policy_args *args = (grpc_lb_policy_args *)gpr_zalloc(sizeof(*args)); args->client_channel_factory = glb_policy->cc_factory; @@ -776,8 +828,6 @@ static void create_rr_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, /* glb_policy->rr_policy may be NULL (initial handover) */ static void rr_handover_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy) { - GPR_ASSERT(glb_policy->serverlist != NULL && - glb_policy->serverlist->num_servers > 0); if (glb_policy->shutting_down) return; grpc_lb_policy_args *args = lb_policy_args_create(exec_ctx, glb_policy); GPR_ASSERT(args != NULL); @@ -926,6 +976,9 @@ static void glb_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { if (glb_policy->serverlist != NULL) { grpc_grpclb_destroy_serverlist(glb_policy->serverlist); } + if (glb_policy->fallback_backend_addresses != NULL) { + grpc_lb_addresses_destroy(exec_ctx, glb_policy->fallback_backend_addresses); + } grpc_fake_resolver_response_generator_unref(glb_policy->response_generator); grpc_subchannel_index_unref(); if (glb_policy->pending_update_args != NULL) { @@ -1067,10 +1120,28 @@ static void glb_cancel_picks_locked(grpc_exec_ctx *exec_ctx, GRPC_ERROR_UNREF(error); } +static void lb_on_fallback_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error); static void query_for_backends_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy); static void start_picking_locked(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy) { + /* start a timer to fall back */ + if (glb_policy->lb_fallback_timeout_ms > 0 && + glb_policy->serverlist == NULL && !glb_policy->fallback_timer_active) { + gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + gpr_timespec deadline = gpr_time_add( + now, + gpr_time_from_millis(glb_policy->lb_fallback_timeout_ms, GPR_TIMESPAN)); + GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "grpclb_fallback_timer"); + GRPC_CLOSURE_INIT(&glb_policy->lb_on_fallback, lb_on_fallback_timer_locked, + glb_policy, + grpc_combiner_scheduler(glb_policy->base.combiner)); + glb_policy->fallback_timer_active = true; + grpc_timer_init(exec_ctx, &glb_policy->lb_fallback_timer, deadline, + &glb_policy->lb_on_fallback, now); + } + glb_policy->started_picking = true; gpr_backoff_reset(&glb_policy->lb_call_backoff_state); query_for_backends_locked(exec_ctx, glb_policy); @@ -1173,6 +1244,58 @@ static void glb_notify_on_state_change_locked(grpc_exec_ctx *exec_ctx, exec_ctx, &glb_policy->state_tracker, current, notify); } +static void lb_call_on_retry_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + glb_lb_policy *glb_policy = (glb_lb_policy *)arg; + glb_policy->retry_timer_active = false; + if (!glb_policy->shutting_down && error == GRPC_ERROR_NONE) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { + gpr_log(GPR_INFO, "Restaring call to LB server (grpclb %p)", + (void *)glb_policy); + } + GPR_ASSERT(glb_policy->lb_call == NULL); + query_for_backends_locked(exec_ctx, glb_policy); + } + GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base, "grpclb_retry_timer"); +} + +static void maybe_restart_lb_call(grpc_exec_ctx *exec_ctx, + glb_lb_policy *glb_policy) { + if (glb_policy->started_picking && glb_policy->updating_lb_call) { + if (glb_policy->retry_timer_active) { + grpc_timer_cancel(exec_ctx, &glb_policy->lb_call_retry_timer); + } + if (!glb_policy->shutting_down) start_picking_locked(exec_ctx, glb_policy); + glb_policy->updating_lb_call = false; + } else if (!glb_policy->shutting_down) { + /* if we aren't shutting down, restart the LB client call after some time */ + gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); + gpr_timespec next_try = + gpr_backoff_step(&glb_policy->lb_call_backoff_state, now); + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { + gpr_log(GPR_DEBUG, "Connection to LB server lost (grpclb: %p)...", + (void *)glb_policy); + gpr_timespec timeout = gpr_time_sub(next_try, now); + if (gpr_time_cmp(timeout, gpr_time_0(timeout.clock_type)) > 0) { + gpr_log(GPR_DEBUG, + "... retry_timer_active in %" PRId64 ".%09d seconds.", + timeout.tv_sec, timeout.tv_nsec); + } else { + gpr_log(GPR_DEBUG, "... retry_timer_active immediately."); + } + } + GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "grpclb_retry_timer"); + GRPC_CLOSURE_INIT(&glb_policy->lb_on_call_retry, + lb_call_on_retry_timer_locked, glb_policy, + grpc_combiner_scheduler(glb_policy->base.combiner)); + glb_policy->retry_timer_active = true; + grpc_timer_init(exec_ctx, &glb_policy->lb_call_retry_timer, next_try, + &glb_policy->lb_on_call_retry, now); + } + GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base, + "lb_on_server_status_received_locked"); +} + static void send_client_load_report_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error); @@ -1203,21 +1326,6 @@ static void client_load_report_done_locked(grpc_exec_ctx *exec_ctx, void *arg, schedule_next_client_load_report(exec_ctx, glb_policy); } -static void do_send_client_load_report_locked(grpc_exec_ctx *exec_ctx, - glb_lb_policy *glb_policy) { - grpc_op op; - memset(&op, 0, sizeof(op)); - op.op = GRPC_OP_SEND_MESSAGE; - op.data.send_message.send_message = glb_policy->client_load_report_payload; - GRPC_CLOSURE_INIT(&glb_policy->client_load_report_closure, - client_load_report_done_locked, glb_policy, - grpc_combiner_scheduler(glb_policy->base.combiner)); - grpc_call_error call_error = grpc_call_start_batch_and_execute( - exec_ctx, glb_policy->lb_call, &op, 1, - &glb_policy->client_load_report_closure); - GPR_ASSERT(GRPC_CALL_OK == call_error); -} - static bool load_report_counters_are_zero(grpc_grpclb_request *request) { grpc_grpclb_dropped_call_counts *drop_entries = (grpc_grpclb_dropped_call_counts *) @@ -1237,6 +1345,9 @@ static void send_client_load_report_locked(grpc_exec_ctx *exec_ctx, void *arg, glb_policy->client_load_report_timer_pending = false; GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base, "client_load_report"); + if (glb_policy->lb_call == NULL) { + maybe_restart_lb_call(exec_ctx, glb_policy); + } return; } // Construct message payload. @@ -1260,17 +1371,23 @@ static void send_client_load_report_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_raw_byte_buffer_create(&request_payload_slice, 1); grpc_slice_unref_internal(exec_ctx, request_payload_slice); grpc_grpclb_request_destroy(request); - // If we've already sent the initial request, then we can go ahead and - // sent the load report. Otherwise, we need to wait until the initial - // request has been sent to send this - // (see lb_on_sent_initial_request_locked() below). - if (glb_policy->initial_request_sent) { - do_send_client_load_report_locked(exec_ctx, glb_policy); + // Send load report message. + grpc_op op; + memset(&op, 0, sizeof(op)); + op.op = GRPC_OP_SEND_MESSAGE; + op.data.send_message.send_message = glb_policy->client_load_report_payload; + GRPC_CLOSURE_INIT(&glb_policy->client_load_report_closure, + client_load_report_done_locked, glb_policy, + grpc_combiner_scheduler(glb_policy->base.combiner)); + grpc_call_error call_error = grpc_call_start_batch_and_execute( + exec_ctx, glb_policy->lb_call, &op, 1, + &glb_policy->client_load_report_closure); + if (call_error != GRPC_CALL_OK) { + gpr_log(GPR_ERROR, "call_error=%d", call_error); + GPR_ASSERT(GRPC_CALL_OK == call_error); } } -static void lb_on_sent_initial_request_locked(grpc_exec_ctx *exec_ctx, - void *arg, grpc_error *error); static void lb_on_server_status_received_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error); static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, @@ -1315,9 +1432,6 @@ static void lb_call_init_locked(grpc_exec_ctx *exec_ctx, grpc_slice_unref_internal(exec_ctx, request_payload_slice); grpc_grpclb_request_destroy(request); - GRPC_CLOSURE_INIT(&glb_policy->lb_on_sent_initial_request, - lb_on_sent_initial_request_locked, glb_policy, - grpc_combiner_scheduler(glb_policy->base.combiner)); GRPC_CLOSURE_INIT(&glb_policy->lb_on_server_status_received, lb_on_server_status_received_locked, glb_policy, grpc_combiner_scheduler(glb_policy->base.combiner)); @@ -1332,7 +1446,6 @@ static void lb_call_init_locked(grpc_exec_ctx *exec_ctx, GRPC_GRPCLB_MIN_CONNECT_TIMEOUT_SECONDS * 1000, GRPC_GRPCLB_RECONNECT_MAX_BACKOFF_SECONDS * 1000); - glb_policy->initial_request_sent = false; glb_policy->seen_initial_response = false; glb_policy->last_client_load_report_counters_were_zero = false; } @@ -1349,7 +1462,7 @@ static void lb_call_destroy_locked(grpc_exec_ctx *exec_ctx, grpc_byte_buffer_destroy(glb_policy->lb_request_payload); grpc_slice_unref_internal(exec_ctx, glb_policy->lb_call_status_details); - if (!glb_policy->client_load_report_timer_pending) { + if (glb_policy->client_load_report_timer_pending) { grpc_timer_cancel(exec_ctx, &glb_policy->client_load_report_timer); } } @@ -1373,7 +1486,7 @@ static void query_for_backends_locked(grpc_exec_ctx *exec_ctx, GPR_ASSERT(glb_policy->lb_call != NULL); grpc_call_error call_error; - grpc_op ops[4]; + grpc_op ops[3]; memset(ops, 0, sizeof(ops)); grpc_op *op = ops; @@ -1394,13 +1507,8 @@ static void query_for_backends_locked(grpc_exec_ctx *exec_ctx, op->flags = 0; op->reserved = NULL; op++; - /* take a weak ref (won't prevent calling of \a glb_shutdown if the strong ref - * count goes to zero) to be unref'd in lb_on_sent_initial_request_locked() */ - GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, - "lb_on_sent_initial_request_locked"); - call_error = grpc_call_start_batch_and_execute( - exec_ctx, glb_policy->lb_call, ops, (size_t)(op - ops), - &glb_policy->lb_on_sent_initial_request); + call_error = grpc_call_start_batch_and_execute(exec_ctx, glb_policy->lb_call, + ops, (size_t)(op - ops), NULL); GPR_ASSERT(GRPC_CALL_OK == call_error); op = ops; @@ -1437,19 +1545,6 @@ static void query_for_backends_locked(grpc_exec_ctx *exec_ctx, GPR_ASSERT(GRPC_CALL_OK == call_error); } -static void lb_on_sent_initial_request_locked(grpc_exec_ctx *exec_ctx, - void *arg, grpc_error *error) { - glb_lb_policy *glb_policy = (glb_lb_policy *)arg; - glb_policy->initial_request_sent = true; - // If we attempted to send a client load report before the initial - // request was sent, send the load report now. - if (glb_policy->client_load_report_payload != NULL) { - do_send_client_load_report_locked(exec_ctx, glb_policy); - } - GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base, - "lb_on_sent_initial_request_locked"); -} - static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { glb_lb_policy *glb_policy = (glb_lb_policy *)arg; @@ -1525,6 +1620,15 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, if (glb_policy->serverlist != NULL) { /* dispose of the old serverlist */ grpc_grpclb_destroy_serverlist(glb_policy->serverlist); + } else { + /* or dispose of the fallback */ + grpc_lb_addresses_destroy(exec_ctx, + glb_policy->fallback_backend_addresses); + glb_policy->fallback_backend_addresses = NULL; + if (glb_policy->fallback_timer_active) { + grpc_timer_cancel(exec_ctx, &glb_policy->lb_fallback_timer); + glb_policy->fallback_timer_active = false; + } } /* and update the copy in the glb_lb_policy instance. This * serverlist instance will be destroyed either upon the next @@ -1535,9 +1639,7 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, } } else { if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { - gpr_log(GPR_INFO, - "Received empty server list. Picks will stay pending until " - "a response with > 0 servers is received"); + gpr_log(GPR_INFO, "Received empty server list, ignoring."); } grpc_grpclb_destroy_serverlist(serverlist); } @@ -1572,19 +1674,25 @@ static void lb_on_response_received_locked(grpc_exec_ctx *exec_ctx, void *arg, } } -static void lb_call_on_retry_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error) { +static void lb_on_fallback_timer_locked(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { glb_lb_policy *glb_policy = (glb_lb_policy *)arg; - glb_policy->retry_timer_active = false; - if (!glb_policy->shutting_down && error == GRPC_ERROR_NONE) { - if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { - gpr_log(GPR_INFO, "Restaring call to LB server (grpclb %p)", - (void *)glb_policy); + glb_policy->fallback_timer_active = false; + /* If we receive a serverlist after the timer fires but before this callback + * actually runs, don't fall back. */ + if (glb_policy->serverlist == NULL) { + if (!glb_policy->shutting_down && error == GRPC_ERROR_NONE) { + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { + gpr_log(GPR_INFO, + "Falling back to use backends from resolver (grpclb %p)", + (void *)glb_policy); + } + GPR_ASSERT(glb_policy->fallback_backend_addresses != NULL); + rr_handover_locked(exec_ctx, glb_policy); } - GPR_ASSERT(glb_policy->lb_call == NULL); - query_for_backends_locked(exec_ctx, glb_policy); } - GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base, "grpclb_retry_timer"); + GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base, + "grpclb_fallback_timer"); } static void lb_on_server_status_received_locked(grpc_exec_ctx *exec_ctx, @@ -1603,66 +1711,30 @@ static void lb_on_server_status_received_locked(grpc_exec_ctx *exec_ctx, } /* We need to perform cleanups no matter what. */ lb_call_destroy_locked(exec_ctx, glb_policy); - if (glb_policy->started_picking && glb_policy->updating_lb_call) { - if (glb_policy->retry_timer_active) { - grpc_timer_cancel(exec_ctx, &glb_policy->lb_call_retry_timer); - } - if (!glb_policy->shutting_down) start_picking_locked(exec_ctx, glb_policy); - glb_policy->updating_lb_call = false; - } else if (!glb_policy->shutting_down) { - /* if we aren't shutting down, restart the LB client call after some time */ - gpr_timespec now = gpr_now(GPR_CLOCK_MONOTONIC); - gpr_timespec next_try = - gpr_backoff_step(&glb_policy->lb_call_backoff_state, now); - if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { - gpr_log(GPR_DEBUG, "Connection to LB server lost (grpclb: %p)...", - (void *)glb_policy); - gpr_timespec timeout = gpr_time_sub(next_try, now); - if (gpr_time_cmp(timeout, gpr_time_0(timeout.clock_type)) > 0) { - gpr_log(GPR_DEBUG, - "... retry_timer_active in %" PRId64 ".%09d seconds.", - timeout.tv_sec, timeout.tv_nsec); - } else { - gpr_log(GPR_DEBUG, "... retry_timer_active immediately."); - } - } - GRPC_LB_POLICY_WEAK_REF(&glb_policy->base, "grpclb_retry_timer"); - GRPC_CLOSURE_INIT(&glb_policy->lb_on_call_retry, - lb_call_on_retry_timer_locked, glb_policy, - grpc_combiner_scheduler(glb_policy->base.combiner)); - glb_policy->retry_timer_active = true; - grpc_timer_init(exec_ctx, &glb_policy->lb_call_retry_timer, next_try, - &glb_policy->lb_on_call_retry, now); + // If the load report timer is still pending, we wait for it to be + // called before restarting the call. Otherwise, we restart the call + // here. + if (!glb_policy->client_load_report_timer_pending) { + maybe_restart_lb_call(exec_ctx, glb_policy); + } +} + +static void fallback_update_locked(grpc_exec_ctx *exec_ctx, + glb_lb_policy *glb_policy, + const grpc_lb_addresses *addresses) { + GPR_ASSERT(glb_policy->fallback_backend_addresses != NULL); + grpc_lb_addresses_destroy(exec_ctx, glb_policy->fallback_backend_addresses); + glb_policy->fallback_backend_addresses = + extract_backend_addresses_locked(exec_ctx, addresses); + if (glb_policy->lb_fallback_timeout_ms > 0 && + !glb_policy->fallback_timer_active) { + rr_handover_locked(exec_ctx, glb_policy); } - GRPC_LB_POLICY_WEAK_UNREF(exec_ctx, &glb_policy->base, - "lb_on_server_status_received_locked"); } static void glb_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, const grpc_lb_policy_args *args) { glb_lb_policy *glb_policy = (glb_lb_policy *)policy; - if (glb_policy->updating_lb_channel) { - if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { - gpr_log(GPR_INFO, - "Update already in progress for grpclb %p. Deferring update.", - (void *)glb_policy); - } - if (glb_policy->pending_update_args != NULL) { - grpc_channel_args_destroy(exec_ctx, - glb_policy->pending_update_args->args); - gpr_free(glb_policy->pending_update_args); - } - glb_policy->pending_update_args = (grpc_lb_policy_args *)gpr_zalloc( - sizeof(*glb_policy->pending_update_args)); - glb_policy->pending_update_args->client_channel_factory = - args->client_channel_factory; - glb_policy->pending_update_args->args = grpc_channel_args_copy(args->args); - glb_policy->pending_update_args->combiner = args->combiner; - return; - } - - glb_policy->updating_lb_channel = true; - // Propagate update to lb_channel (pick first). const grpc_arg *arg = grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES); if (arg == NULL || arg->type != GRPC_ARG_POINTER) { @@ -1680,13 +1752,43 @@ static void glb_update_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, "ignoring.", (void *)glb_policy); } + return; } const grpc_lb_addresses *addresses = (const grpc_lb_addresses *)arg->value.pointer.p; + + if (glb_policy->serverlist == NULL) { + // If a non-empty serverlist hasn't been received from the balancer, + // propagate the update to fallback_backend_addresses. + fallback_update_locked(exec_ctx, glb_policy, addresses); + } else if (glb_policy->updating_lb_channel) { + // If we have recieved serverlist from the balancer, we need to defer update + // when there is an in-progress one. + if (GRPC_TRACER_ON(grpc_lb_glb_trace)) { + gpr_log(GPR_INFO, + "Update already in progress for grpclb %p. Deferring update.", + (void *)glb_policy); + } + if (glb_policy->pending_update_args != NULL) { + grpc_channel_args_destroy(exec_ctx, + glb_policy->pending_update_args->args); + gpr_free(glb_policy->pending_update_args); + } + glb_policy->pending_update_args = (grpc_lb_policy_args *)gpr_zalloc( + sizeof(*glb_policy->pending_update_args)); + glb_policy->pending_update_args->client_channel_factory = + args->client_channel_factory; + glb_policy->pending_update_args->args = grpc_channel_args_copy(args->args); + glb_policy->pending_update_args->combiner = args->combiner; + return; + } + + glb_policy->updating_lb_channel = true; GPR_ASSERT(glb_policy->lb_channel != NULL); grpc_channel_args *lb_channel_args = build_lb_channel_args( exec_ctx, addresses, glb_policy->response_generator, args->args); - /* Propagate updates to the LB channel through the fake resolver */ + /* Propagate updates to the LB channel (pick first) through the fake resolver + */ grpc_fake_resolver_response_generator_set_response( exec_ctx, glb_policy->response_generator, lb_channel_args); grpc_channel_args_destroy(exec_ctx, lb_channel_args); @@ -1789,13 +1891,7 @@ static const grpc_lb_policy_vtable glb_lb_policy_vtable = { static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, grpc_lb_policy_factory *factory, grpc_lb_policy_args *args) { - /* Count the number of gRPC-LB addresses. There must be at least one. - * TODO(roth): For now, we ignore non-balancer addresses, but in the - * future, we may change the behavior such that we fall back to using - * the non-balancer addresses if we cannot reach any balancers. In the - * fallback case, we should use the LB policy indicated by - * GRPC_ARG_LB_POLICY_NAME (although if that specifies grpclb or is - * unset, we should default to pick_first). */ + /* Count the number of gRPC-LB addresses. There must be at least one. */ const grpc_arg *arg = grpc_channel_args_find(args->args, GRPC_ARG_LB_ADDRESSES); if (arg == NULL || arg->type != GRPC_ARG_POINTER) { @@ -1831,6 +1927,11 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, glb_policy->lb_call_timeout_ms = grpc_channel_arg_get_integer(arg, (grpc_integer_options){0, 0, INT_MAX}); + arg = grpc_channel_args_find(args->args, GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS); + glb_policy->lb_fallback_timeout_ms = grpc_channel_arg_get_integer( + arg, (grpc_integer_options){GRPC_GRPCLB_DEFAULT_FALLBACK_TIMEOUT_MS, 0, + INT_MAX}); + // Make sure that GRPC_ARG_LB_POLICY_NAME is set in channel args, // since we use this to trigger the client_load_reporting filter. grpc_arg new_arg = grpc_channel_arg_string_create( @@ -1839,6 +1940,11 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, glb_policy->args = grpc_channel_args_copy_and_add_and_remove( args->args, args_to_remove, GPR_ARRAY_SIZE(args_to_remove), &new_arg, 1); + /* Extract the backend addresses (may be empty) from the resolver for + * fallback. */ + glb_policy->fallback_backend_addresses = + extract_backend_addresses_locked(exec_ctx, addresses); + /* Create a client channel over them to communicate with a LB service */ glb_policy->response_generator = grpc_fake_resolver_response_generator_create(); diff --git a/src/core/ext/filters/client_channel/lb_policy_factory.c b/src/core/ext/filters/client_channel/lb_policy_factory.c index 4d1405454c9..05ab43d0b69 100644 --- a/src/core/ext/filters/client_channel/lb_policy_factory.c +++ b/src/core/ext/filters/client_channel/lb_policy_factory.c @@ -56,7 +56,7 @@ grpc_lb_addresses* grpc_lb_addresses_copy(const grpc_lb_addresses* addresses) { } void grpc_lb_addresses_set_address(grpc_lb_addresses* addresses, size_t index, - void* address, size_t address_len, + const void* address, size_t address_len, bool is_balancer, const char* balancer_name, void* user_data) { GPR_ASSERT(index < addresses->num_addresses); diff --git a/src/core/ext/filters/client_channel/lb_policy_factory.h b/src/core/ext/filters/client_channel/lb_policy_factory.h index 9d9fb143df5..cf0f8cb6157 100644 --- a/src/core/ext/filters/client_channel/lb_policy_factory.h +++ b/src/core/ext/filters/client_channel/lb_policy_factory.h @@ -73,7 +73,7 @@ grpc_lb_addresses *grpc_lb_addresses_copy(const grpc_lb_addresses *addresses); * \a address is a socket address of length \a address_len. * Takes ownership of \a balancer_name. */ void grpc_lb_addresses_set_address(grpc_lb_addresses *addresses, size_t index, - void *address, size_t address_len, + const void *address, size_t address_len, bool is_balancer, const char *balancer_name, void *user_data); diff --git a/src/core/lib/debug/stats_data.c b/src/core/lib/debug/stats_data.c index fb6055f7957..c0aec63c1d3 100644 --- a/src/core/lib/debug/stats_data.c +++ b/src/core/lib/debug/stats_data.c @@ -109,8 +109,6 @@ const char *grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = { "executor_wakeup_initiated", "executor_queue_drained", "executor_push_retries", - "executor_threads_created", - "executor_threads_used", "server_requested_calls", "server_slowpath_requests_queued", }; @@ -219,8 +217,6 @@ const char *grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = { "Number of times an executor queue was drained", "Number of times we raced and were forced to retry pushing a closure to " "the executor", - "Size of the backing thread pool for overflow gRPC Core work", - "How many executor threads actually got used", "How many calls were requested (not necessarily received) by the server", "How many times was the server slow path taken (indicates too few " "outstanding requests)", @@ -238,7 +234,6 @@ const char *grpc_stats_histogram_name[GRPC_STATS_HISTOGRAM_COUNT] = { "http2_send_message_per_write", "http2_send_trailing_metadata_per_write", "http2_send_flowctl_per_write", - "executor_closures_per_wakeup", "server_cqs_checked", }; const char *grpc_stats_histogram_doc[GRPC_STATS_HISTOGRAM_COUNT] = { @@ -254,7 +249,6 @@ const char *grpc_stats_histogram_doc[GRPC_STATS_HISTOGRAM_COUNT] = { "Number of streams whose payload was written per TCP write", "Number of streams terminated per TCP write", "Number of flow control updates written per TCP write", - "Number of closures executed each time an executor wakes up", "How many completion queues were checked looking for a CQ that had " "requested the incoming call", }; @@ -326,7 +320,6 @@ const uint8_t grpc_stats_table_7[102] = { const int grpc_stats_table_8[9] = {0, 1, 2, 4, 7, 13, 23, 39, 64}; const uint8_t grpc_stats_table_9[9] = {0, 0, 1, 2, 2, 3, 4, 4, 5}; void grpc_stats_inc_call_initial_size(grpc_exec_ctx *exec_ctx, int value) { - /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 262144); if (value < 6) { GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_CALL_INITIAL_SIZE, @@ -352,7 +345,6 @@ void grpc_stats_inc_call_initial_size(grpc_exec_ctx *exec_ctx, int value) { (exec_ctx), value, grpc_stats_table_0, 64)); } void grpc_stats_inc_poll_events_returned(grpc_exec_ctx *exec_ctx, int value) { - /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 1024); if (value < 29) { GRPC_STATS_INC_HISTOGRAM((exec_ctx), @@ -379,7 +371,6 @@ void grpc_stats_inc_poll_events_returned(grpc_exec_ctx *exec_ctx, int value) { (exec_ctx), value, grpc_stats_table_2, 128)); } void grpc_stats_inc_tcp_write_size(grpc_exec_ctx *exec_ctx, int value) { - /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 16777216); if (value < 5) { GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_WRITE_SIZE, @@ -405,7 +396,6 @@ void grpc_stats_inc_tcp_write_size(grpc_exec_ctx *exec_ctx, int value) { (exec_ctx), value, grpc_stats_table_4, 64)); } void grpc_stats_inc_tcp_write_iov_size(grpc_exec_ctx *exec_ctx, int value) { - /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 1024); if (value < 13) { GRPC_STATS_INC_HISTOGRAM((exec_ctx), @@ -431,7 +421,6 @@ void grpc_stats_inc_tcp_write_iov_size(grpc_exec_ctx *exec_ctx, int value) { (exec_ctx), value, grpc_stats_table_6, 64)); } void grpc_stats_inc_tcp_read_size(grpc_exec_ctx *exec_ctx, int value) { - /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 16777216); if (value < 5) { GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_SIZE, @@ -457,7 +446,6 @@ void grpc_stats_inc_tcp_read_size(grpc_exec_ctx *exec_ctx, int value) { (exec_ctx), value, grpc_stats_table_4, 64)); } void grpc_stats_inc_tcp_read_offer(grpc_exec_ctx *exec_ctx, int value) { - /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 16777216); if (value < 5) { GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_TCP_READ_OFFER, @@ -484,7 +472,6 @@ void grpc_stats_inc_tcp_read_offer(grpc_exec_ctx *exec_ctx, int value) { } void grpc_stats_inc_tcp_read_offer_iov_size(grpc_exec_ctx *exec_ctx, int value) { - /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 1024); if (value < 13) { GRPC_STATS_INC_HISTOGRAM( @@ -512,7 +499,6 @@ void grpc_stats_inc_tcp_read_offer_iov_size(grpc_exec_ctx *exec_ctx, } void grpc_stats_inc_http2_send_message_size(grpc_exec_ctx *exec_ctx, int value) { - /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 16777216); if (value < 5) { GRPC_STATS_INC_HISTOGRAM( @@ -540,7 +526,6 @@ void grpc_stats_inc_http2_send_message_size(grpc_exec_ctx *exec_ctx, } void grpc_stats_inc_http2_send_initial_metadata_per_write( grpc_exec_ctx *exec_ctx, int value) { - /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 1024); if (value < 13) { GRPC_STATS_INC_HISTOGRAM( @@ -570,7 +555,6 @@ void grpc_stats_inc_http2_send_initial_metadata_per_write( } void grpc_stats_inc_http2_send_message_per_write(grpc_exec_ctx *exec_ctx, int value) { - /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 1024); if (value < 13) { GRPC_STATS_INC_HISTOGRAM( @@ -598,7 +582,6 @@ void grpc_stats_inc_http2_send_message_per_write(grpc_exec_ctx *exec_ctx, } void grpc_stats_inc_http2_send_trailing_metadata_per_write( grpc_exec_ctx *exec_ctx, int value) { - /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 1024); if (value < 13) { GRPC_STATS_INC_HISTOGRAM( @@ -628,7 +611,6 @@ void grpc_stats_inc_http2_send_trailing_metadata_per_write( } void grpc_stats_inc_http2_send_flowctl_per_write(grpc_exec_ctx *exec_ctx, int value) { - /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 1024); if (value < 13) { GRPC_STATS_INC_HISTOGRAM( @@ -654,36 +636,7 @@ void grpc_stats_inc_http2_send_flowctl_per_write(grpc_exec_ctx *exec_ctx, grpc_stats_histo_find_bucket_slow( (exec_ctx), value, grpc_stats_table_6, 64)); } -void grpc_stats_inc_executor_closures_per_wakeup(grpc_exec_ctx *exec_ctx, - int value) { - /* Automatically generated by tools/codegen/core/gen_stats_data.py */ - value = GPR_CLAMP(value, 0, 1024); - if (value < 13) { - GRPC_STATS_INC_HISTOGRAM( - (exec_ctx), GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP, value); - return; - } - union { - double dbl; - uint64_t uint; - } _val, _bkt; - _val.dbl = value; - if (_val.uint < 4637863191261478912ull) { - int bucket = - grpc_stats_table_7[((_val.uint - 4623507967449235456ull) >> 48)] + 13; - _bkt.dbl = grpc_stats_table_6[bucket]; - bucket -= (_val.uint < _bkt.uint); - GRPC_STATS_INC_HISTOGRAM( - (exec_ctx), GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP, bucket); - return; - } - GRPC_STATS_INC_HISTOGRAM((exec_ctx), - GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP, - grpc_stats_histo_find_bucket_slow( - (exec_ctx), value, grpc_stats_table_6, 64)); -} void grpc_stats_inc_server_cqs_checked(grpc_exec_ctx *exec_ctx, int value) { - /* Automatically generated by tools/codegen/core/gen_stats_data.py */ value = GPR_CLAMP(value, 0, 64); if (value < 3) { GRPC_STATS_INC_HISTOGRAM((exec_ctx), @@ -708,17 +661,17 @@ void grpc_stats_inc_server_cqs_checked(grpc_exec_ctx *exec_ctx, int value) { grpc_stats_histo_find_bucket_slow( (exec_ctx), value, grpc_stats_table_8, 8)); } -const int grpc_stats_histo_buckets[14] = {64, 128, 64, 64, 64, 64, 64, - 64, 64, 64, 64, 64, 64, 8}; -const int grpc_stats_histo_start[14] = {0, 64, 192, 256, 320, 384, 448, - 512, 576, 640, 704, 768, 832, 896}; -const int *const grpc_stats_histo_bucket_boundaries[14] = { +const int grpc_stats_histo_buckets[13] = {64, 128, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 8}; +const int grpc_stats_histo_start[13] = {0, 64, 192, 256, 320, 384, 448, + 512, 576, 640, 704, 768, 832}; +const int *const grpc_stats_histo_bucket_boundaries[13] = { grpc_stats_table_0, grpc_stats_table_2, grpc_stats_table_4, grpc_stats_table_6, grpc_stats_table_4, grpc_stats_table_4, grpc_stats_table_6, grpc_stats_table_4, grpc_stats_table_6, grpc_stats_table_6, grpc_stats_table_6, grpc_stats_table_6, - grpc_stats_table_6, grpc_stats_table_8}; -void (*const grpc_stats_inc_histogram[14])(grpc_exec_ctx *exec_ctx, int x) = { + grpc_stats_table_8}; +void (*const grpc_stats_inc_histogram[13])(grpc_exec_ctx *exec_ctx, int x) = { grpc_stats_inc_call_initial_size, grpc_stats_inc_poll_events_returned, grpc_stats_inc_tcp_write_size, @@ -731,5 +684,4 @@ void (*const grpc_stats_inc_histogram[14])(grpc_exec_ctx *exec_ctx, int x) = { grpc_stats_inc_http2_send_message_per_write, grpc_stats_inc_http2_send_trailing_metadata_per_write, grpc_stats_inc_http2_send_flowctl_per_write, - grpc_stats_inc_executor_closures_per_wakeup, grpc_stats_inc_server_cqs_checked}; diff --git a/src/core/lib/debug/stats_data.h b/src/core/lib/debug/stats_data.h index 6c0ad30543f..28dab00117b 100644 --- a/src/core/lib/debug/stats_data.h +++ b/src/core/lib/debug/stats_data.h @@ -111,8 +111,6 @@ typedef enum { GRPC_STATS_COUNTER_EXECUTOR_WAKEUP_INITIATED, GRPC_STATS_COUNTER_EXECUTOR_QUEUE_DRAINED, GRPC_STATS_COUNTER_EXECUTOR_PUSH_RETRIES, - GRPC_STATS_COUNTER_EXECUTOR_THREADS_CREATED, - GRPC_STATS_COUNTER_EXECUTOR_THREADS_USED, GRPC_STATS_COUNTER_SERVER_REQUESTED_CALLS, GRPC_STATS_COUNTER_SERVER_SLOWPATH_REQUESTS_QUEUED, GRPC_STATS_COUNTER_COUNT @@ -132,7 +130,6 @@ typedef enum { GRPC_STATS_HISTOGRAM_HTTP2_SEND_MESSAGE_PER_WRITE, GRPC_STATS_HISTOGRAM_HTTP2_SEND_TRAILING_METADATA_PER_WRITE, GRPC_STATS_HISTOGRAM_HTTP2_SEND_FLOWCTL_PER_WRITE, - GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP, GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED, GRPC_STATS_HISTOGRAM_COUNT } grpc_stats_histograms; @@ -163,11 +160,9 @@ typedef enum { GRPC_STATS_HISTOGRAM_HTTP2_SEND_TRAILING_METADATA_PER_WRITE_BUCKETS = 64, GRPC_STATS_HISTOGRAM_HTTP2_SEND_FLOWCTL_PER_WRITE_FIRST_SLOT = 768, GRPC_STATS_HISTOGRAM_HTTP2_SEND_FLOWCTL_PER_WRITE_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP_FIRST_SLOT = 832, - GRPC_STATS_HISTOGRAM_EXECUTOR_CLOSURES_PER_WAKEUP_BUCKETS = 64, - GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED_FIRST_SLOT = 896, + GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED_FIRST_SLOT = 832, GRPC_STATS_HISTOGRAM_SERVER_CQS_CHECKED_BUCKETS = 8, - GRPC_STATS_HISTOGRAM_BUCKETS = 904 + GRPC_STATS_HISTOGRAM_BUCKETS = 840 } grpc_stats_histogram_constants; #define GRPC_STATS_INC_CLIENT_CALLS_CREATED(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_CLIENT_CALLS_CREATED) @@ -417,11 +412,6 @@ typedef enum { GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_EXECUTOR_QUEUE_DRAINED) #define GRPC_STATS_INC_EXECUTOR_PUSH_RETRIES(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_EXECUTOR_PUSH_RETRIES) -#define GRPC_STATS_INC_EXECUTOR_THREADS_CREATED(exec_ctx) \ - GRPC_STATS_INC_COUNTER((exec_ctx), \ - GRPC_STATS_COUNTER_EXECUTOR_THREADS_CREATED) -#define GRPC_STATS_INC_EXECUTOR_THREADS_USED(exec_ctx) \ - GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_EXECUTOR_THREADS_USED) #define GRPC_STATS_INC_SERVER_REQUESTED_CALLS(exec_ctx) \ GRPC_STATS_INC_COUNTER((exec_ctx), GRPC_STATS_COUNTER_SERVER_REQUESTED_CALLS) #define GRPC_STATS_INC_SERVER_SLOWPATH_REQUESTS_QUEUED(exec_ctx) \ @@ -468,17 +458,13 @@ void grpc_stats_inc_http2_send_trailing_metadata_per_write( grpc_stats_inc_http2_send_flowctl_per_write((exec_ctx), (int)(value)) void grpc_stats_inc_http2_send_flowctl_per_write(grpc_exec_ctx *exec_ctx, int x); -#define GRPC_STATS_INC_EXECUTOR_CLOSURES_PER_WAKEUP(exec_ctx, value) \ - grpc_stats_inc_executor_closures_per_wakeup((exec_ctx), (int)(value)) -void grpc_stats_inc_executor_closures_per_wakeup(grpc_exec_ctx *exec_ctx, - int x); #define GRPC_STATS_INC_SERVER_CQS_CHECKED(exec_ctx, value) \ grpc_stats_inc_server_cqs_checked((exec_ctx), (int)(value)) void grpc_stats_inc_server_cqs_checked(grpc_exec_ctx *exec_ctx, int x); -extern const int grpc_stats_histo_buckets[14]; -extern const int grpc_stats_histo_start[14]; -extern const int *const grpc_stats_histo_bucket_boundaries[14]; -extern void (*const grpc_stats_inc_histogram[14])(grpc_exec_ctx *exec_ctx, +extern const int grpc_stats_histo_buckets[13]; +extern const int grpc_stats_histo_start[13]; +extern const int *const grpc_stats_histo_bucket_boundaries[13]; +extern void (*const grpc_stats_inc_histogram[13])(grpc_exec_ctx *exec_ctx, int x); #endif /* GRPC_CORE_LIB_DEBUG_STATS_DATA_H */ diff --git a/src/core/lib/debug/stats_data.yaml b/src/core/lib/debug/stats_data.yaml index de575f01c73..b5c15ff55c0 100644 --- a/src/core/lib/debug/stats_data.yaml +++ b/src/core/lib/debug/stats_data.yaml @@ -259,14 +259,6 @@ - counter: executor_push_retries doc: Number of times we raced and were forced to retry pushing a closure to the executor -- counter: executor_threads_created - doc: Size of the backing thread pool for overflow gRPC Core work -- counter: executor_threads_used - doc: How many executor threads actually got used -- histogram: executor_closures_per_wakeup - max: 1024 - buckets: 64 - doc: Number of closures executed each time an executor wakes up # server - counter: server_requested_calls doc: How many calls were requested (not necessarily received) by the server diff --git a/src/core/lib/debug/stats_data_bq_schema.sql b/src/core/lib/debug/stats_data_bq_schema.sql index 0611ccaff0e..f96e40c00ef 100644 --- a/src/core/lib/debug/stats_data_bq_schema.sql +++ b/src/core/lib/debug/stats_data_bq_schema.sql @@ -84,7 +84,5 @@ executor_scheduled_to_self_per_iteration:FLOAT, executor_wakeup_initiated_per_iteration:FLOAT, executor_queue_drained_per_iteration:FLOAT, executor_push_retries_per_iteration:FLOAT, -executor_threads_created_per_iteration:FLOAT, -executor_threads_used_per_iteration:FLOAT, server_requested_calls_per_iteration:FLOAT, server_slowpath_requests_queued_per_iteration:FLOAT diff --git a/src/core/lib/iomgr/executor.c b/src/core/lib/iomgr/executor.c index 2439f15a8a9..892385d7d78 100644 --- a/src/core/lib/iomgr/executor.c +++ b/src/core/lib/iomgr/executor.c @@ -32,14 +32,16 @@ #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/support/spinlock.h" +#define MAX_DEPTH 2 + typedef struct { gpr_mu mu; gpr_cv cv; grpc_closure_list elems; + size_t depth; bool shutdown; bool queued_long_job; gpr_thd_id id; - grpc_closure_list local_elems; } thread_state; static thread_state *g_thread_state; @@ -54,35 +56,32 @@ static grpc_tracer_flag executor_trace = static void executor_thread(void *arg); -static void run_closures(grpc_exec_ctx *exec_ctx, grpc_closure_list *list) { - int n = 0; // number of closures executed +static size_t run_closures(grpc_exec_ctx *exec_ctx, grpc_closure_list list) { + size_t n = 0; - while (!grpc_closure_list_empty(*list)) { - grpc_closure *c = list->head; - grpc_closure_list_init(list); - while (c != NULL) { - grpc_closure *next = c->next_data.next; - grpc_error *error = c->error_data.error; - if (GRPC_TRACER_ON(executor_trace)) { + grpc_closure *c = list.head; + while (c != NULL) { + grpc_closure *next = c->next_data.next; + grpc_error *error = c->error_data.error; + if (GRPC_TRACER_ON(executor_trace)) { #ifndef NDEBUG - gpr_log(GPR_DEBUG, "EXECUTOR: run %p [created by %s:%d]", c, - c->file_created, c->line_created); + gpr_log(GPR_DEBUG, "EXECUTOR: run %p [created by %s:%d]", c, + c->file_created, c->line_created); #else - gpr_log(GPR_DEBUG, "EXECUTOR: run %p", c); + gpr_log(GPR_DEBUG, "EXECUTOR: run %p", c); #endif - } + } #ifndef NDEBUG - c->scheduled = false; + c->scheduled = false; #endif - n++; - c->cb(exec_ctx, c->cb_arg, error); - GRPC_ERROR_UNREF(error); - c = next; - grpc_exec_ctx_flush(exec_ctx); - } + c->cb(exec_ctx, c->cb_arg, error); + GRPC_ERROR_UNREF(error); + c = next; + n++; + grpc_exec_ctx_flush(exec_ctx); } - GRPC_STATS_INC_EXECUTOR_CLOSURES_PER_WAKEUP(exec_ctx, n); + return n; } bool grpc_executor_is_threaded() { @@ -127,7 +126,7 @@ void grpc_executor_set_threading(grpc_exec_ctx *exec_ctx, bool threading) { for (size_t i = 0; i < g_max_threads; i++) { gpr_mu_destroy(&g_thread_state[i].mu); gpr_cv_destroy(&g_thread_state[i].cv); - run_closures(exec_ctx, &g_thread_state[i].elems); + run_closures(exec_ctx, g_thread_state[i].elems); } gpr_free(g_thread_state); gpr_tls_destroy(&g_this_thread_state); @@ -151,14 +150,14 @@ static void executor_thread(void *arg) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INITIALIZER(0, grpc_never_ready_to_finish, NULL); - GRPC_STATS_INC_EXECUTOR_THREADS_CREATED(&exec_ctx); - - bool used = false; + size_t subtract_depth = 0; for (;;) { if (GRPC_TRACER_ON(executor_trace)) { - gpr_log(GPR_DEBUG, "EXECUTOR[%d]: step", (int)(ts - g_thread_state)); + gpr_log(GPR_DEBUG, "EXECUTOR[%d]: step (sub_depth=%" PRIdPTR ")", + (int)(ts - g_thread_state), subtract_depth); } gpr_mu_lock(&ts->mu); + ts->depth -= subtract_depth; while (grpc_closure_list_empty(ts->elems) && !ts->shutdown) { ts->queued_long_job = false; gpr_cv_wait(&ts->cv, &ts->mu, gpr_inf_future(GPR_CLOCK_REALTIME)); @@ -171,20 +170,15 @@ static void executor_thread(void *arg) { gpr_mu_unlock(&ts->mu); break; } - if (!used) { - GRPC_STATS_INC_EXECUTOR_THREADS_USED(&exec_ctx); - used = true; - } GRPC_STATS_INC_EXECUTOR_QUEUE_DRAINED(&exec_ctx); - GPR_ASSERT(grpc_closure_list_empty(ts->local_elems)); - ts->local_elems = ts->elems; + grpc_closure_list exec = ts->elems; ts->elems = (grpc_closure_list)GRPC_CLOSURE_LIST_INIT; gpr_mu_unlock(&ts->mu); if (GRPC_TRACER_ON(executor_trace)) { gpr_log(GPR_DEBUG, "EXECUTOR[%d]: execute", (int)(ts - g_thread_state)); } - run_closures(&exec_ctx, &ts->local_elems); + subtract_depth = run_closures(&exec_ctx, exec); } grpc_exec_ctx_finish(&exec_ctx); } @@ -217,10 +211,6 @@ static void executor_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure, ts = &g_thread_state[GPR_HASH_POINTER(exec_ctx, cur_thread_count)]; } else { GRPC_STATS_INC_EXECUTOR_SCHEDULED_TO_SELF(exec_ctx); - if (is_short) { - grpc_closure_list_append(&ts->local_elems, closure, error); - return; - } } thread_state *orig_ts = ts; @@ -260,7 +250,8 @@ static void executor_push(grpc_exec_ctx *exec_ctx, grpc_closure *closure, gpr_cv_signal(&ts->cv); } grpc_closure_list_append(&ts->elems, closure, error); - try_new_thread = ts->elems.head != closure && + ts->depth++; + try_new_thread = ts->depth > MAX_DEPTH && cur_thread_count < g_max_threads && !ts->shutdown; if (!is_short) ts->queued_long_job = true; gpr_mu_unlock(&ts->mu); diff --git a/src/core/lib/security/credentials/composite/composite_credentials.c b/src/core/lib/security/credentials/composite/composite_credentials.c index 09fd60a12c2..b67ff48d0f4 100644 --- a/src/core/lib/security/credentials/composite/composite_credentials.c +++ b/src/core/lib/security/credentials/composite/composite_credentials.c @@ -87,6 +87,7 @@ static bool composite_call_get_request_metadata( ctx->on_request_metadata = on_request_metadata; GRPC_CLOSURE_INIT(&ctx->internal_on_request_metadata, composite_call_metadata_cb, ctx, grpc_schedule_on_exec_ctx); + bool synchronous = true; while (ctx->creds_index < ctx->composite_creds->inner.num_creds) { grpc_call_credentials *inner_creds = ctx->composite_creds->inner.creds_array[ctx->creds_index++]; @@ -95,19 +96,12 @@ static bool composite_call_get_request_metadata( ctx->md_array, &ctx->internal_on_request_metadata, error)) { if (*error != GRPC_ERROR_NONE) break; } else { + synchronous = false; // Async return. break; } } - // If we got through all creds synchronously or we got a synchronous - // error on one of them, return synchronously. - if (ctx->creds_index == ctx->composite_creds->inner.num_creds || - *error != GRPC_ERROR_NONE) { - gpr_free(ctx); - return true; - } - // At least one inner cred is returning asynchronously, so we'll - // return asynchronously as well. - return false; + if (synchronous) gpr_free(ctx); + return synchronous; } static void composite_call_cancel_get_request_metadata( diff --git a/src/cpp/common/channel_arguments.cc b/src/cpp/common/channel_arguments.cc index f130aecd4b5..f89f5f1f03d 100644 --- a/src/cpp/common/channel_arguments.cc +++ b/src/cpp/common/channel_arguments.cc @@ -86,6 +86,10 @@ void ChannelArguments::SetCompressionAlgorithm( SetInt(GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM, algorithm); } +void ChannelArguments::SetGrpclbFallbackTimeout(int fallback_timeout) { + SetInt(GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS, fallback_timeout); +} + void ChannelArguments::SetSocketMutator(grpc_socket_mutator* mutator) { if (!mutator) { return; diff --git a/src/python/grpcio/support.py b/src/python/grpcio/support.py index 510bf422a07..f2395eb26c2 100644 --- a/src/python/grpcio/support.py +++ b/src/python/grpcio/support.py @@ -94,7 +94,7 @@ def diagnose_attribute_error(build_ext, error): _ERROR_DIAGNOSES = { errors.CompileError: diagnose_compile_error, - AttributeError: diagnose_attribute_error + AttributeError: diagnose_attribute_error, } @@ -102,8 +102,10 @@ def diagnose_build_ext_error(build_ext, error, formatted): diagnostic = _ERROR_DIAGNOSES.get(type(error)) if diagnostic is None: raise commands.CommandError( - "\n\nWe could not diagnose your build failure. Please file an issue at " - "http://www.github.com/grpc/grpc with `[Python install]` in the title." - "\n\n{}".format(formatted)) + "\n\nWe could not diagnose your build failure. If you are unable to " + "proceed, please file an issue at http://www.github.com/grpc/grpc " + "with `[Python install]` in the title; please attach the whole log " + "(including everything that may have appeared above the Python " + "backtrace).\n\n{}".format(formatted)) else: diagnostic(build_ext, error) diff --git a/src/ruby/lib/grpc/google_rpc_status_utils.rb b/src/ruby/lib/grpc/google_rpc_status_utils.rb index fdadd6b76e4..f253b082b63 100644 --- a/src/ruby/lib/grpc/google_rpc_status_utils.rb +++ b/src/ruby/lib/grpc/google_rpc_status_utils.rb @@ -19,10 +19,17 @@ require 'google/rpc/status_pb' module GRPC # GoogleRpcStatusUtils provides utilities to convert between a # GRPC::Core::Status and a deserialized Google::Rpc::Status proto + # Returns nil if the grpc-status-details-bin trailer could not be + # converted to a GoogleRpcStatus due to the server not providing + # the necessary trailers. + # Raises an error if the server did provide the necessary trailers + # but they fail to deseriliaze into a GoogleRpcStatus protobuf. class GoogleRpcStatusUtils def self.extract_google_rpc_status(status) fail ArgumentError, 'bad type' unless status.is_a? Struct::Status - Google::Rpc::Status.decode(status.metadata['grpc-status-details-bin']) + grpc_status_details_bin_trailer = 'grpc-status-details-bin' + return nil if status.metadata[grpc_status_details_bin_trailer].nil? + Google::Rpc::Status.decode(status.metadata[grpc_status_details_bin_trailer]) end end end diff --git a/src/ruby/spec/google_rpc_status_utils_spec.rb b/src/ruby/spec/google_rpc_status_utils_spec.rb index fe221c30ddd..6f2a06b1d93 100644 --- a/src/ruby/spec/google_rpc_status_utils_spec.rb +++ b/src/ruby/spec/google_rpc_status_utils_spec.rb @@ -31,12 +31,11 @@ describe 'conversion from a status struct to a google protobuf status' do expect(exception.message.include?('bad type')).to be true end - it 'fails with some error if the header key is missing' do + it 'returns nil if the header key is missing' do status = Struct::Status.new(1, 'details', key: 'val') expect(status.metadata.nil?).to be false - expect do - GRPC::GoogleRpcStatusUtils.extract_google_rpc_status(status) - end.to raise_error(StandardError) + expect(GRPC::GoogleRpcStatusUtils.extract_google_rpc_status( + status)).to be(nil) end it 'fails with some error if the header key fails to deserialize' do @@ -221,3 +220,73 @@ describe 'receving a google rpc status from a remote endpoint' do status_from_exception)).to eq(rpc_status) end end + +# A test service that fails without explicitly setting the +# grpc-status-details-bin trailer. Tests assumptions about value +# of grpc-status-details-bin on the client side when the trailer wasn't +# set explicitly. +class NoStatusDetailsBinTestService + include GRPC::GenericService + rpc :an_rpc, EchoMsg, EchoMsg + + def an_rpc(_, _) + fail GRPC::Unknown + end +end + +NoStatusDetailsBinTestServiceStub = NoStatusDetailsBinTestService.rpc_stub_class + +describe 'when the endpoint doesnt send grpc-status-details-bin' do + def start_server + @srv = GRPC::RpcServer.new(pool_size: 1) + @server_port = @srv.add_http2_port('localhost:0', + :this_port_is_insecure) + @srv.handle(NoStatusDetailsBinTestService) + @server_thd = Thread.new { @srv.run } + @srv.wait_till_running + end + + def stop_server + expect(@srv.stopped?).to be(false) + @srv.stop + @server_thd.join + expect(@srv.stopped?).to be(true) + end + + before(:each) do + start_server + end + + after(:each) do + stop_server + end + + it 'should receive nil when we extract try to extract a google '\ + 'rpc status from a BadStatus exception that didnt have it' do + stub = NoStatusDetailsBinTestServiceStub.new("localhost:#{@server_port}", + :this_channel_is_insecure) + begin + stub.an_rpc(EchoMsg.new) + rescue GRPC::Unknown => e + rpc_status = GRPC::GoogleRpcStatusUtils.extract_google_rpc_status( + e.to_status) + end + expect(rpc_status).to be(nil) + end + + it 'should receive nil when we extract try to extract a google '\ + 'rpc status from an op views status object that didnt have it' do + stub = NoStatusDetailsBinTestServiceStub.new("localhost:#{@server_port}", + :this_channel_is_insecure) + op = stub.an_rpc(EchoMsg.new, return_op: true) + begin + op.execute + rescue GRPC::Unknown => e + status_from_exception = e.to_status + end + expect(GRPC::GoogleRpcStatusUtils.extract_google_rpc_status( + status_from_exception)).to be(nil) + expect(GRPC::GoogleRpcStatusUtils.extract_google_rpc_status( + op.status)).to be nil + end +end diff --git a/templates/test/cpp/naming/create_private_dns_zone.sh.template b/templates/test/cpp/naming/create_private_dns_zone.sh.template new file mode 100644 index 00000000000..14324b098c5 --- /dev/null +++ b/templates/test/cpp/naming/create_private_dns_zone.sh.template @@ -0,0 +1,4 @@ +%YAML 1.2 +--- | + <%namespace file="create_private_dns_zone_defs.include" import="*"/>\ + ${create_private_dns_zone(resolver_gce_integration_tests_zone_id, resolver_tests_common_zone_name)} diff --git a/templates/test/cpp/naming/create_private_dns_zone_defs.include b/templates/test/cpp/naming/create_private_dns_zone_defs.include new file mode 100644 index 00000000000..465dd6394bf --- /dev/null +++ b/templates/test/cpp/naming/create_private_dns_zone_defs.include @@ -0,0 +1,32 @@ +<%def name="create_private_dns_zone(resolver_gce_integration_tests_zone_id, resolver_tests_common_zone_name)">#!/bin/bash +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is auto-generated + +set -ex + +cd $(dirname $0)/../../.. + +gcloud alpha dns managed-zones create \\ + + ${resolver_gce_integration_tests_zone_id} \\ + + --dns-name=${resolver_tests_common_zone_name} \\ + + --description="GCE-DNS-private-zone-for-GRPC-testing" \\ + + --visibility=private \\ + + --networks=default diff --git a/templates/test/cpp/naming/private_dns_zone_init.sh.template b/templates/test/cpp/naming/private_dns_zone_init.sh.template new file mode 100644 index 00000000000..d5ffd04add1 --- /dev/null +++ b/templates/test/cpp/naming/private_dns_zone_init.sh.template @@ -0,0 +1,4 @@ +%YAML 1.2 +--- | + <%namespace file="private_dns_zone_init_defs.include" import="*"/>\ + ${private_dns_zone_init(all_integration_test_records, resolver_gce_integration_tests_zone_id, resolver_tests_common_zone_name)} diff --git a/templates/test/cpp/naming/private_dns_zone_init_defs.include b/templates/test/cpp/naming/private_dns_zone_init_defs.include new file mode 100644 index 00000000000..06bc8adb94e --- /dev/null +++ b/templates/test/cpp/naming/private_dns_zone_init_defs.include @@ -0,0 +1,40 @@ +<%def name="private_dns_zone_init(records,resolver_gce_integration_tests_zone_id,resolver_tests_common_zone_name)">#!/bin/bash +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is auto-generated + +set -ex + +cd $(dirname $0)/../../.. + +gcloud dns record-sets transaction start -z=${resolver_gce_integration_tests_zone_id} + +% for r in records: +gcloud dns record-sets transaction add \\ + + -z=${resolver_gce_integration_tests_zone_id} \\ + + --name=${r['name']}.${resolver_tests_common_zone_name} \\ + + --type=${r['type']} \\ + + --ttl=${r['ttl']} \\ + + ${r['data']} + +% endfor +gcloud dns record-sets transaction describe -z=${resolver_gce_integration_tests_zone_id} +gcloud dns record-sets transaction execute -z=${resolver_gce_integration_tests_zone_id} +gcloud dns record-sets list -z=${resolver_gce_integration_tests_zone_id} diff --git a/templates/test/cpp/naming/resolver_gce_integration_tests_defs.include b/templates/test/cpp/naming/resolver_gce_integration_tests_defs.include new file mode 100644 index 00000000000..2413ec57d09 --- /dev/null +++ b/templates/test/cpp/naming/resolver_gce_integration_tests_defs.include @@ -0,0 +1,64 @@ +<%def name="resolver_gce_integration_tests(tests, records, resolver_tests_common_zone_name)">#!/bin/bash +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is auto-generated + +set -ex + +if [[ "$GRPC_DNS_RESOLVER" == "" ]]; then + export GRPC_DNS_RESOLVER=ares +elif [[ "$GRPC_DNS_RESOLVER" != ares ]]; then + echo "Unexpected: GRPC_DNS_RESOLVER=$GRPC_DNS_RESOLVER. This test only works with c-ares resolver" + exit 1 +fi + +cd $(dirname $0)/../../.. + +if [[ "$CONFIG" == "" ]]; then + export CONFIG=opt +fi +make resolver_component_test +echo "Sanity check DNS records are resolveable with dig:" +EXIT_CODE=0 + +% for r in records: +ONE_FAILED=0 +dig ${r['type']} ${r['name']}.${resolver_tests_common_zone_name} | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig ${r['type']} ${r['name']}.${resolver_tests_common_zone_name} FAILED" + exit 1 +fi + +% endfor +echo "Sanity check PASSED. Run resolver tests:" + +% for test in tests: +ONE_FAILED=0 +bins/$CONFIG/resolver_component_test \\ + + --target_name='${test['target_name']}' \\ + + --expected_addrs='${test['expected_addrs']}' \\ + + --expected_chosen_service_config='${test['expected_chosen_service_config']}' \\ + + --expected_lb_policy='${test['expected_lb_policy']}' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Test based on target record: ${test['target_name']} FAILED" + EXIT_CODE=1 +fi + +% endfor +exit $EXIT_CODE diff --git a/templates/test/cpp/naming/resolver_gce_integration_tests_runner.sh.template b/templates/test/cpp/naming/resolver_gce_integration_tests_runner.sh.template new file mode 100644 index 00000000000..c728784d29a --- /dev/null +++ b/templates/test/cpp/naming/resolver_gce_integration_tests_runner.sh.template @@ -0,0 +1,4 @@ +%YAML 1.2 +--- | + <%namespace file="resolver_gce_integration_tests_defs.include" import="*"/>\ + ${resolver_gce_integration_tests(resolver_gce_integration_test_cases, all_integration_test_records, resolver_tests_common_zone_name)} diff --git a/test/core/bad_client/bad_client.c b/test/core/bad_client/bad_client.c index 383d1240cb0..fff0c793ed6 100644 --- a/test/core/bad_client/bad_client.c +++ b/test/core/bad_client/bad_client.c @@ -134,9 +134,12 @@ void grpc_run_bad_client_test( grpc_endpoint_write(&exec_ctx, sfd.client, &outgoing, &done_write_closure); grpc_exec_ctx_finish(&exec_ctx); - /* Await completion */ - GPR_ASSERT( - gpr_event_wait(&a.done_write, grpc_timeout_seconds_to_deadline(5))); + /* Await completion, unless the request is large and write may not finish + * before the peer shuts down. */ + if (!(flags & GRPC_BAD_CLIENT_LARGE_REQUEST)) { + GPR_ASSERT( + gpr_event_wait(&a.done_write, grpc_timeout_seconds_to_deadline(5))); + } if (flags & GRPC_BAD_CLIENT_DISCONNECT) { grpc_endpoint_shutdown( @@ -186,6 +189,8 @@ void grpc_run_bad_client_test( grpc_exec_ctx_finish(&exec_ctx); } + GPR_ASSERT( + gpr_event_wait(&a.done_write, grpc_timeout_seconds_to_deadline(1))); shutdown_cq = grpc_completion_queue_create_for_pluck(NULL); grpc_server_shutdown_and_notify(a.server, shutdown_cq, NULL); GPR_ASSERT(grpc_completion_queue_pluck( diff --git a/test/core/bad_client/bad_client.h b/test/core/bad_client/bad_client.h index 22f1a3abc7a..a5b01f7f2c1 100644 --- a/test/core/bad_client/bad_client.h +++ b/test/core/bad_client/bad_client.h @@ -37,6 +37,7 @@ typedef bool (*grpc_bad_client_client_stream_validator)( grpc_slice_buffer *incoming); #define GRPC_BAD_CLIENT_DISCONNECT 1 +#define GRPC_BAD_CLIENT_LARGE_REQUEST 2 /* Test runner. diff --git a/test/core/bad_client/tests/window_overflow.c b/test/core/bad_client/tests/window_overflow.c index 1f29bd32fbd..18c647ad8a7 100644 --- a/test/core/bad_client/tests/window_overflow.c +++ b/test/core/bad_client/tests/window_overflow.c @@ -90,7 +90,8 @@ int main(int argc, char **argv) { addbuf(message, sizeof(message)); } } - grpc_run_bad_client_test(verifier, NULL, g_buffer, g_count, 0); + grpc_run_bad_client_test(verifier, NULL, g_buffer, g_count, + GRPC_BAD_CLIENT_LARGE_REQUEST); gpr_free(g_buffer); return 0; diff --git a/test/core/iomgr/pollset_set_test.c b/test/core/iomgr/pollset_set_test.c index 70efca8b16f..5750ac0f4b3 100644 --- a/test/core/iomgr/pollset_set_test.c +++ b/test/core/iomgr/pollset_set_test.c @@ -24,7 +24,6 @@ #include #include -#include #include #include #include @@ -434,7 +433,8 @@ int main(int argc, char **argv) { const char *poll_strategy = grpc_get_poll_strategy_name(); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_test_init(argc, argv); - grpc_init(); + grpc_iomgr_init(&exec_ctx); + grpc_iomgr_start(&exec_ctx); if (poll_strategy != NULL && (strcmp(poll_strategy, "epoll") == 0 || @@ -449,8 +449,8 @@ int main(int argc, char **argv) { poll_strategy); } + grpc_iomgr_shutdown(&exec_ctx); grpc_exec_ctx_finish(&exec_ctx); - grpc_shutdown(); return 0; } #else /* defined(GRPC_LINUX_EPOLL) */ diff --git a/test/core/tsi/transport_security_test_lib.c b/test/core/tsi/transport_security_test_lib.c index 7d66e110f2e..329b2371bf1 100644 --- a/test/core/tsi/transport_security_test_lib.c +++ b/test/core/tsi/transport_security_test_lib.c @@ -23,9 +23,26 @@ #include #include #include +#include #include "src/core/lib/security/transport/tsi_error.h" #include "test/core/tsi/transport_security_test_lib.h" +static void notification_signal(tsi_test_fixture *fixture) { + gpr_mu_lock(&fixture->mu); + fixture->notified = true; + gpr_cv_signal(&fixture->cv); + gpr_mu_unlock(&fixture->mu); +} + +static void notification_wait(tsi_test_fixture *fixture) { + gpr_mu_lock(&fixture->mu); + while (!fixture->notified) { + gpr_cv_wait(&fixture->cv, &fixture->mu, gpr_inf_future(GPR_CLOCK_REALTIME)); + } + fixture->notified = false; + gpr_mu_unlock(&fixture->mu); +} + typedef struct handshaker_args { tsi_test_fixture *fixture; unsigned char *handshake_buffer; @@ -273,9 +290,11 @@ grpc_error *on_handshake_next_done(tsi_result result, void *user_data, /* Read more data if we need to. */ if (result == TSI_INCOMPLETE_DATA) { GPR_ASSERT(bytes_to_send_size == 0); + notification_signal(fixture); return error; } if (result != TSI_OK) { + notification_signal(fixture); return grpc_set_tsi_error_result( GRPC_ERROR_CREATE_FROM_STATIC_STRING("Handshake failed"), result); } @@ -295,6 +314,7 @@ grpc_error *on_handshake_next_done(tsi_result result, void *user_data, if (handshaker_result != NULL) { maybe_append_unused_bytes(args); } + notification_signal(fixture); return error; } @@ -345,7 +365,11 @@ static void do_handshaker_next(handshaker_args *args) { if (result != TSI_ASYNC) { args->error = on_handshake_next_done(result, args, bytes_to_send, bytes_to_send_size, handshaker_result); + if (args->error != GRPC_ERROR_NONE) { + return; + } } + notification_wait(fixture); } void tsi_test_do_handshake(tsi_test_fixture *fixture) { @@ -532,6 +556,9 @@ void tsi_test_fixture_init(tsi_test_fixture *fixture) { fixture->bytes_read_from_server_channel = 0; fixture->test_unused_bytes = true; fixture->has_client_finished_first = false; + gpr_mu_init(&fixture->mu); + gpr_cv_init(&fixture->cv); + fixture->notified = false; } void tsi_test_fixture_destroy(tsi_test_fixture *fixture) { @@ -546,5 +573,7 @@ void tsi_test_fixture_destroy(tsi_test_fixture *fixture) { GPR_ASSERT(fixture->vtable != NULL); GPR_ASSERT(fixture->vtable->destruct != NULL); fixture->vtable->destruct(fixture); + gpr_mu_destroy(&fixture->mu); + gpr_cv_destroy(&fixture->cv); gpr_free(fixture); } diff --git a/test/core/tsi/transport_security_test_lib.h b/test/core/tsi/transport_security_test_lib.h index 8ae2024ee49..ed8ff856dfa 100644 --- a/test/core/tsi/transport_security_test_lib.h +++ b/test/core/tsi/transport_security_test_lib.h @@ -21,6 +21,10 @@ #include "src/core/tsi/transport_security_interface.h" +#ifdef __cplusplus +extern "C" { +#endif + #define TSI_TEST_TINY_HANDSHAKE_BUFFER_SIZE 32 #define TSI_TEST_SMALL_HANDSHAKE_BUFFER_SIZE 128 #define TSI_TEST_SMALL_READ_BUFFER_ALLOCATED_SIZE 41 @@ -56,10 +60,10 @@ typedef struct tsi_test_fixture_vtable { void (*setup_handshakers)(tsi_test_fixture *fixture); void (*check_handshaker_peers)(tsi_test_fixture *fixture); void (*destruct)(tsi_test_fixture *fixture); -} tranport_security_test_vtable; +} tsi_test_fixture_vtable; struct tsi_test_fixture { - const struct tsi_test_fixture_vtable *vtable; + const tsi_test_fixture_vtable *vtable; /* client/server TSI handshaker used to perform TSI handshakes, and will get instantiated during the call to setup_handshakers. */ tsi_handshaker *client_handshaker; @@ -95,6 +99,13 @@ struct tsi_test_fixture { (https://github.com/grpc/grpc/issues/12164). */ bool test_unused_bytes; + /* These objects will be used coordinate client/server handshakers with TSI + thread to perform TSI handshakes in an asynchronous manner (for GTS TSI + implementations). + */ + gpr_cv cv; + gpr_mu mu; + bool notified; }; struct tsi_test_frame_protector_config { @@ -162,4 +173,8 @@ void tsi_test_do_handshake(tsi_test_fixture *fixture); the client and server switching its role. */ void tsi_test_do_round_trip(tsi_test_fixture *fixture); +#ifdef __cplusplus +} +#endif + #endif // GRPC_TEST_CORE_TSI_TRANSPORT_SECURITY_TEST_LIB_H_ diff --git a/test/cpp/end2end/grpclb_end2end_test.cc b/test/cpp/end2end/grpclb_end2end_test.cc index 77ed1552923..f73a9c17917 100644 --- a/test/cpp/end2end/grpclb_end2end_test.cc +++ b/test/cpp/end2end/grpclb_end2end_test.cc @@ -368,8 +368,9 @@ class GrpclbEnd2endTest : public ::testing::Test { grpc_fake_resolver_response_generator_unref(response_generator_); } - void ResetStub() { + void ResetStub(int fallback_timeout = 0) { ChannelArguments args; + args.SetGrpclbFallbackTimeout(fallback_timeout); args.SetPointer(GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR, response_generator_); std::ostringstream uri; @@ -470,10 +471,10 @@ class GrpclbEnd2endTest : public ::testing::Test { grpc_exec_ctx_finish(&exec_ctx); } - const std::vector GetBackendPorts() const { + const std::vector GetBackendPorts(const size_t start_index = 0) const { std::vector backend_ports; - for (const auto& bs : backend_servers_) { - backend_ports.push_back(bs.port_); + for (size_t i = start_index; i < backend_servers_.size(); ++i) { + backend_ports.push_back(backend_servers_[i].port_); } return backend_ports; } @@ -642,6 +643,177 @@ TEST_F(SingleBalancerTest, InitiallyEmptyServerlist) { EXPECT_EQ("grpclb", channel_->GetLoadBalancingPolicyName()); } +TEST_F(SingleBalancerTest, Fallback) { + const int kFallbackTimeoutMs = 200 * grpc_test_slowdown_factor(); + const int kServerlistDelayMs = 500 * grpc_test_slowdown_factor(); + const size_t kNumBackendInResolution = backends_.size() / 2; + + ResetStub(kFallbackTimeoutMs); + std::vector addresses; + addresses.emplace_back(AddressData{balancer_servers_[0].port_, true, ""}); + for (size_t i = 0; i < kNumBackendInResolution; ++i) { + addresses.emplace_back(AddressData{backend_servers_[i].port_, false, ""}); + } + SetNextResolution(addresses); + + // Send non-empty serverlist only after kServerlistDelayMs. + ScheduleResponseForBalancer( + 0, BalancerServiceImpl::BuildResponseForBackends( + GetBackendPorts(kNumBackendInResolution /* start_index */), {}), + kServerlistDelayMs); + + // Wait until all the fallback backends are reachable. + for (size_t i = 0; i < kNumBackendInResolution; ++i) { + WaitForBackend(i); + } + + // The first request. + gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH =========="); + CheckRpcSendOk(kNumBackendInResolution); + gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH =========="); + + // Fallback is used: each backend returned by the resolver should have + // gotten one request. + for (size_t i = 0; i < kNumBackendInResolution; ++i) { + EXPECT_EQ(1U, backend_servers_[i].service_->request_count()); + } + for (size_t i = kNumBackendInResolution; i < backends_.size(); ++i) { + EXPECT_EQ(0U, backend_servers_[i].service_->request_count()); + } + + // Wait until the serverlist reception has been processed and all backends + // in the serverlist are reachable. + for (size_t i = kNumBackendInResolution; i < backends_.size(); ++i) { + WaitForBackend(i); + } + + // Send out the second request. + gpr_log(GPR_INFO, "========= BEFORE SECOND BATCH =========="); + CheckRpcSendOk(backends_.size() - kNumBackendInResolution); + gpr_log(GPR_INFO, "========= DONE WITH SECOND BATCH =========="); + + // Serverlist is used: each backend returned by the balancer should + // have gotten one request. + for (size_t i = 0; i < kNumBackendInResolution; ++i) { + EXPECT_EQ(0U, backend_servers_[i].service_->request_count()); + } + for (size_t i = kNumBackendInResolution; i < backends_.size(); ++i) { + EXPECT_EQ(1U, backend_servers_[i].service_->request_count()); + } + + balancers_[0]->NotifyDoneWithServerlists(); + // The balancer got a single request. + EXPECT_EQ(1U, balancer_servers_[0].service_->request_count()); + // and sent a single response. + EXPECT_EQ(1U, balancer_servers_[0].service_->response_count()); +} + +TEST_F(SingleBalancerTest, FallbackUpdate) { + const int kFallbackTimeoutMs = 200 * grpc_test_slowdown_factor(); + const int kServerlistDelayMs = 500 * grpc_test_slowdown_factor(); + const size_t kNumBackendInResolution = backends_.size() / 3; + const size_t kNumBackendInResolutionUpdate = backends_.size() / 3; + + ResetStub(kFallbackTimeoutMs); + std::vector addresses; + addresses.emplace_back(AddressData{balancer_servers_[0].port_, true, ""}); + for (size_t i = 0; i < kNumBackendInResolution; ++i) { + addresses.emplace_back(AddressData{backend_servers_[i].port_, false, ""}); + } + SetNextResolution(addresses); + + // Send non-empty serverlist only after kServerlistDelayMs. + ScheduleResponseForBalancer( + 0, BalancerServiceImpl::BuildResponseForBackends( + GetBackendPorts(kNumBackendInResolution + + kNumBackendInResolutionUpdate /* start_index */), + {}), + kServerlistDelayMs); + + // Wait until all the fallback backends are reachable. + for (size_t i = 0; i < kNumBackendInResolution; ++i) { + WaitForBackend(i); + } + + // The first request. + gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH =========="); + CheckRpcSendOk(kNumBackendInResolution); + gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH =========="); + + // Fallback is used: each backend returned by the resolver should have + // gotten one request. + for (size_t i = 0; i < kNumBackendInResolution; ++i) { + EXPECT_EQ(1U, backend_servers_[i].service_->request_count()); + } + for (size_t i = kNumBackendInResolution; i < backends_.size(); ++i) { + EXPECT_EQ(0U, backend_servers_[i].service_->request_count()); + } + + addresses.clear(); + addresses.emplace_back(AddressData{balancer_servers_[0].port_, true, ""}); + for (size_t i = kNumBackendInResolution; + i < kNumBackendInResolution + kNumBackendInResolutionUpdate; ++i) { + addresses.emplace_back(AddressData{backend_servers_[i].port_, false, ""}); + } + SetNextResolution(addresses); + + // Wait until the resolution update has been processed and all the new + // fallback backends are reachable. + for (size_t i = kNumBackendInResolution; + i < kNumBackendInResolution + kNumBackendInResolutionUpdate; ++i) { + WaitForBackend(i); + } + + // Send out the second request. + gpr_log(GPR_INFO, "========= BEFORE SECOND BATCH =========="); + CheckRpcSendOk(kNumBackendInResolutionUpdate); + gpr_log(GPR_INFO, "========= DONE WITH SECOND BATCH =========="); + + // The resolution update is used: each backend in the resolution update should + // have gotten one request. + for (size_t i = 0; i < kNumBackendInResolution; ++i) { + EXPECT_EQ(0U, backend_servers_[i].service_->request_count()); + } + for (size_t i = kNumBackendInResolution; + i < kNumBackendInResolution + kNumBackendInResolutionUpdate; ++i) { + EXPECT_EQ(1U, backend_servers_[i].service_->request_count()); + } + for (size_t i = kNumBackendInResolution + kNumBackendInResolutionUpdate; + i < backends_.size(); ++i) { + EXPECT_EQ(0U, backend_servers_[i].service_->request_count()); + } + + // Wait until the serverlist reception has been processed and all backends + // in the serverlist are reachable. + for (size_t i = kNumBackendInResolution + kNumBackendInResolutionUpdate; + i < backends_.size(); ++i) { + WaitForBackend(i); + } + + // Send out the third request. + gpr_log(GPR_INFO, "========= BEFORE THIRD BATCH =========="); + CheckRpcSendOk(backends_.size() - kNumBackendInResolution - + kNumBackendInResolutionUpdate); + gpr_log(GPR_INFO, "========= DONE WITH THIRD BATCH =========="); + + // Serverlist is used: each backend returned by the balancer should + // have gotten one request. + for (size_t i = 0; + i < kNumBackendInResolution + kNumBackendInResolutionUpdate; ++i) { + EXPECT_EQ(0U, backend_servers_[i].service_->request_count()); + } + for (size_t i = kNumBackendInResolution + kNumBackendInResolutionUpdate; + i < backends_.size(); ++i) { + EXPECT_EQ(1U, backend_servers_[i].service_->request_count()); + } + + balancers_[0]->NotifyDoneWithServerlists(); + // The balancer got a single request. + EXPECT_EQ(1U, balancer_servers_[0].service_->request_count()); + // and sent a single response. + EXPECT_EQ(1U, balancer_servers_[0].service_->response_count()); +} + TEST_F(SingleBalancerTest, BackendsRestart) { const size_t kNumRpcsPerAddress = 100; ScheduleResponseForBalancer( diff --git a/test/cpp/naming/README.md b/test/cpp/naming/README.md new file mode 100644 index 00000000000..e33184620c0 --- /dev/null +++ b/test/cpp/naming/README.md @@ -0,0 +1,43 @@ +# Resolver Tests + +This directory has tests and infrastructure for unit tests and GCE +integration tests of gRPC resolver functionality. + +There are two different tests here: + +## Resolver unit tests (resolver "component" tests) + +These tests run per-change, along with the rest of the grpc unit tests. +They query a local testing DNS server. + +## GCE integration tests + +These tests use the same test binary and the same test records +as the unit tests, but they run against GCE DNS (this is done by +running the test on a GCE instance and not specifying an authority +in uris). These tests run in a background job, which needs to be +actively monitored. + +## Making changes to test records + +After making a change to `resolver_test_record_groups.yaml`: + +1. Increment the "version number" in the `resolver_tests_common_zone_name` + DNS zone (this is a yaml field at the top + of `resolver_test_record_groups.yaml`). + +2. Regenerate projects. + +3. From the repo root, run: + +``` +$ test/cpp/naming/create_dns_private_zone.sh +$ test/cpp/naming/private_dns_zone_init.sh +``` + +Note that these commands must be ran in environment that +has access to the grpc-testing GCE project. + +If everything runs smoothly, then once the change is merged, +the GCE DNS integration testing job will transition to the +new records and continue passing. diff --git a/test/cpp/naming/create_private_dns_zone.sh b/test/cpp/naming/create_private_dns_zone.sh new file mode 100755 index 00000000000..3d7520b90a8 --- /dev/null +++ b/test/cpp/naming/create_private_dns_zone.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is auto-generated + +set -ex + +cd $(dirname $0)/../../.. + +gcloud alpha dns managed-zones create \ + resolver-tests-version-1-grpctestingexp-zone-id \ + --dns-name=resolver-tests-version-1.grpctestingexp. \ + --description="GCE-DNS-private-zone-for-GRPC-testing" \ + --visibility=private \ + --networks=default diff --git a/test/cpp/naming/gen_build_yaml.py b/test/cpp/naming/gen_build_yaml.py index 3a51fef7a0e..91718156e9a 100755 --- a/test/cpp/naming/gen_build_yaml.py +++ b/test/cpp/naming/gen_build_yaml.py @@ -24,6 +24,12 @@ import json _LOCAL_DNS_SERVER_ADDRESS = '127.0.0.1:15353' +_TARGET_RECORDS_TO_SKIP_AGAINST_GCE = [ + # TODO: enable this once able to upload the very large TXT record + # in this group to GCE DNS. + 'ipv4-config-causing-fallback-to-tcp', +] + def _append_zone_name(name, zone_name): return '%s.%s' % (name, zone_name) @@ -33,21 +39,107 @@ def _build_expected_addrs_cmd_arg(expected_addrs): out.append('%s,%s' % (addr['address'], str(addr['is_balancer']))) return ';'.join(out) +def _data_for_type(r_type, r_data, common_zone_name): + if r_type in ['A', 'AAAA']: + return ' '.join(map(lambda x: '\"%s\"' % x, r_data)) + if r_type == 'SRV': + assert len(r_data) == 1 + target = r_data[0].split(' ')[3] + uploadable_target = '%s.%s' % (target, common_zone_name) + uploadable = r_data[0].split(' ') + uploadable[3] = uploadable_target + return '\"%s\"' % ' '.join(uploadable) + if r_type == 'TXT': + assert len(r_data) == 1 + chunks = [] + all_data = r_data[0] + cur = 0 + # Split TXT records that span more than 255 characters (the single + # string length-limit in DNS) into multiple strings. Each string + # needs to be wrapped with double-quotes, and all inner double-quotes + # are escaped. The wrapping double-quotes and inner backslashes can be + # counted towards the 255 character length limit (as observed with gcloud), + # so make sure all strings fit within that limit. + while len(all_data[cur:]) > 0: + next_chunk = '\"' + while len(next_chunk) < 254 and len(all_data[cur:]) > 0: + if all_data[cur] == '\"': + if len(next_chunk) < 253: + next_chunk += '\\\"' + else: + break + else: + next_chunk += all_data[cur] + cur += 1 + next_chunk += '\"' + if len(next_chunk) > 255: + raise Exception('Bug: next chunk is too long.') + chunks.append(next_chunk) + # Wrap the whole record in single quotes to make sure all strings + # are associated with the same TXT record (to make it one bash token for + # gcloud) + return '\'%s\'' % ' '.join(chunks) + +# Convert DNS records from their "within a test group" format +# of the yaml file to an easier form for the templates to use. +def _gcloud_uploadable_form(test_cases, common_zone_name): + out = [] + for group in test_cases: + if group['record_to_resolve'] in _TARGET_RECORDS_TO_SKIP_AGAINST_GCE: + continue + for record_name in group['records'].keys(): + r_ttl = None + all_r_data = {} + for r_data in group['records'][record_name]: + # enforce records have the same TTL only for simplicity + if r_ttl is None: + r_ttl = r_data['TTL'] + assert r_ttl == r_data['TTL'], '%s and %s differ' % (r_ttl, r_data['TTL']) + r_type = r_data['type'] + if all_r_data.get(r_type) is None: + all_r_data[r_type] = [] + all_r_data[r_type].append(r_data['data']) + for r_type in all_r_data.keys(): + for r in out: + assert r['name'] != record_name or r['type'] != r_type, 'attempt to add a duplicate record' + out.append({ + 'name': record_name, + 'ttl': r_ttl, + 'type': r_type, + 'data': _data_for_type(r_type, all_r_data[r_type], common_zone_name) + }) + return out + +def _gce_dns_zone_id(resolver_component_data): + dns_name = resolver_component_data['resolver_tests_common_zone_name'] + return dns_name.replace('.', '-') + 'zone-id' + +def _resolver_test_cases(resolver_component_data, records_to_skip): + out = [] + for test_case in resolver_component_data['resolver_component_tests']: + if test_case['record_to_resolve'] in records_to_skip: + continue + out.append({ + 'target_name': _append_zone_name(test_case['record_to_resolve'], + resolver_component_data['resolver_tests_common_zone_name']), + 'expected_addrs': _build_expected_addrs_cmd_arg(test_case['expected_addrs']), + 'expected_chosen_service_config': (test_case['expected_chosen_service_config'] or ''), + 'expected_lb_policy': (test_case['expected_lb_policy'] or ''), + }) + return out + def main(): resolver_component_data = '' with open('test/cpp/naming/resolver_test_record_groups.yaml') as f: resolver_component_data = yaml.load(f) json = { - 'resolver_component_test_cases': [ - { - 'target_name': _append_zone_name(test_case['record_to_resolve'], - resolver_component_data['resolver_component_tests_common_zone_name']), - 'expected_addrs': _build_expected_addrs_cmd_arg(test_case['expected_addrs']), - 'expected_chosen_service_config': (test_case['expected_chosen_service_config'] or ''), - 'expected_lb_policy': (test_case['expected_lb_policy'] or ''), - } for test_case in resolver_component_data['resolver_component_tests'] - ], + 'resolver_tests_common_zone_name': resolver_component_data['resolver_tests_common_zone_name'], + 'resolver_gce_integration_tests_zone_id': _gce_dns_zone_id(resolver_component_data), + 'all_integration_test_records': _gcloud_uploadable_form(resolver_component_data['resolver_component_tests'], + resolver_component_data['resolver_tests_common_zone_name']), + 'resolver_gce_integration_test_cases': _resolver_test_cases(resolver_component_data, _TARGET_RECORDS_TO_SKIP_AGAINST_GCE), + 'resolver_component_test_cases': _resolver_test_cases(resolver_component_data, []), 'targets': [ { 'name': 'resolver_component_test' + unsecure_build_config_suffix, diff --git a/test/cpp/naming/private_dns_zone_init.sh b/test/cpp/naming/private_dns_zone_init.sh new file mode 100755 index 00000000000..4eaf750ab71 --- /dev/null +++ b/test/cpp/naming/private_dns_zone_init.sh @@ -0,0 +1,215 @@ +#!/bin/bash +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is auto-generated + +set -ex + +cd $(dirname $0)/../../.. + +gcloud dns record-sets transaction start -z=resolver-tests-version-1-grpctestingexp-zone-id + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=_grpclb._tcp.srv-ipv4-single-target.resolver-tests-version-1.grpctestingexp. \ + --type=SRV \ + --ttl=2100 \ + "0 0 1234 ipv4-single-target.resolver-tests-version-1.grpctestingexp." + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=ipv4-single-target.resolver-tests-version-1.grpctestingexp. \ + --type=A \ + --ttl=2100 \ + "1.2.3.4" + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=_grpclb._tcp.srv-ipv4-multi-target.resolver-tests-version-1.grpctestingexp. \ + --type=SRV \ + --ttl=2100 \ + "0 0 1234 ipv4-multi-target.resolver-tests-version-1.grpctestingexp." + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=ipv4-multi-target.resolver-tests-version-1.grpctestingexp. \ + --type=A \ + --ttl=2100 \ + "1.2.3.5" "1.2.3.6" "1.2.3.7" + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=_grpclb._tcp.srv-ipv6-single-target.resolver-tests-version-1.grpctestingexp. \ + --type=SRV \ + --ttl=2100 \ + "0 0 1234 ipv6-single-target.resolver-tests-version-1.grpctestingexp." + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=ipv6-single-target.resolver-tests-version-1.grpctestingexp. \ + --type=AAAA \ + --ttl=2100 \ + "2607:f8b0:400a:801::1001" + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=_grpclb._tcp.srv-ipv6-multi-target.resolver-tests-version-1.grpctestingexp. \ + --type=SRV \ + --ttl=2100 \ + "0 0 1234 ipv6-multi-target.resolver-tests-version-1.grpctestingexp." + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=ipv6-multi-target.resolver-tests-version-1.grpctestingexp. \ + --type=AAAA \ + --ttl=2100 \ + "2607:f8b0:400a:801::1002" "2607:f8b0:400a:801::1003" "2607:f8b0:400a:801::1004" + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. \ + --type=TXT \ + --ttl=2100 \ + '"grpc_config=[{\"serviceConfig\":{\"loadBalancingPolicy\":\"round_robin\",\"methodConfig\":[{\"name\":[{\"method\":\"Foo\",\"service\":\"SimpleService\",\"waitForReady\":true}]}]}}]"' + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. \ + --type=A \ + --ttl=2100 \ + "1.2.3.4" + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=_grpclb._tcp.srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. \ + --type=SRV \ + --ttl=2100 \ + "0 0 1234 ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp." + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp. \ + --type=A \ + --ttl=2100 \ + "1.2.3.4" + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp. \ + --type=TXT \ + --ttl=2100 \ + '"grpc_config=[{\"serviceConfig\":{\"loadBalancingPolicy\":\"round_robin\",\"methodConfig\":[{\"name\":[{\"method\":\"Foo\",\"service\":\"NoSrvSimpleService\",\"waitForReady\":true}]}]}}]"' + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp. \ + --type=A \ + --ttl=2100 \ + "1.2.3.4" + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp. \ + --type=TXT \ + --ttl=2100 \ + '"grpc_config=[{\"clientLanguage\":[\"python\"],\"serviceConfig\":{\"loadBalancingPolicy\":\"round_robin\",\"methodConfig\":[{\"name\":[{\"method\":\"Foo\",\"service\":\"PythonService\",\"waitForReady\":true}]}]}}]"' + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp. \ + --type=A \ + --ttl=2100 \ + "1.2.3.4" + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp. \ + --type=TXT \ + --ttl=2100 \ + '"grpc_config=[{\"percentage\":0,\"serviceConfig\":{\"loadBalancingPolicy\":\"round_robin\",\"methodConfig\":[{\"name\":[{\"method\":\"Foo\",\"service\":\"CppService\",\"waitForReady\":true}]}]}}]"' + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp. \ + --type=A \ + --ttl=2100 \ + "1.2.3.4" + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp. \ + --type=TXT \ + --ttl=2100 \ + '"grpc_config=[{\"clientLanguage\":[\"go\"],\"serviceConfig\":{\"loadBalancingPolicy\":\"round_robin\",\"methodConfig\":[{\"name\":[{\"method\":\"Foo\",\"service\":\"GoService\",\"waitForReady\":true}]}]}},{\"clientLanguage\":[\"c++\"],\"serviceConfig\":{" "\"loadBalancingPolicy\":\"round_robin\",\"methodConfig\":[{\"name\":[{\"method\":\"Foo\",\"service\":\"CppService\",\"waitForReady\":true}]}]}}]"' + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp. \ + --type=A \ + --ttl=2100 \ + "1.2.3.4" + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp. \ + --type=TXT \ + --ttl=2100 \ + '"grpc_config=[{\"percentage\":0,\"serviceConfig\":{\"loadBalancingPolicy\":\"round_robin\",\"methodConfig\":[{\"name\":[{\"method\":\"Foo\",\"service\":\"NeverPickedService\",\"waitForReady\":true}]}]}},{\"percentage\":100,\"serviceConfig\":{\"loadBalanc" "ingPolicy\":\"round_robin\",\"methodConfig\":[{\"name\":[{\"method\":\"Foo\",\"service\":\"AlwaysPickedService\",\"waitForReady\":true}]}]}}]"' + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=_grpclb._tcp.srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. \ + --type=SRV \ + --ttl=2100 \ + "0 0 1234 balancer-for-ipv4-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp." + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=balancer-for-ipv4-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. \ + --type=A \ + --ttl=2100 \ + "1.2.3.4" + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. \ + --type=A \ + --ttl=2100 \ + "1.2.3.4" + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=_grpclb._tcp.srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. \ + --type=SRV \ + --ttl=2100 \ + "0 0 1234 balancer-for-ipv6-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp." + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=balancer-for-ipv6-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. \ + --type=AAAA \ + --ttl=2100 \ + "2607:f8b0:400a:801::1002" + +gcloud dns record-sets transaction add \ + -z=resolver-tests-version-1-grpctestingexp-zone-id \ + --name=srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. \ + --type=AAAA \ + --ttl=2100 \ + "2607:f8b0:400a:801::1002" + +gcloud dns record-sets transaction describe -z=resolver-tests-version-1-grpctestingexp-zone-id +gcloud dns record-sets transaction execute -z=resolver-tests-version-1-grpctestingexp-zone-id +gcloud dns record-sets list -z=resolver-tests-version-1-grpctestingexp-zone-id diff --git a/test/cpp/naming/resolver_component_tests_runner.sh b/test/cpp/naming/resolver_component_tests_runner.sh index cf71c9dcf97..407db5ed668 100755 --- a/test/cpp/naming/resolver_component_tests_runner.sh +++ b/test/cpp/naming/resolver_component_tests_runner.sh @@ -73,7 +73,7 @@ EXIT_CODE=0 # in the resolver. $FLAGS_test_bin_path \ - --target_name='srv-ipv4-single-target.resolver-tests.grpctestingexp.' \ + --target_name='srv-ipv4-single-target.resolver-tests-version-1.grpctestingexp.' \ --expected_addrs='1.2.3.4:1234,True' \ --expected_chosen_service_config='' \ --expected_lb_policy='' \ @@ -81,7 +81,7 @@ $FLAGS_test_bin_path \ wait $! || EXIT_CODE=1 $FLAGS_test_bin_path \ - --target_name='srv-ipv4-multi-target.resolver-tests.grpctestingexp.' \ + --target_name='srv-ipv4-multi-target.resolver-tests-version-1.grpctestingexp.' \ --expected_addrs='1.2.3.5:1234,True;1.2.3.6:1234,True;1.2.3.7:1234,True' \ --expected_chosen_service_config='' \ --expected_lb_policy='' \ @@ -89,7 +89,7 @@ $FLAGS_test_bin_path \ wait $! || EXIT_CODE=1 $FLAGS_test_bin_path \ - --target_name='srv-ipv6-single-target.resolver-tests.grpctestingexp.' \ + --target_name='srv-ipv6-single-target.resolver-tests-version-1.grpctestingexp.' \ --expected_addrs='[2607:f8b0:400a:801::1001]:1234,True' \ --expected_chosen_service_config='' \ --expected_lb_policy='' \ @@ -97,7 +97,7 @@ $FLAGS_test_bin_path \ wait $! || EXIT_CODE=1 $FLAGS_test_bin_path \ - --target_name='srv-ipv6-multi-target.resolver-tests.grpctestingexp.' \ + --target_name='srv-ipv6-multi-target.resolver-tests-version-1.grpctestingexp.' \ --expected_addrs='[2607:f8b0:400a:801::1002]:1234,True;[2607:f8b0:400a:801::1003]:1234,True;[2607:f8b0:400a:801::1004]:1234,True' \ --expected_chosen_service_config='' \ --expected_lb_policy='' \ @@ -105,7 +105,7 @@ $FLAGS_test_bin_path \ wait $! || EXIT_CODE=1 $FLAGS_test_bin_path \ - --target_name='srv-ipv4-simple-service-config.resolver-tests.grpctestingexp.' \ + --target_name='srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp.' \ --expected_addrs='1.2.3.4:1234,True' \ --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]}]}' \ --expected_lb_policy='round_robin' \ @@ -113,7 +113,7 @@ $FLAGS_test_bin_path \ wait $! || EXIT_CODE=1 $FLAGS_test_bin_path \ - --target_name='ipv4-no-srv-simple-service-config.resolver-tests.grpctestingexp.' \ + --target_name='ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp.' \ --expected_addrs='1.2.3.4:443,False' \ --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"NoSrvSimpleService","waitForReady":true}]}]}' \ --expected_lb_policy='round_robin' \ @@ -121,7 +121,7 @@ $FLAGS_test_bin_path \ wait $! || EXIT_CODE=1 $FLAGS_test_bin_path \ - --target_name='ipv4-no-config-for-cpp.resolver-tests.grpctestingexp.' \ + --target_name='ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp.' \ --expected_addrs='1.2.3.4:443,False' \ --expected_chosen_service_config='' \ --expected_lb_policy='' \ @@ -129,7 +129,7 @@ $FLAGS_test_bin_path \ wait $! || EXIT_CODE=1 $FLAGS_test_bin_path \ - --target_name='ipv4-cpp-config-has-zero-percentage.resolver-tests.grpctestingexp.' \ + --target_name='ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp.' \ --expected_addrs='1.2.3.4:443,False' \ --expected_chosen_service_config='' \ --expected_lb_policy='' \ @@ -137,7 +137,7 @@ $FLAGS_test_bin_path \ wait $! || EXIT_CODE=1 $FLAGS_test_bin_path \ - --target_name='ipv4-second-language-is-cpp.resolver-tests.grpctestingexp.' \ + --target_name='ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp.' \ --expected_addrs='1.2.3.4:443,False' \ --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"CppService","waitForReady":true}]}]}' \ --expected_lb_policy='round_robin' \ @@ -145,7 +145,7 @@ $FLAGS_test_bin_path \ wait $! || EXIT_CODE=1 $FLAGS_test_bin_path \ - --target_name='ipv4-config-with-percentages.resolver-tests.grpctestingexp.' \ + --target_name='ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp.' \ --expected_addrs='1.2.3.4:443,False' \ --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"AlwaysPickedService","waitForReady":true}]}]}' \ --expected_lb_policy='round_robin' \ @@ -153,7 +153,7 @@ $FLAGS_test_bin_path \ wait $! || EXIT_CODE=1 $FLAGS_test_bin_path \ - --target_name='srv-ipv4-target-has-backend-and-balancer.resolver-tests.grpctestingexp.' \ + --target_name='srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp.' \ --expected_addrs='1.2.3.4:1234,True;1.2.3.4:443,False' \ --expected_chosen_service_config='' \ --expected_lb_policy='' \ @@ -161,7 +161,7 @@ $FLAGS_test_bin_path \ wait $! || EXIT_CODE=1 $FLAGS_test_bin_path \ - --target_name='srv-ipv6-target-has-backend-and-balancer.resolver-tests.grpctestingexp.' \ + --target_name='srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp.' \ --expected_addrs='[2607:f8b0:400a:801::1002]:1234,True;[2607:f8b0:400a:801::1002]:443,False' \ --expected_chosen_service_config='' \ --expected_lb_policy='' \ @@ -169,7 +169,7 @@ $FLAGS_test_bin_path \ wait $! || EXIT_CODE=1 $FLAGS_test_bin_path \ - --target_name='ipv4-config-causing-fallback-to-tcp.resolver-tests.grpctestingexp.' \ + --target_name='ipv4-config-causing-fallback-to-tcp.resolver-tests-version-1.grpctestingexp.' \ --expected_addrs='1.2.3.4:443,False' \ --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwo","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooThree","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFour","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooFive","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSix","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooSeven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEight","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooNine","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTen","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooEleven","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]},{"name":[{"method":"FooTwelve","service":"SimpleService","waitForReady":true}]}]}' \ --expected_lb_policy='' \ diff --git a/test/cpp/naming/resolver_gce_integration_tests_runner.sh b/test/cpp/naming/resolver_gce_integration_tests_runner.sh new file mode 100755 index 00000000000..b20d18d9d10 --- /dev/null +++ b/test/cpp/naming/resolver_gce_integration_tests_runner.sh @@ -0,0 +1,359 @@ +#!/bin/bash +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is auto-generated + +set -ex + +if [[ "$GRPC_DNS_RESOLVER" == "" ]]; then + export GRPC_DNS_RESOLVER=ares +elif [[ "$GRPC_DNS_RESOLVER" != ares ]]; then + echo "Unexpected: GRPC_DNS_RESOLVER=$GRPC_DNS_RESOLVER. This test only works with c-ares resolver" + exit 1 +fi + +cd $(dirname $0)/../../.. + +if [[ "$CONFIG" == "" ]]; then + export CONFIG=opt +fi +make resolver_component_test +echo "Sanity check DNS records are resolveable with dig:" +EXIT_CODE=0 + +ONE_FAILED=0 +dig SRV _grpclb._tcp.srv-ipv4-single-target.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig SRV _grpclb._tcp.srv-ipv4-single-target.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig A ipv4-single-target.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig A ipv4-single-target.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig SRV _grpclb._tcp.srv-ipv4-multi-target.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig SRV _grpclb._tcp.srv-ipv4-multi-target.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig A ipv4-multi-target.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig A ipv4-multi-target.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig SRV _grpclb._tcp.srv-ipv6-single-target.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig SRV _grpclb._tcp.srv-ipv6-single-target.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig AAAA ipv6-single-target.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig AAAA ipv6-single-target.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig SRV _grpclb._tcp.srv-ipv6-multi-target.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig SRV _grpclb._tcp.srv-ipv6-multi-target.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig AAAA ipv6-multi-target.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig AAAA ipv6-multi-target.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig TXT srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig TXT srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig A ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig A ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig SRV _grpclb._tcp.srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig SRV _grpclb._tcp.srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig A ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig A ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig TXT ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig TXT ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig A ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig A ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig TXT ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig TXT ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig A ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig A ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig TXT ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig TXT ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig A ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig A ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig TXT ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig TXT ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig A ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig A ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig TXT ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig TXT ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig SRV _grpclb._tcp.srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig SRV _grpclb._tcp.srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig A balancer-for-ipv4-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig A balancer-for-ipv4-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig A srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig A srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig SRV _grpclb._tcp.srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig SRV _grpclb._tcp.srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig AAAA balancer-for-ipv6-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig AAAA balancer-for-ipv6-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +ONE_FAILED=0 +dig AAAA srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. | grep 'ANSWER SECTION' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Sanity check: dig AAAA srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. FAILED" + exit 1 +fi + +echo "Sanity check PASSED. Run resolver tests:" + +ONE_FAILED=0 +bins/$CONFIG/resolver_component_test \ + --target_name='srv-ipv4-single-target.resolver-tests-version-1.grpctestingexp.' \ + --expected_addrs='1.2.3.4:1234,True' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Test based on target record: srv-ipv4-single-target.resolver-tests-version-1.grpctestingexp. FAILED" + EXIT_CODE=1 +fi + +ONE_FAILED=0 +bins/$CONFIG/resolver_component_test \ + --target_name='srv-ipv4-multi-target.resolver-tests-version-1.grpctestingexp.' \ + --expected_addrs='1.2.3.5:1234,True;1.2.3.6:1234,True;1.2.3.7:1234,True' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Test based on target record: srv-ipv4-multi-target.resolver-tests-version-1.grpctestingexp. FAILED" + EXIT_CODE=1 +fi + +ONE_FAILED=0 +bins/$CONFIG/resolver_component_test \ + --target_name='srv-ipv6-single-target.resolver-tests-version-1.grpctestingexp.' \ + --expected_addrs='[2607:f8b0:400a:801::1001]:1234,True' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Test based on target record: srv-ipv6-single-target.resolver-tests-version-1.grpctestingexp. FAILED" + EXIT_CODE=1 +fi + +ONE_FAILED=0 +bins/$CONFIG/resolver_component_test \ + --target_name='srv-ipv6-multi-target.resolver-tests-version-1.grpctestingexp.' \ + --expected_addrs='[2607:f8b0:400a:801::1002]:1234,True;[2607:f8b0:400a:801::1003]:1234,True;[2607:f8b0:400a:801::1004]:1234,True' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Test based on target record: srv-ipv6-multi-target.resolver-tests-version-1.grpctestingexp. FAILED" + EXIT_CODE=1 +fi + +ONE_FAILED=0 +bins/$CONFIG/resolver_component_test \ + --target_name='srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp.' \ + --expected_addrs='1.2.3.4:1234,True' \ + --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"SimpleService","waitForReady":true}]}]}' \ + --expected_lb_policy='round_robin' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Test based on target record: srv-ipv4-simple-service-config.resolver-tests-version-1.grpctestingexp. FAILED" + EXIT_CODE=1 +fi + +ONE_FAILED=0 +bins/$CONFIG/resolver_component_test \ + --target_name='ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"NoSrvSimpleService","waitForReady":true}]}]}' \ + --expected_lb_policy='round_robin' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Test based on target record: ipv4-no-srv-simple-service-config.resolver-tests-version-1.grpctestingexp. FAILED" + EXIT_CODE=1 +fi + +ONE_FAILED=0 +bins/$CONFIG/resolver_component_test \ + --target_name='ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Test based on target record: ipv4-no-config-for-cpp.resolver-tests-version-1.grpctestingexp. FAILED" + EXIT_CODE=1 +fi + +ONE_FAILED=0 +bins/$CONFIG/resolver_component_test \ + --target_name='ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Test based on target record: ipv4-cpp-config-has-zero-percentage.resolver-tests-version-1.grpctestingexp. FAILED" + EXIT_CODE=1 +fi + +ONE_FAILED=0 +bins/$CONFIG/resolver_component_test \ + --target_name='ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"CppService","waitForReady":true}]}]}' \ + --expected_lb_policy='round_robin' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Test based on target record: ipv4-second-language-is-cpp.resolver-tests-version-1.grpctestingexp. FAILED" + EXIT_CODE=1 +fi + +ONE_FAILED=0 +bins/$CONFIG/resolver_component_test \ + --target_name='ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp.' \ + --expected_addrs='1.2.3.4:443,False' \ + --expected_chosen_service_config='{"loadBalancingPolicy":"round_robin","methodConfig":[{"name":[{"method":"Foo","service":"AlwaysPickedService","waitForReady":true}]}]}' \ + --expected_lb_policy='round_robin' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Test based on target record: ipv4-config-with-percentages.resolver-tests-version-1.grpctestingexp. FAILED" + EXIT_CODE=1 +fi + +ONE_FAILED=0 +bins/$CONFIG/resolver_component_test \ + --target_name='srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp.' \ + --expected_addrs='1.2.3.4:1234,True;1.2.3.4:443,False' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Test based on target record: srv-ipv4-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. FAILED" + EXIT_CODE=1 +fi + +ONE_FAILED=0 +bins/$CONFIG/resolver_component_test \ + --target_name='srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp.' \ + --expected_addrs='[2607:f8b0:400a:801::1002]:1234,True;[2607:f8b0:400a:801::1002]:443,False' \ + --expected_chosen_service_config='' \ + --expected_lb_policy='' || ONE_FAILED=1 +if [[ "$ONE_FAILED" != 0 ]]; then + echo "Test based on target record: srv-ipv6-target-has-backend-and-balancer.resolver-tests-version-1.grpctestingexp. FAILED" + EXIT_CODE=1 +fi + +exit $EXIT_CODE diff --git a/test/cpp/naming/resolver_test_record_groups.yaml b/test/cpp/naming/resolver_test_record_groups.yaml index 33d774ca701..2b3204335c4 100644 --- a/test/cpp/naming/resolver_test_record_groups.yaml +++ b/test/cpp/naming/resolver_test_record_groups.yaml @@ -1,4 +1,4 @@ -resolver_component_tests_common_zone_name: resolver-tests.grpctestingexp. +resolver_tests_common_zone_name: resolver-tests-version-1.grpctestingexp. resolver_component_tests: - expected_addrs: - {address: '1.2.3.4:1234', is_balancer: true} diff --git a/test/cpp/naming/test_dns_server.py b/test/cpp/naming/test_dns_server.py index 9d4b89cffb5..9f42f65ee61 100755 --- a/test/cpp/naming/test_dns_server.py +++ b/test/cpp/naming/test_dns_server.py @@ -66,7 +66,7 @@ def start_local_dns_server(args): with open(args.records_config_path) as config: test_records_config = yaml.load(config) - common_zone_name = test_records_config['resolver_component_tests_common_zone_name'] + common_zone_name = test_records_config['resolver_tests_common_zone_name'] for group in test_records_config['resolver_component_tests']: for name in group['records'].keys(): for record in group['records'][name]: diff --git a/tools/codegen/core/gen_stats_data.py b/tools/codegen/core/gen_stats_data.py index 10ad0cc831f..8359734c848 100755 --- a/tools/codegen/core/gen_stats_data.py +++ b/tools/codegen/core/gen_stats_data.py @@ -147,8 +147,7 @@ def gen_bucket_code(histogram): shift_data = find_ideal_shift(code_bounds[first_nontrivial:], 256 * histogram.buckets) #print first_nontrivial, shift_data, bounds #if shift_data is not None: print [hex(x >> shift_data[0]) for x in code_bounds[first_nontrivial:]] - code = '\n/* Automatically generated by tools/codegen/core/gen_stats_data.py */\n' - code += 'value = GPR_CLAMP(value, 0, %d);\n' % histogram.max + code = 'value = GPR_CLAMP(value, 0, %d);\n' % histogram.max map_table = gen_map_table(code_bounds[first_nontrivial:], shift_data) if first_nontrivial is None: code += ('GRPC_STATS_INC_HISTOGRAM((exec_ctx), GRPC_STATS_HISTOGRAM_%s, value);\n' @@ -408,3 +407,4 @@ with open('src/core/lib/debug/stats_data_bq_schema.sql', 'w') as S: for counter in inst_map['Counter']: columns.append(('%s_per_iteration' % counter.name, 'FLOAT')) print >>S, ',\n'.join('%s:%s' % x for x in columns) + diff --git a/tools/github/pr_latency.py b/tools/github/pr_latency.py new file mode 100644 index 00000000000..5d635835e54 --- /dev/null +++ b/tools/github/pr_latency.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Measure the time between PR creation and completion of all tests. + +You'll need a github API token to avoid being rate-limited. See +https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/ + +This script goes over the most recent 100 pull requests. For PRs with a single +commit, it uses the PR's creation as the initial time; othewise, it uses the +date of the last commit. This is somewhat fragile, and imposed by the fact that +GitHub reports a PR's updated timestamp for any event that modifies the PR (e.g. +comments), not just the addition of new commits. + +In addition, it ignores latencies greater than five hours, as that's likely due +to a manual re-run of tests. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import json +import logging +import pprint +import urllib2 + +from datetime import datetime, timedelta + +logging.basicConfig(format='%(asctime)s %(message)s') + +PRS = 'https://api.github.com/repos/grpc/grpc/pulls?state=open&per_page=100' +COMMITS = 'https://api.github.com/repos/grpc/grpc/pulls/{pr_number}/commits' + + +def gh(url): + request = urllib2.Request(url) + if TOKEN: + request.add_header('Authorization', 'token {}'.format(TOKEN)) + response = urllib2.urlopen(request) + return response.read() + + +def print_csv_header(): + print('pr,base_time,test_time,latency_seconds,successes,failures,errors') + + +def output(pr, base_time, test_time, diff_time, successes, failures, errors, mode='human'): + if mode == 'human': + print("PR #{} base time: {} UTC, Tests completed at: {} UTC. Latency: {}." + "\n\tSuccesses: {}, Failures: {}, Errors: {}".format( + pr, base_time, test_time, diff_time, successes, failures, errors)) + elif mode == 'csv': + print(','.join([str(pr), str(base_time), + str(test_time), str(int((test_time-base_time).total_seconds())), + str(successes), str(failures), str(errors)])) + + +def parse_timestamp(datetime_str): + return datetime.strptime(datetime_str, '%Y-%m-%dT%H:%M:%SZ') + + +def to_posix_timestamp(dt): + return str((dt - datetime(1970, 1, 1)).total_seconds()) + + +def get_pr_data(): + latest_prs = json.loads(gh(PRS)) + res = [{'number': pr['number'], + 'created_at': parse_timestamp(pr['created_at']), + 'updated_at': parse_timestamp(pr['updated_at']), + 'statuses_url': pr['statuses_url']} + for pr in latest_prs] + return res + + +def get_commits_data(pr_number): + commits = json.loads(gh(COMMITS.format(pr_number=pr_number))) + return {'num_commits': len(commits), + 'most_recent_date': parse_timestamp(commits[-1]['commit']['author']['date'])} + + +def get_status_data(statuses_url, system): + status_url = statuses_url.replace('statuses', 'status') + statuses = json.loads(gh(status_url + '?per_page=100')) + successes = 0 + failures = 0 + errors = 0 + latest_datetime = None + if not statuses: return None + if system == 'kokoro': string_in_target_url = 'kokoro' + elif system == 'jenkins': string_in_target_url = 'grpc-testing' + for status in statuses['statuses']: + if not status['target_url'] or string_in_target_url not in status['target_url']: continue # Ignore jenkins + if status['state'] == 'pending': return None + elif status['state'] == 'success': successes += 1 + elif status['state'] == 'failure': failures += 1 + elif status['state'] == 'error': errors += 1 + if not latest_datetime: + latest_datetime = parse_timestamp(status['updated_at']) + else: + latest_datetime = max(latest_datetime, parse_timestamp(status['updated_at'])) + # First status is the most recent one. + if any([successes, failures, errors]) and sum([successes, failures, errors]) > 15: + return {'latest_datetime': latest_datetime, + 'successes': successes, + 'failures': failures, + 'errors': errors} + else: return None + + +def build_args_parser(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--format', type=str, choices=['human', 'csv'], + default='human', + help='Output format: are you a human or a machine?') + parser.add_argument('--system', type=str, choices=['jenkins', 'kokoro'], + required=True, help='Consider only the given CI system') + parser.add_argument('--token', type=str, default='', + help='GitHub token to use its API with a higher rate limit') + return parser + + +def main(): + import sys + global TOKEN + args_parser = build_args_parser() + args = args_parser.parse_args() + TOKEN = args.token + if args.format == 'csv': print_csv_header() + for pr_data in get_pr_data(): + commit_data = get_commits_data(pr_data['number']) + # PR with a single commit -> use the PRs creation time. + # else -> use the latest commit's date. + base_timestamp = pr_data['updated_at'] + if commit_data['num_commits'] > 1: + base_timestamp = commit_data['most_recent_date'] + else: + base_timestamp = pr_data['created_at'] + last_status = get_status_data(pr_data['statuses_url'], args.system) + if last_status: + diff = last_status['latest_datetime'] - base_timestamp + if diff < timedelta(hours=5): + output(pr_data['number'], base_timestamp, last_status['latest_datetime'], + diff, last_status['successes'], last_status['failures'], + last_status['errors'], mode=args.format) + + +if __name__ == '__main__': + main() diff --git a/tools/internal_ci/linux/grpc_performance_profile_daily.sh b/tools/internal_ci/linux/grpc_performance_profile_daily.sh index 25523e21b80..34d41bc04c6 100755 --- a/tools/internal_ci/linux/grpc_performance_profile_daily.sh +++ b/tools/internal_ci/linux/grpc_performance_profile_daily.sh @@ -22,6 +22,8 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc CPUS=`python -c 'import multiprocessing; print multiprocessing.cpu_count()'` +./tools/run_tests/start_port_server.py || true + make CONFIG=opt memory_profile_test memory_profile_client memory_profile_server -j $CPUS bins/opt/memory_profile_test bq load microbenchmarks.memory memory_usage.csv diff --git a/tools/jenkins/run_performance_profile_daily.sh b/tools/jenkins/run_performance_profile_daily.sh index 04a2464aee1..48d82a9b7f6 100755 --- a/tools/jenkins/run_performance_profile_daily.sh +++ b/tools/jenkins/run_performance_profile_daily.sh @@ -29,4 +29,6 @@ fi BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata" +./tools/run_tests/start_port_server.py || true + $PYTHON tools/run_tests/run_microbenchmark.py --collect summary perf latency -b $BENCHMARKS_TO_RUN diff --git a/tools/run_tests/performance/massage_qps_stats.py b/tools/run_tests/performance/massage_qps_stats.py index e1620adad26..9b9355308a3 100644 --- a/tools/run_tests/performance/massage_qps_stats.py +++ b/tools/run_tests/performance/massage_qps_stats.py @@ -106,8 +106,6 @@ def massage_qps_stats(scenario_result): stats["core_executor_wakeup_initiated"] = massage_qps_stats_helpers.counter(core_stats, "executor_wakeup_initiated") stats["core_executor_queue_drained"] = massage_qps_stats_helpers.counter(core_stats, "executor_queue_drained") stats["core_executor_push_retries"] = massage_qps_stats_helpers.counter(core_stats, "executor_push_retries") - stats["core_executor_threads_created"] = massage_qps_stats_helpers.counter(core_stats, "executor_threads_created") - stats["core_executor_threads_used"] = massage_qps_stats_helpers.counter(core_stats, "executor_threads_used") stats["core_server_requested_calls"] = massage_qps_stats_helpers.counter(core_stats, "server_requested_calls") stats["core_server_slowpath_requests_queued"] = massage_qps_stats_helpers.counter(core_stats, "server_slowpath_requests_queued") h = massage_qps_stats_helpers.histogram(core_stats, "call_initial_size") @@ -182,12 +180,6 @@ def massage_qps_stats(scenario_result): stats["core_http2_send_flowctl_per_write_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) stats["core_http2_send_flowctl_per_write_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) stats["core_http2_send_flowctl_per_write_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) - h = massage_qps_stats_helpers.histogram(core_stats, "executor_closures_per_wakeup") - stats["core_executor_closures_per_wakeup"] = ",".join("%f" % x for x in h.buckets) - stats["core_executor_closures_per_wakeup_bkts"] = ",".join("%f" % x for x in h.boundaries) - stats["core_executor_closures_per_wakeup_50p"] = massage_qps_stats_helpers.percentile(h.buckets, 50, h.boundaries) - stats["core_executor_closures_per_wakeup_95p"] = massage_qps_stats_helpers.percentile(h.buckets, 95, h.boundaries) - stats["core_executor_closures_per_wakeup_99p"] = massage_qps_stats_helpers.percentile(h.buckets, 99, h.boundaries) h = massage_qps_stats_helpers.histogram(core_stats, "server_cqs_checked") stats["core_server_cqs_checked"] = ",".join("%f" % x for x in h.buckets) stats["core_server_cqs_checked_bkts"] = ",".join("%f" % x for x in h.boundaries) diff --git a/tools/run_tests/performance/scenario_result_schema.json b/tools/run_tests/performance/scenario_result_schema.json index c7b1904bd19..2f0fd916d4f 100644 --- a/tools/run_tests/performance/scenario_result_schema.json +++ b/tools/run_tests/performance/scenario_result_schema.json @@ -540,16 +540,6 @@ "name": "core_executor_push_retries", "type": "INTEGER" }, - { - "mode": "NULLABLE", - "name": "core_executor_threads_created", - "type": "INTEGER" - }, - { - "mode": "NULLABLE", - "name": "core_executor_threads_used", - "type": "INTEGER" - }, { "mode": "NULLABLE", "name": "core_server_requested_calls", @@ -860,31 +850,6 @@ "name": "core_http2_send_flowctl_per_write_99p", "type": "FLOAT" }, - { - "mode": "NULLABLE", - "name": "core_executor_closures_per_wakeup", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "core_executor_closures_per_wakeup_bkts", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "core_executor_closures_per_wakeup_50p", - "type": "FLOAT" - }, - { - "mode": "NULLABLE", - "name": "core_executor_closures_per_wakeup_95p", - "type": "FLOAT" - }, - { - "mode": "NULLABLE", - "name": "core_executor_closures_per_wakeup_99p", - "type": "FLOAT" - }, { "mode": "NULLABLE", "name": "core_server_cqs_checked", @@ -1367,16 +1332,6 @@ "name": "core_executor_push_retries", "type": "INTEGER" }, - { - "mode": "NULLABLE", - "name": "core_executor_threads_created", - "type": "INTEGER" - }, - { - "mode": "NULLABLE", - "name": "core_executor_threads_used", - "type": "INTEGER" - }, { "mode": "NULLABLE", "name": "core_server_requested_calls", @@ -1687,31 +1642,6 @@ "name": "core_http2_send_flowctl_per_write_99p", "type": "FLOAT" }, - { - "mode": "NULLABLE", - "name": "core_executor_closures_per_wakeup", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "core_executor_closures_per_wakeup_bkts", - "type": "STRING" - }, - { - "mode": "NULLABLE", - "name": "core_executor_closures_per_wakeup_50p", - "type": "FLOAT" - }, - { - "mode": "NULLABLE", - "name": "core_executor_closures_per_wakeup_95p", - "type": "FLOAT" - }, - { - "mode": "NULLABLE", - "name": "core_executor_closures_per_wakeup_99p", - "type": "FLOAT" - }, { "mode": "NULLABLE", "name": "core_server_cqs_checked", diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index b38108d456c..6697149dd9c 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -717,6 +717,9 @@ class PythonLanguage(object): return (pypy32_config,) elif args.compiler == 'python_alpine': return (python27_config,) + elif args.compiler == 'all_the_cpythons': + return (python27_config, python34_config, python35_config, + python36_config,) else: raise Exception('Compiler %s not supported.' % args.compiler) @@ -1214,7 +1217,7 @@ argp.add_argument('--compiler', choices=['default', 'gcc4.4', 'gcc4.6', 'gcc4.8', 'gcc4.9', 'gcc5.3', 'gcc_musl', 'clang3.4', 'clang3.5', 'clang3.6', 'clang3.7', - 'python2.7', 'python3.4', 'python3.5', 'python3.6', 'pypy', 'pypy3', 'python_alpine', + 'python2.7', 'python3.4', 'python3.5', 'python3.6', 'pypy', 'pypy3', 'python_alpine', 'all_the_cpythons', 'node0.12', 'node4', 'node5', 'node6', 'node7', 'node8', 'electron1.3', 'electron1.6', 'coreclr',