From 8aace513d090679bd15ceb68eb4824cec3dc044e Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Mon, 15 Aug 2016 14:55:12 -0700 Subject: [PATCH 01/25] Introduced grpc_lb_policy_pick_args and added some LB docs --- src/core/ext/client_config/client_channel.c | 7 +-- src/core/ext/client_config/lb_policy.c | 7 +-- src/core/ext/client_config/lb_policy.h | 50 ++++++++++++------- src/core/ext/lb_policy/grpclb/grpclb.c | 32 ++++++------ .../ext/lb_policy/pick_first/pick_first.c | 10 ++-- .../ext/lb_policy/round_robin/round_robin.c | 10 ++-- 6 files changed, 62 insertions(+), 54 deletions(-) diff --git a/src/core/ext/client_config/client_channel.c b/src/core/ext/client_config/client_channel.c index 2c0c4abffc7..8e653a8eeed 100644 --- a/src/core/ext/client_config/client_channel.c +++ b/src/core/ext/client_config/client_channel.c @@ -401,9 +401,10 @@ static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *elemp, int r; GRPC_LB_POLICY_REF(lb_policy, "cc_pick_subchannel"); gpr_mu_unlock(&chand->mu_config); - r = grpc_lb_policy_pick(exec_ctx, lb_policy, calld->pollent, - initial_metadata, initial_metadata_flags, - connected_subchannel, on_ready); + const grpc_lb_policy_pick_args inputs = {calld->pollent, initial_metadata, + initial_metadata_flags}; + r = grpc_lb_policy_pick(exec_ctx, lb_policy, &inputs, connected_subchannel, + on_ready); GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "cc_pick_subchannel"); GPR_TIMER_END("cc_pick_subchannel", 0); return r; diff --git a/src/core/ext/client_config/lb_policy.c b/src/core/ext/client_config/lb_policy.c index 8b980b2cca2..71170f5655b 100644 --- a/src/core/ext/client_config/lb_policy.c +++ b/src/core/ext/client_config/lb_policy.c @@ -100,13 +100,10 @@ void grpc_lb_policy_weak_unref(grpc_exec_ctx *exec_ctx, } int grpc_lb_policy_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, - grpc_polling_entity *pollent, - grpc_metadata_batch *initial_metadata, - uint32_t initial_metadata_flags, + const grpc_lb_policy_pick_args *pick_args, grpc_connected_subchannel **target, grpc_closure *on_complete) { - return policy->vtable->pick(exec_ctx, policy, pollent, initial_metadata, - initial_metadata_flags, target, on_complete); + return policy->vtable->pick(exec_ctx, policy, pick_args, target, on_complete); } void grpc_lb_policy_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, diff --git a/src/core/ext/client_config/lb_policy.h b/src/core/ext/client_config/lb_policy.h index a2f5446fc6c..6b9d30c6e5d 100644 --- a/src/core/ext/client_config/lb_policy.h +++ b/src/core/ext/client_config/lb_policy.h @@ -53,23 +53,35 @@ struct grpc_lb_policy { grpc_pollset_set *interested_parties; }; +/** Extra arguments for an LB pick */ +typedef struct grpc_lb_policy_pick_args { + /** Parties interested in the pick's progress */ + grpc_polling_entity *pollent; + /** Initial metadata associated with the picking call. */ + grpc_metadata_batch *initial_metadata; + /** See \a GRPC_INITIAL_METADATA_* in grpc_types.h */ + uint32_t initial_metadata_flags; +} grpc_lb_policy_pick_args; + struct grpc_lb_policy_vtable { void (*destroy)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy); - void (*shutdown)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy); - /** implement grpc_lb_policy_pick */ + /** \see grpc_lb_policy_pick */ int (*pick)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, - grpc_polling_entity *pollent, - grpc_metadata_batch *initial_metadata, - uint32_t initial_metadata_flags, + const grpc_lb_policy_pick_args *pick_args, grpc_connected_subchannel **target, grpc_closure *on_complete); + + /** \see grpc_lb_policy_cancel_pick */ void (*cancel_pick)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, grpc_connected_subchannel **target); + + /** \see grpc_lb_policy_cancel_picks */ void (*cancel_picks)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, uint32_t initial_metadata_flags_mask, uint32_t initial_metadata_flags_eq); + /** \see grpc_lb_policy_ping_one */ void (*ping_one)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, grpc_closure *closure); @@ -83,8 +95,7 @@ struct grpc_lb_policy_vtable { /** call notify when the connectivity state of a channel changes from *state. Updates *state with the new state of the policy. Calling with a NULL \a - state cancels the subscription. - */ + state cancels the subscription. */ void (*notify_on_state_change)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, grpc_connectivity_state *state, @@ -124,26 +135,31 @@ void grpc_lb_policy_weak_unref(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy); void grpc_lb_policy_init(grpc_lb_policy *policy, const grpc_lb_policy_vtable *vtable); -/** Given initial metadata in \a initial_metadata, find an appropriate - target for this rpc, and 'return' it by calling \a on_complete after setting - \a target. - Picking can be asynchronous. Any IO should be done under \a pollent. */ +/** Find an appropriate target for this call, based on \a pick_args. + Upon completion \a on_complete will be called, with \a *target set to an + appropriate connected subchannel if the pick was successful or NULL + otherwise. + Picking can be asynchronous. Any IO should be done under \a + pick_args->pollent. */ int grpc_lb_policy_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, - grpc_polling_entity *pollent, - grpc_metadata_batch *initial_metadata, - uint32_t initial_metadata_flags, + const grpc_lb_policy_pick_args *pick_args, grpc_connected_subchannel **target, grpc_closure *on_complete); +/** Perform a connected subchannel ping (see \a grpc_connected_subchannel_ping) + against one of the connected subchannels managed by \a policy. */ void grpc_lb_policy_ping_one(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, grpc_closure *closure); +/** Cancel picks for \a target. + The \a on_complete callback of the pending picks will be invoked with \a + *target set to NULL. */ void grpc_lb_policy_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, grpc_connected_subchannel **target); -/** Cancel all pending picks which have: - (initial_metadata_flags & initial_metadata_flags_mask) == - initial_metadata_flags_eq */ +/** Cancel all pending picks for which their \a initial_metadata_flags (as given + in the call to \a grpc_lb_policy_pick) matches \a initial_metadata_flags_eq + when AND'd with \a initial_metadata_flags_mask */ void grpc_lb_policy_cancel_picks(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, uint32_t initial_metadata_flags_mask, diff --git a/src/core/ext/lb_policy/grpclb/grpclb.c b/src/core/ext/lb_policy/grpclb/grpclb.c index af913d8a9df..693145bd6cb 100644 --- a/src/core/ext/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/lb_policy/grpclb/grpclb.c @@ -180,19 +180,18 @@ typedef struct pending_pick { wrapped_rr_closure_arg wrapped_on_complete_arg; } pending_pick; -static void add_pending_pick(pending_pick **root, grpc_polling_entity *pollent, - grpc_metadata_batch *initial_metadata, - uint32_t initial_metadata_flags, +static void add_pending_pick(pending_pick **root, + const grpc_lb_policy_pick_args *pick_args, grpc_connected_subchannel **target, grpc_closure *on_complete) { pending_pick *pp = gpr_malloc(sizeof(*pp)); memset(pp, 0, sizeof(pending_pick)); memset(&pp->wrapped_on_complete_arg, 0, sizeof(wrapped_rr_closure_arg)); pp->next = *root; - pp->pollent = pollent; + pp->pollent = pick_args->pollent; pp->target = target; - pp->initial_metadata = initial_metadata; - pp->initial_metadata_flags = initial_metadata_flags; + pp->initial_metadata = pick_args->initial_metadata; + pp->initial_metadata_flags = pick_args->initial_metadata_flags; pp->wrapped_on_complete_arg.wrapped_closure = on_complete; grpc_closure_init(&pp->wrapped_on_complete, wrapped_rr_closure, &pp->wrapped_on_complete_arg); @@ -359,9 +358,10 @@ static void rr_handover(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, gpr_log(GPR_INFO, "Pending pick about to PICK from 0x%" PRIxPTR "", (intptr_t)glb_policy->rr_policy); } - grpc_lb_policy_pick(exec_ctx, glb_policy->rr_policy, pp->pollent, - pp->initial_metadata, pp->initial_metadata_flags, - pp->target, &pp->wrapped_on_complete); + const grpc_lb_policy_pick_args pick_args = { + pp->pollent, pp->initial_metadata, pp->initial_metadata_flags}; + grpc_lb_policy_pick(exec_ctx, glb_policy->rr_policy, &pick_args, pp->target, + &pp->wrapped_on_complete); pp->wrapped_on_complete_arg.owning_pending_node = pp; } @@ -603,9 +603,7 @@ static void glb_exit_idle(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { } static int glb_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, - grpc_polling_entity *pollent, - grpc_metadata_batch *initial_metadata, - uint32_t initial_metadata_flags, + const grpc_lb_policy_pick_args *pick_args, grpc_connected_subchannel **target, grpc_closure *on_complete) { glb_lb_policy *glb_policy = (glb_lb_policy *)pol; @@ -623,8 +621,8 @@ static int glb_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, glb_policy->wc_arg.wrapped_closure = on_complete; grpc_closure_init(&glb_policy->wrapped_on_complete, wrapped_rr_closure, &glb_policy->wc_arg); - r = grpc_lb_policy_pick(exec_ctx, glb_policy->rr_policy, pollent, - initial_metadata, initial_metadata_flags, target, + + r = grpc_lb_policy_pick(exec_ctx, glb_policy->rr_policy, pick_args, target, &glb_policy->wrapped_on_complete); if (r != 0) { /* the call to grpc_lb_policy_pick has been sychronous. Unreffing the RR @@ -639,10 +637,10 @@ static int glb_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, GRPC_ERROR_NONE, NULL); } } else { - grpc_polling_entity_add_to_pollset_set(exec_ctx, pollent, + grpc_polling_entity_add_to_pollset_set(exec_ctx, pick_args->pollent, glb_policy->base.interested_parties); - add_pending_pick(&glb_policy->pending_picks, pollent, initial_metadata, - initial_metadata_flags, target, on_complete); + add_pending_pick(&glb_policy->pending_picks, pick_args, target, + on_complete); if (!glb_policy->started_picking) { start_picking(exec_ctx, glb_policy); diff --git a/src/core/ext/lb_policy/pick_first/pick_first.c b/src/core/ext/lb_policy/pick_first/pick_first.c index 9decf70692a..e1277b353fe 100644 --- a/src/core/ext/lb_policy/pick_first/pick_first.c +++ b/src/core/ext/lb_policy/pick_first/pick_first.c @@ -199,9 +199,7 @@ static void pf_exit_idle(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { } static int pf_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, - grpc_polling_entity *pollent, - grpc_metadata_batch *initial_metadata, - uint32_t initial_metadata_flags, + const grpc_lb_policy_pick_args *pick_args, grpc_connected_subchannel **target, grpc_closure *on_complete) { pick_first_lb_policy *p = (pick_first_lb_policy *)pol; @@ -225,13 +223,13 @@ static int pf_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, if (!p->started_picking) { start_picking(exec_ctx, p); } - grpc_polling_entity_add_to_pollset_set(exec_ctx, pollent, + grpc_polling_entity_add_to_pollset_set(exec_ctx, pick_args->pollent, p->base.interested_parties); pp = gpr_malloc(sizeof(*pp)); pp->next = p->pending_picks; - pp->pollent = pollent; + pp->pollent = pick_args->pollent; pp->target = target; - pp->initial_metadata_flags = initial_metadata_flags; + pp->initial_metadata_flags = pick_args->initial_metadata_flags; pp->on_complete = on_complete; p->pending_picks = pp; gpr_mu_unlock(&p->mu); diff --git a/src/core/ext/lb_policy/round_robin/round_robin.c b/src/core/ext/lb_policy/round_robin/round_robin.c index 7bcf608ab9b..49e724a0f23 100644 --- a/src/core/ext/lb_policy/round_robin/round_robin.c +++ b/src/core/ext/lb_policy/round_robin/round_robin.c @@ -361,9 +361,7 @@ static void rr_exit_idle(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { } static int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, - grpc_polling_entity *pollent, - grpc_metadata_batch *initial_metadata, - uint32_t initial_metadata_flags, + const grpc_lb_policy_pick_args *pick_args, grpc_connected_subchannel **target, grpc_closure *on_complete) { round_robin_lb_policy *p = (round_robin_lb_policy *)pol; @@ -385,14 +383,14 @@ static int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, if (!p->started_picking) { start_picking(exec_ctx, p); } - grpc_polling_entity_add_to_pollset_set(exec_ctx, pollent, + grpc_polling_entity_add_to_pollset_set(exec_ctx, pick_args->pollent, p->base.interested_parties); pp = gpr_malloc(sizeof(*pp)); pp->next = p->pending_picks; - pp->pollent = pollent; + pp->pollent = pick_args->pollent; pp->target = target; pp->on_complete = on_complete; - pp->initial_metadata_flags = initial_metadata_flags; + pp->initial_metadata_flags = pick_args->initial_metadata_flags; p->pending_picks = pp; gpr_mu_unlock(&p->mu); return 0; From 5b0e9462f0d40b171d50c03b29016c36588a3520 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Mon, 15 Aug 2016 19:38:39 -0700 Subject: [PATCH 02/25] Introduced LB token initial metadata propagation --- src/core/ext/client_config/client_channel.c | 3 +- src/core/ext/client_config/lb_policy.h | 2 + .../ext/client_config/lb_policy_factory.h | 8 ++ .../client_config/subchannel_call_holder.h | 2 + src/core/ext/lb_policy/grpclb/grpclb.c | 33 +++++- .../ext/lb_policy/round_robin/round_robin.c | 105 +++++++++++++++--- test/cpp/grpclb/grpclb_test.cc | 13 ++- 7 files changed, 146 insertions(+), 20 deletions(-) diff --git a/src/core/ext/client_config/client_channel.c b/src/core/ext/client_config/client_channel.c index 8e653a8eeed..c1c5e38cb0f 100644 --- a/src/core/ext/client_config/client_channel.c +++ b/src/core/ext/client_config/client_channel.c @@ -402,7 +402,8 @@ static int cc_pick_subchannel(grpc_exec_ctx *exec_ctx, void *elemp, GRPC_LB_POLICY_REF(lb_policy, "cc_pick_subchannel"); gpr_mu_unlock(&chand->mu_config); const grpc_lb_policy_pick_args inputs = {calld->pollent, initial_metadata, - initial_metadata_flags}; + initial_metadata_flags, + &calld->lb_token_mdelem}; r = grpc_lb_policy_pick(exec_ctx, lb_policy, &inputs, connected_subchannel, on_ready); GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "cc_pick_subchannel"); diff --git a/src/core/ext/client_config/lb_policy.h b/src/core/ext/client_config/lb_policy.h index 6b9d30c6e5d..6f133a29488 100644 --- a/src/core/ext/client_config/lb_policy.h +++ b/src/core/ext/client_config/lb_policy.h @@ -61,6 +61,8 @@ typedef struct grpc_lb_policy_pick_args { grpc_metadata_batch *initial_metadata; /** See \a GRPC_INITIAL_METADATA_* in grpc_types.h */ uint32_t initial_metadata_flags; + /** Storage for LB token in \a initial_metadata, or NULL if not used */ + grpc_linked_mdelem *lb_token_mdelem_storage; } grpc_lb_policy_pick_args; struct grpc_lb_policy_vtable { diff --git a/src/core/ext/client_config/lb_policy_factory.h b/src/core/ext/client_config/lb_policy_factory.h index 1c89b28b59a..635efb6e6a9 100644 --- a/src/core/ext/client_config/lb_policy_factory.h +++ b/src/core/ext/client_config/lb_policy_factory.h @@ -49,8 +49,16 @@ struct grpc_lb_policy_factory { const grpc_lb_policy_factory_vtable *vtable; }; +typedef struct grpc_lb_policy_address_token { + uint8_t *token; + size_t token_size; +} grpc_lb_policy_address_token; + typedef struct grpc_lb_policy_args { grpc_resolved_addresses *addresses; + /* It not NULL, array of load balancing tokens associated with \a addresses, + * on a 1:1 correspondence. Some indices may be NULL for missing tokens. */ + grpc_lb_policy_address_token *tokens; grpc_client_channel_factory *client_channel_factory; } grpc_lb_policy_args; diff --git a/src/core/ext/client_config/subchannel_call_holder.h b/src/core/ext/client_config/subchannel_call_holder.h index 8d2deb02f38..40a0681a3bb 100644 --- a/src/core/ext/client_config/subchannel_call_holder.h +++ b/src/core/ext/client_config/subchannel_call_holder.h @@ -81,6 +81,8 @@ typedef struct grpc_subchannel_call_holder { grpc_closure next_step; grpc_call_stack *owning_call; + + grpc_linked_mdelem lb_token_mdelem; } grpc_subchannel_call_holder; void grpc_subchannel_call_holder_init( diff --git a/src/core/ext/lb_policy/grpclb/grpclb.c b/src/core/ext/lb_policy/grpclb/grpclb.c index 693145bd6cb..3044f164c24 100644 --- a/src/core/ext/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/lb_policy/grpclb/grpclb.c @@ -164,6 +164,9 @@ typedef struct pending_pick { /* the initial metadata for the pick. See grpc_lb_policy_pick() */ grpc_metadata_batch *initial_metadata; + /* storage for the lb token initial metadata mdelem */ + grpc_linked_mdelem *lb_token_mdelem_storage; + /* bitmask passed to pick() and used for selective cancelling. See * grpc_lb_policy_cancel_picks() */ uint32_t initial_metadata_flags; @@ -188,6 +191,7 @@ static void add_pending_pick(pending_pick **root, memset(pp, 0, sizeof(pending_pick)); memset(&pp->wrapped_on_complete_arg, 0, sizeof(wrapped_rr_closure_arg)); pp->next = *root; + pp->lb_token_mdelem_storage = pick_args->lb_token_mdelem_storage; pp->pollent = pick_args->pollent; pp->target = target; pp->initial_metadata = pick_args->initial_metadata; @@ -294,7 +298,10 @@ static grpc_lb_policy *create_rr(grpc_exec_ctx *exec_ctx, (const char **)host_ports, serverlist->num_servers, ",", &uri_path_len); grpc_lb_policy_args args; + memset(&args, 0, sizeof(args)); args.client_channel_factory = glb_policy->cc_factory; + args.tokens = gpr_malloc(sizeof(grpc_lb_policy_address_token) * + serverlist->num_servers); args.addresses = gpr_malloc(sizeof(grpc_resolved_addresses)); args.addresses->naddrs = serverlist->num_servers; args.addresses->addrs = @@ -309,6 +316,14 @@ static grpc_lb_policy *create_rr(grpc_exec_ctx *exec_ctx, memcpy(args.addresses->addrs[out_addrs_idx].addr, &sa, sa_len); args.addresses->addrs[out_addrs_idx].len = sa_len; ++out_addrs_idx; + const size_t token_max_size = + GPR_ARRAY_SIZE(serverlist->servers[i]->load_balance_token); + serverlist->servers[i]->load_balance_token[token_max_size - 1] = '\0'; + args.tokens[i].token_size = + strlen(serverlist->servers[i]->load_balance_token); + args.tokens[i].token = gpr_malloc(args.tokens[i].token_size); + memcpy(args.tokens[i].token, serverlist->servers[i]->load_balance_token, + args.tokens[i].token_size); } else { gpr_log(GPR_ERROR, "Invalid LB service address '%s', ignoring.", host_ports[i]); @@ -324,11 +339,14 @@ static grpc_lb_policy *create_rr(grpc_exec_ctx *exec_ctx, gpr_free(host_ports); gpr_free(args.addresses->addrs); gpr_free(args.addresses); + gpr_free(args.tokens); return rr; } static void rr_handover(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, grpc_error *error) { + GPR_ASSERT(glb_policy->serverlist != NULL && + glb_policy->serverlist->num_servers > 0); GRPC_ERROR_REF(error); glb_policy->rr_policy = create_rr(exec_ctx, glb_policy->serverlist, glb_policy); @@ -359,7 +377,8 @@ static void rr_handover(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, (intptr_t)glb_policy->rr_policy); } const grpc_lb_policy_pick_args pick_args = { - pp->pollent, pp->initial_metadata, pp->initial_metadata_flags}; + pp->pollent, pp->initial_metadata, pp->initial_metadata_flags, + pp->lb_token_mdelem_storage}; grpc_lb_policy_pick(exec_ctx, glb_policy->rr_policy, &pick_args, pp->target, &pp->wrapped_on_complete); pp->wrapped_on_complete_arg.owning_pending_node = pp; @@ -607,6 +626,18 @@ static int glb_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, grpc_connected_subchannel **target, grpc_closure *on_complete) { glb_lb_policy *glb_policy = (glb_lb_policy *)pol; + + if (pick_args->lb_token_mdelem_storage == NULL) { + /* TODO(dgq): should this be an assert? If storage is NULL, something has + * gone very wrong at the client channel filter */ + gpr_log(GPR_ERROR, + "No mdelem storage for the LB token. Load reporting won't work " + "without it. Failing"); + *target = NULL; + grpc_exec_ctx_sched(exec_ctx, on_complete, GRPC_ERROR_NONE, NULL); + return 1; + } + gpr_mu_lock(&glb_policy->mu); int r; diff --git a/src/core/ext/lb_policy/round_robin/round_robin.c b/src/core/ext/lb_policy/round_robin/round_robin.c index 49e724a0f23..8fda405fb87 100644 --- a/src/core/ext/lb_policy/round_robin/round_robin.c +++ b/src/core/ext/lb_policy/round_robin/round_robin.c @@ -66,6 +66,7 @@ #include "src/core/ext/client_config/lb_policy_registry.h" #include "src/core/lib/debug/trace.h" #include "src/core/lib/transport/connectivity_state.h" +#include "src/core/lib/transport/static_metadata.h" typedef struct round_robin_lb_policy round_robin_lb_policy; @@ -76,15 +77,33 @@ int grpc_lb_round_robin_trace = 0; * Once a pick is available, \a target is updated and \a on_complete called. */ typedef struct pending_pick { struct pending_pick *next; + + /* polling entity for the pick()'s async notification */ grpc_polling_entity *pollent; + + /* the initial metadata for the pick. See grpc_lb_policy_pick() */ + grpc_metadata_batch *initial_metadata; + + /* storage for the lb token initial metadata mdelem */ + grpc_linked_mdelem *lb_token_mdelem_storage; + + /* bitmask passed to pick() and used for selective cancelling. See + * grpc_lb_policy_cancel_picks() */ uint32_t initial_metadata_flags; + + /* output argument where to store the pick()ed connected subchannel, or NULL + * upon error. */ grpc_connected_subchannel **target; + + /* to be invoked once the pick() has completed (regardless of success) */ grpc_closure *on_complete; } pending_pick; /** List of subchannels in a connectivity READY state */ typedef struct ready_list { grpc_subchannel *subchannel; + /* references namesake entry in subchannel_data */ + grpc_lb_policy_address_token *lb_token; struct ready_list *next; struct ready_list *prev; } ready_list; @@ -102,12 +121,19 @@ typedef struct { ready_list *ready_list_node; /** last observed connectivity */ grpc_connectivity_state connectivity_state; + /** the subchannel's target LB token */ + grpc_lb_policy_address_token *lb_token; } subchannel_data; struct round_robin_lb_policy { /** base policy: must be first */ grpc_lb_policy base; + /** total number of addresses received at creation time */ + size_t num_addresses; + /** load balancing tokens, one per incoming address */ + grpc_lb_policy_address_token *lb_tokens; + /** all our subchannels */ size_t num_subchannels; subchannel_data **subchannels; @@ -166,16 +192,19 @@ static void advance_last_picked_locked(round_robin_lb_policy *p) { if (grpc_lb_round_robin_trace) { gpr_log(GPR_DEBUG, "[READYLIST] ADVANCED LAST PICK. NOW AT NODE %p (SC %p)", - p->ready_list_last_pick, p->ready_list_last_pick->subchannel); + (void *)p->ready_list_last_pick, + (void *)p->ready_list_last_pick->subchannel); } } /** Prepends (relative to the root at p->ready_list) the connected subchannel \a * csc to the list of ready subchannels. */ static ready_list *add_connected_sc_locked(round_robin_lb_policy *p, - grpc_subchannel *sc) { + subchannel_data *sd) { ready_list *new_elem = gpr_malloc(sizeof(ready_list)); - new_elem->subchannel = sc; + memset(new_elem, 0, sizeof(ready_list)); + new_elem->subchannel = sd->subchannel; + new_elem->lb_token = sd->lb_token; if (p->ready_list.prev == NULL) { /* first element */ new_elem->next = &p->ready_list; @@ -189,7 +218,8 @@ static ready_list *add_connected_sc_locked(round_robin_lb_policy *p, p->ready_list.prev = new_elem; } if (grpc_lb_round_robin_trace) { - gpr_log(GPR_DEBUG, "[READYLIST] ADDING NODE %p (SC %p)", new_elem, sc); + gpr_log(GPR_DEBUG, "[READYLIST] ADDING NODE %p (Conn. SC %p)", + (void *)new_elem, (void *)sd->subchannel); } return new_elem; } @@ -217,7 +247,7 @@ static void remove_disconnected_sc_locked(round_robin_lb_policy *p, if (grpc_lb_round_robin_trace) { gpr_log(GPR_DEBUG, "[READYLIST] REMOVED NODE %p (SC %p)", node, - node->subchannel); + (void *)node->subchannel); } node->next = NULL; @@ -251,6 +281,13 @@ static void rr_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { gpr_free(elem); elem = tmp; } + + if (p->lb_tokens != NULL) { + for (i = 0; i < p->num_addresses; i++) { + gpr_free(p->lb_tokens[i].token); + } + gpr_free(p->lb_tokens); + } gpr_free(p); } @@ -337,7 +374,7 @@ static void start_picking(grpc_exec_ctx *exec_ctx, round_robin_lb_policy *p) { p->started_picking = 1; if (grpc_lb_round_robin_trace) { - gpr_log(GPR_DEBUG, "LB_POLICY: p=%p num_subchannels=%" PRIuPTR, p, + gpr_log(GPR_DEBUG, "LB_POLICY: p=%p num_subchannels=%" PRIuPTR, (void *)p, p->num_subchannels); } @@ -360,6 +397,23 @@ static void rr_exit_idle(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { gpr_mu_unlock(&p->mu); } +/* add lb_token of selected subchannel (address) to the call's initial + * metadata */ +static void initial_metadata_add_lb_token( + grpc_metadata_batch *initial_metadata, + grpc_linked_mdelem *lb_token_mdelem_storage, + grpc_lb_policy_address_token *lb_token) { + if (lb_token != NULL && lb_token->token_size > 0) { + GPR_ASSERT(lb_token->token != NULL); + grpc_mdstr *lb_token_mdstr = + grpc_mdstr_from_buffer(lb_token->token, lb_token->token_size); + grpc_metadata_batch_add_tail( + initial_metadata, lb_token_mdelem_storage, + grpc_mdelem_from_metadata_strings(GRPC_MDSTR_LOAD_REPORTING_INITIAL, + lb_token_mdstr)); + } +} + static int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, const grpc_lb_policy_pick_args *pick_args, grpc_connected_subchannel **target, @@ -369,17 +423,22 @@ static int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, ready_list *selected; gpr_mu_lock(&p->mu); if ((selected = peek_next_connected_locked(p))) { + /* readily available, report right away */ gpr_mu_unlock(&p->mu); *target = grpc_subchannel_get_connected_subchannel(selected->subchannel); + initial_metadata_add_lb_token(pick_args->initial_metadata, + pick_args->lb_token_mdelem_storage, + selected->lb_token); if (grpc_lb_round_robin_trace) { gpr_log(GPR_DEBUG, - "[RR PICK] TARGET <-- CONNECTED SUBCHANNEL %p (NODE %p)", *target, - selected); + "[RR PICK] TARGET <-- CONNECTED SUBCHANNEL %p (NODE %p)", + (void *)*target, (void *)selected); } /* only advance the last picked pointer if the selection was used */ advance_last_picked_locked(p); return 1; } else { + /* no pick currently available. Save for later in list of pending picks */ if (!p->started_picking) { start_picking(exec_ctx, p); } @@ -390,7 +449,9 @@ static int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, pp->pollent = pick_args->pollent; pp->target = target; pp->on_complete = on_complete; + pp->initial_metadata = pick_args->initial_metadata; pp->initial_metadata_flags = pick_args->initial_metadata_flags; + pp->lb_token_mdelem_storage = pick_args->lb_token_mdelem_storage; p->pending_picks = pp; gpr_mu_unlock(&p->mu); return 0; @@ -419,7 +480,7 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, "connecting_ready"); /* add the newly connected subchannel to the list of connected ones. * Note that it goes to the "end of the line". */ - sd->ready_list_node = add_connected_sc_locked(p, sd->subchannel); + sd->ready_list_node = add_connected_sc_locked(p, sd); /* at this point we know there's at least one suitable subchannel. Go * ahead and pick one and notify the pending suitors in * p->pending_picks. This preemtively replicates rr_pick()'s actions. */ @@ -431,12 +492,16 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, } while ((pp = p->pending_picks)) { p->pending_picks = pp->next; + + initial_metadata_add_lb_token(pp->initial_metadata, + pp->lb_token_mdelem_storage, + selected->lb_token); *pp->target = grpc_subchannel_get_connected_subchannel(selected->subchannel); if (grpc_lb_round_robin_trace) { gpr_log(GPR_DEBUG, "[RR CONN CHANGED] TARGET <-- SUBCHANNEL %p (NODE %p)", - selected->subchannel, selected); + (void *)selected->subchannel, (void *)selected); } grpc_polling_entity_del_from_pollset_set(exec_ctx, pp->pollent, p->base.interested_parties); @@ -572,13 +637,21 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx, round_robin_lb_policy *p = gpr_malloc(sizeof(*p)); memset(p, 0, sizeof(*p)); - p->subchannels = - gpr_malloc(sizeof(*p->subchannels) * args->addresses->naddrs); - memset(p->subchannels, 0, sizeof(*p->subchannels) * args->addresses->naddrs); + p->num_addresses = args->addresses->naddrs; + if (args->tokens != NULL) { + /* we need to copy because args contents aren't owned */ + p->lb_tokens = + gpr_malloc(sizeof(grpc_lb_policy_address_token) * p->num_addresses); + memcpy(p->lb_tokens, args->tokens, + sizeof(grpc_lb_policy_address_token) * p->num_addresses); + } + + p->subchannels = gpr_malloc(sizeof(subchannel_data) * p->num_addresses); + memset(p->subchannels, 0, sizeof(*p->subchannels) * p->num_addresses); grpc_subchannel_args sc_args; size_t subchannel_idx = 0; - for (size_t i = 0; i < args->addresses->naddrs; i++) { + for (size_t i = 0; i < p->num_addresses; i++) { memset(&sc_args, 0, sizeof(grpc_subchannel_args)); sc_args.addr = (struct sockaddr *)(args->addresses->addrs[i].addr); sc_args.addr_len = (size_t)args->addresses->addrs[i].len; @@ -593,12 +666,16 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx, sd->policy = p; sd->index = subchannel_idx; sd->subchannel = subchannel; + if (p->lb_tokens != NULL) { + sd->lb_token = &p->lb_tokens[i]; + } ++subchannel_idx; grpc_closure_init(&sd->connectivity_changed_closure, rr_connectivity_changed, sd); } } if (subchannel_idx == 0) { + /* couldn't create any subchannel. Bail out */ gpr_free(p->subchannels); gpr_free(p); return NULL; diff --git a/test/cpp/grpclb/grpclb_test.cc b/test/cpp/grpclb/grpclb_test.cc index b2fdce2963d..f83935fa1a1 100644 --- a/test/cpp/grpclb/grpclb_test.cc +++ b/test/cpp/grpclb/grpclb_test.cc @@ -61,6 +61,7 @@ extern "C" { #include "src/proto/grpc/lb/v1/load_balancer.pb.h" #define NUM_BACKENDS 4 +#define PAYLOAD "hello you" // TODO(dgq): Other scenarios in need of testing: // - Send an empty serverlist update and verify that the client request blocks @@ -303,6 +304,12 @@ static void start_backend_server(server_fixture *sf) { return; } GPR_ASSERT(ev.type == GRPC_OP_COMPLETE); + char *expected_token; + GPR_ASSERT(gpr_asprintf(&expected_token, "token%d", sf->port) > 0); + GPR_ASSERT(contains_metadata(&request_metadata_recv, + "load-reporting-initial", expected_token)); + gpr_free(expected_token); + gpr_log(GPR_INFO, "Server[%s] after tag 100", sf->servers_hostport); op = ops; @@ -321,8 +328,7 @@ static void start_backend_server(server_fixture *sf) { gpr_log(GPR_INFO, "Server[%s] after tag 101", sf->servers_hostport); bool exit = false; - gpr_slice response_payload_slice = - gpr_slice_from_copied_string("hello you"); + gpr_slice response_payload_slice = gpr_slice_from_copied_string(PAYLOAD); while (!exit) { op = ops; op->op = GRPC_OP_RECV_MESSAGE; @@ -474,10 +480,9 @@ static void perform_request(client_fixture *cf) { error = grpc_call_start_batch(c, ops, (size_t)(op - ops), tag(2), NULL); GPR_ASSERT(GRPC_CALL_OK == error); - peer = grpc_call_get_peer(c); cq_expect_completion(cqv, tag(2), 1); cq_verify(cqv); - gpr_free(peer); + GPR_ASSERT(byte_buffer_eq_string(response_payload_recv, PAYLOAD)); grpc_byte_buffer_destroy(request_payload); grpc_byte_buffer_destroy(response_payload_recv); From 601bb128b4769adeb26d7ba5e8b7d804a5732a51 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Thu, 18 Aug 2016 15:03:59 -0700 Subject: [PATCH 03/25] Fixed op ordering in grpclb internal client --- src/core/ext/lb_policy/grpclb/grpclb.c | 36 +++++--------------- test/cpp/grpclb/grpclb_test.cc | 47 ++++++++++++++++++-------- 2 files changed, 41 insertions(+), 42 deletions(-) diff --git a/src/core/ext/lb_policy/grpclb/grpclb.c b/src/core/ext/lb_policy/grpclb/grpclb.c index af913d8a9df..1ef475d086e 100644 --- a/src/core/ext/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/lb_policy/grpclb/grpclb.c @@ -546,7 +546,6 @@ static void glb_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, *target = NULL; grpc_exec_ctx_sched(exec_ctx, &pp->wrapped_on_complete, GRPC_ERROR_CANCELLED, NULL); - gpr_free(pp); } else { pp->next = glb_policy->pending_picks; glb_policy->pending_picks = pp; @@ -576,7 +575,6 @@ static void glb_cancel_picks(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, exec_ctx, pp->pollent, glb_policy->base.interested_parties); grpc_exec_ctx_sched(exec_ctx, &pp->wrapped_on_complete, GRPC_ERROR_CANCELLED, NULL); - gpr_free(pp); } else { pp->next = glb_policy->pending_picks; glb_policy->pending_picks = pp; @@ -702,9 +700,6 @@ typedef struct lb_client_data { /* called once initial metadata's been sent */ grpc_closure md_sent; - /* called once initial metadata's been received */ - grpc_closure md_rcvd; - /* called once the LoadBalanceRequest has been sent to the LB server. See * src/proto/grpc/.../load_balancer.proto */ grpc_closure req_sent; @@ -741,7 +736,6 @@ typedef struct lb_client_data { } lb_client_data; static void md_sent_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error); -static void md_recv_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error); static void req_sent_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error); static void res_recv_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error); static void close_sent_cb(grpc_exec_ctx *exec_ctx, void *arg, @@ -756,7 +750,6 @@ static lb_client_data *lb_client_data_create(glb_lb_policy *glb_policy) { gpr_mu_init(&lb_client->mu); grpc_closure_init(&lb_client->md_sent, md_sent_cb, lb_client); - grpc_closure_init(&lb_client->md_rcvd, md_recv_cb, lb_client); grpc_closure_init(&lb_client->req_sent, req_sent_cb, lb_client); grpc_closure_init(&lb_client->res_rcvd, res_recv_cb, lb_client); grpc_closure_init(&lb_client->close_sent, close_sent_cb, lb_client); @@ -855,23 +848,6 @@ static void md_sent_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { grpc_op ops[1]; memset(ops, 0, sizeof(ops)); grpc_op *op = ops; - op->op = GRPC_OP_RECV_INITIAL_METADATA; - op->data.recv_initial_metadata = &lb_client->initial_metadata_recv; - op->flags = 0; - op->reserved = NULL; - op++; - grpc_call_error call_error = grpc_call_start_batch_and_execute( - exec_ctx, lb_client->lb_call, ops, (size_t)(op - ops), - &lb_client->md_rcvd); - GPR_ASSERT(GRPC_CALL_OK == call_error); -} - -static void md_recv_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { - lb_client_data *lb_client = arg; - GPR_ASSERT(lb_client->lb_call); - grpc_op ops[1]; - memset(ops, 0, sizeof(ops)); - grpc_op *op = ops; op->op = GRPC_OP_SEND_MESSAGE; op->data.send_message = lb_client->request_payload; @@ -886,11 +862,18 @@ static void md_recv_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { static void req_sent_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { lb_client_data *lb_client = arg; + GPR_ASSERT(lb_client->lb_call); - grpc_op ops[1]; + grpc_op ops[2]; memset(ops, 0, sizeof(ops)); grpc_op *op = ops; + op->op = GRPC_OP_RECV_INITIAL_METADATA; + op->data.recv_initial_metadata = &lb_client->initial_metadata_recv; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_RECV_MESSAGE; op->data.recv_message = &lb_client->response_payload; op->flags = 0; @@ -909,8 +892,7 @@ static void res_recv_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { grpc_op *op = ops; if (lb_client->response_payload != NULL) { /* Received data from the LB server. Look inside - * lb_client->response_payload, for - * a serverlist. */ + * lb_client->response_payload, for a serverlist. */ grpc_byte_buffer_reader bbr; grpc_byte_buffer_reader_init(&bbr, lb_client->response_payload); gpr_slice response_slice = grpc_byte_buffer_reader_readall(&bbr); diff --git a/test/cpp/grpclb/grpclb_test.cc b/test/cpp/grpclb/grpclb_test.cc index b2fdce2963d..4720ddcdf38 100644 --- a/test/cpp/grpclb/grpclb_test.cc +++ b/test/cpp/grpclb/grpclb_test.cc @@ -39,6 +39,7 @@ extern "C" { #include +#include #include #include #include @@ -181,20 +182,9 @@ static void start_lb_server(server_fixture *sf, int *ports, size_t nports, cq_verify(cqv); gpr_log(GPR_INFO, "LB Server[%s] after tag 200", sf->servers_hostport); - op = ops; - op->op = GRPC_OP_SEND_INITIAL_METADATA; - op->data.send_initial_metadata.count = 0; - op->flags = 0; - op->reserved = NULL; - op++; - op->op = GRPC_OP_RECV_CLOSE_ON_SERVER; - op->data.recv_close_on_server.cancelled = &was_cancelled; - op->flags = 0; - op->reserved = NULL; - op++; - error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(201), NULL); - GPR_ASSERT(GRPC_CALL_OK == error); - gpr_log(GPR_INFO, "LB Server[%s] after tag 201", sf->servers_hostport); + // make sure we've received the initial metadata from the grpclb request. + GPR_ASSERT(request_metadata_recv.count > 0); + GPR_ASSERT(request_metadata_recv.metadata != NULL); // receive request for backends op = ops; @@ -208,9 +198,36 @@ static void start_lb_server(server_fixture *sf, int *ports, size_t nports, cq_expect_completion(cqv, tag(202), 1); cq_verify(cqv); gpr_log(GPR_INFO, "LB Server[%s] after RECV_MSG", sf->servers_hostport); - // TODO(dgq): validate request. + + // validate initial request. + grpc_byte_buffer_reader bbr; + grpc_byte_buffer_reader_init(&bbr, request_payload_recv); + gpr_slice request_payload_slice = grpc_byte_buffer_reader_readall(&bbr); + grpc::lb::v1::LoadBalanceRequest request; + request.ParseFromArray(GPR_SLICE_START_PTR(request_payload_slice), + GPR_SLICE_LENGTH(request_payload_slice)); + GPR_ASSERT(request.has_initial_request()); + GPR_ASSERT(request.initial_request().name() == "load.balanced.service.name"); + gpr_slice_unref(request_payload_slice); + grpc_byte_buffer_reader_destroy(&bbr); grpc_byte_buffer_destroy(request_payload_recv); + gpr_slice response_payload_slice; + op = ops; + op->op = GRPC_OP_SEND_INITIAL_METADATA; + op->data.send_initial_metadata.count = 0; + op->flags = 0; + op->reserved = NULL; + op++; + op->op = GRPC_OP_RECV_CLOSE_ON_SERVER; + op->data.recv_close_on_server.cancelled = &was_cancelled; + op->flags = 0; + op->reserved = NULL; + op++; + error = grpc_call_start_batch(s, ops, (size_t)(op - ops), tag(201), NULL); + GPR_ASSERT(GRPC_CALL_OK == error); + gpr_log(GPR_INFO, "LB Server[%s] after tag 201", sf->servers_hostport); + for (int i = 0; i < 2; i++) { if (i == 0) { // First half of the ports. From 348cfdb5b62c936b3df7f6ea5c637c6b7e24e830 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Fri, 19 Aug 2016 12:19:43 -0700 Subject: [PATCH 04/25] Fixed error references in grpclb.c --- src/core/ext/lb_policy/grpclb/grpclb.c | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/core/ext/lb_policy/grpclb/grpclb.c b/src/core/ext/lb_policy/grpclb/grpclb.c index 5e97d4c6ac9..1b7104bca66 100644 --- a/src/core/ext/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/lb_policy/grpclb/grpclb.c @@ -76,9 +76,9 @@ * operations in progress over the old RR instance. This is done by * decreasing the reference count on the old policy. The moment no more * references are held on the old RR policy, it'll be destroyed and \a - * rr_connectivity_changed notified with a \a GRPC_CHANNEL_SHUTDOWN state. - * At this point we can transition to a new RR instance safely, which is done - * once again via \a rr_handover(). + * glb_rr_connectivity_changed notified with a \a GRPC_CHANNEL_SHUTDOWN + * state. At this point we can transition to a new RR instance safely, which + * is done once again via \a rr_handover(). * * * Once a RR policy instance is in place (and getting updated as described), @@ -347,7 +347,6 @@ static void rr_handover(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, grpc_error *error) { GPR_ASSERT(glb_policy->serverlist != NULL && glb_policy->serverlist->num_servers > 0); - GRPC_ERROR_REF(error); glb_policy->rr_policy = create_rr(exec_ctx, glb_policy->serverlist, glb_policy); @@ -362,8 +361,8 @@ static void rr_handover(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, exec_ctx, glb_policy->rr_policy, &glb_policy->rr_connectivity->state, &glb_policy->rr_connectivity->on_change); grpc_connectivity_state_set(exec_ctx, &glb_policy->state_tracker, - glb_policy->rr_connectivity->state, error, - "rr_handover"); + glb_policy->rr_connectivity->state, + GRPC_ERROR_REF(error), "rr_handover"); grpc_lb_policy_exit_idle(exec_ctx, glb_policy->rr_policy); /* flush pending ops */ @@ -397,13 +396,13 @@ static void rr_handover(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, &pping->wrapped_notify); pping->wrapped_notify_arg.owning_pending_node = pping; } - GRPC_ERROR_UNREF(error); } -static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, - grpc_error *error) { +static void glb_rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { rr_connectivity_data *rr_conn_data = arg; glb_lb_policy *glb_policy = rr_conn_data->glb_policy; + if (rr_conn_data->state == GRPC_CHANNEL_SHUTDOWN) { if (glb_policy->serverlist != NULL) { /* a RR policy is shutting down but there's a serverlist available -> @@ -417,8 +416,8 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, if (error == GRPC_ERROR_NONE) { /* RR not shutting down. Mimic the RR's policy state */ grpc_connectivity_state_set(exec_ctx, &glb_policy->state_tracker, - rr_conn_data->state, error, - "rr_connectivity_changed"); + rr_conn_data->state, GRPC_ERROR_REF(error), + "glb_rr_connectivity_changed"); /* resubscribe */ grpc_lb_policy_notify_on_state_change(exec_ctx, glb_policy->rr_policy, &rr_conn_data->state, @@ -427,7 +426,6 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, gpr_free(rr_conn_data); } } - GRPC_ERROR_UNREF(error); } static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, @@ -482,7 +480,7 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, rr_connectivity_data *rr_connectivity = gpr_malloc(sizeof(rr_connectivity_data)); memset(rr_connectivity, 0, sizeof(rr_connectivity_data)); - grpc_closure_init(&rr_connectivity->on_change, rr_connectivity_changed, + grpc_closure_init(&rr_connectivity->on_change, glb_rr_connectivity_changed, rr_connectivity); rr_connectivity->glb_policy = glb_policy; glb_policy->rr_connectivity = rr_connectivity; @@ -958,7 +956,7 @@ static void res_recv_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { } else { /* unref the RR policy, eventually leading to its substitution with a * new one constructed from the received serverlist (see - * rr_connectivity_changed) */ + * glb_rr_connectivity_changed) */ GRPC_LB_POLICY_UNREF(exec_ctx, lb_client->glb_policy->rr_policy, "serverlist_received"); } From 1f350ed506efb242d0a863c7c78d28da7364a727 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Fri, 19 Aug 2016 22:51:31 -0700 Subject: [PATCH 05/25] fixed address ignoring when invalid addr given --- src/core/ext/lb_policy/grpclb/grpclb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/ext/lb_policy/grpclb/grpclb.c b/src/core/ext/lb_policy/grpclb/grpclb.c index 1b7104bca66..43674504b53 100644 --- a/src/core/ext/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/lb_policy/grpclb/grpclb.c @@ -303,9 +303,8 @@ static grpc_lb_policy *create_rr(grpc_exec_ctx *exec_ctx, args.tokens = gpr_malloc(sizeof(grpc_lb_policy_address_token) * serverlist->num_servers); args.addresses = gpr_malloc(sizeof(grpc_resolved_addresses)); - args.addresses->naddrs = serverlist->num_servers; args.addresses->addrs = - gpr_malloc(sizeof(grpc_resolved_address) * args.addresses->naddrs); + gpr_malloc(sizeof(grpc_resolved_address) * serverlist->num_servers); size_t out_addrs_idx = 0; for (size_t i = 0; i < serverlist->num_servers; ++i) { grpc_uri uri; @@ -329,6 +328,7 @@ static grpc_lb_policy *create_rr(grpc_exec_ctx *exec_ctx, host_ports[i]); } } + args.addresses->naddrs = out_addrs_idx; grpc_lb_policy *rr = grpc_lb_policy_create(exec_ctx, "round_robin", &args); From 7a6db63bb8608c9a97f6acdbec66ecbe18d3463c Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Sat, 20 Aug 2016 16:00:06 -0700 Subject: [PATCH 06/25] Updated LB proto and nanopb version --- .../proto/grpc/lb/v1/load_balancer.pb.c | 8 +++-- .../proto/grpc/lb/v1/load_balancer.pb.h | 26 ++++++++++++----- src/proto/grpc/lb/v1/load_balancer.proto | 29 +++++++++---------- third_party/nanopb | 2 +- tools/run_tests/sanity/check_submodules.sh | 2 +- 5 files changed, 39 insertions(+), 28 deletions(-) diff --git a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c index 52e11c40bb0..2676714175a 100644 --- a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c +++ b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c @@ -31,10 +31,11 @@ * */ /* Automatically generated nanopb constant definitions */ -/* Generated by nanopb-0.3.5-dev */ +/* Generated by nanopb-0.3.7-dev */ #include "src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h" +/* @@protoc_insertion_point(includes) */ #if PB_PROTO_HEADER_VERSION != 30 #error Regenerate this file with the current version of nanopb generator. #endif @@ -72,7 +73,7 @@ const pb_field_t grpc_lb_v1_LoadBalanceResponse_fields[3] = { }; const pb_field_t grpc_lb_v1_InitialLoadBalanceResponse_fields[3] = { - PB_FIELD( 2, STRING , OPTIONAL, STATIC , FIRST, grpc_lb_v1_InitialLoadBalanceResponse, load_balancer_delegate, load_balancer_delegate, 0), + PB_FIELD( 1, STRING , OPTIONAL, STATIC , FIRST, grpc_lb_v1_InitialLoadBalanceResponse, load_balancer_delegate, load_balancer_delegate, 0), PB_FIELD( 3, MESSAGE , OPTIONAL, STATIC , OTHER, grpc_lb_v1_InitialLoadBalanceResponse, client_stats_report_interval, load_balancer_delegate, &grpc_lb_v1_Duration_fields), PB_LAST_FIELD }; @@ -84,7 +85,7 @@ const pb_field_t grpc_lb_v1_ServerList_fields[3] = { }; const pb_field_t grpc_lb_v1_Server_fields[5] = { - PB_FIELD( 1, STRING , OPTIONAL, STATIC , FIRST, grpc_lb_v1_Server, ip_address, ip_address, 0), + PB_FIELD( 1, BYTES , OPTIONAL, STATIC , FIRST, grpc_lb_v1_Server, ip_address, ip_address, 0), PB_FIELD( 2, INT32 , OPTIONAL, STATIC , OTHER, grpc_lb_v1_Server, port, ip_address, 0), PB_FIELD( 3, STRING , OPTIONAL, STATIC , OTHER, grpc_lb_v1_Server, load_balance_token, port, 0), PB_FIELD( 4, BOOL , OPTIONAL, STATIC , OTHER, grpc_lb_v1_Server, drop_request, load_balance_token, 0), @@ -116,3 +117,4 @@ PB_STATIC_ASSERT((pb_membersize(grpc_lb_v1_LoadBalanceRequest, initial_request) #endif +/* @@protoc_insertion_point(eof) */ diff --git a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h index 46fe588f72d..3c11074b229 100644 --- a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h +++ b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h @@ -31,11 +31,12 @@ * */ /* Automatically generated nanopb header */ -/* Generated by nanopb-0.3.5-dev */ +/* Generated by nanopb-0.3.7-dev */ -#ifndef GRPC_CORE_EXT_LB_POLICY_GRPCLB_PROTO_GRPC_LB_V1_LOAD_BALANCER_PB_H -#define GRPC_CORE_EXT_LB_POLICY_GRPCLB_PROTO_GRPC_LB_V1_LOAD_BALANCER_PB_H +#ifndef PB_GRPC_LB_V1_LOAD_BALANCER_PB_H_INCLUDED +#define PB_GRPC_LB_V1_LOAD_BALANCER_PB_H_INCLUDED #include "third_party/nanopb/pb.h" +/* @@protoc_insertion_point(includes) */ #if PB_PROTO_HEADER_VERSION != 30 #error Regenerate this file with the current version of nanopb generator. #endif @@ -52,6 +53,7 @@ typedef struct _grpc_lb_v1_ClientStats { int64_t client_rpc_errors; bool has_dropped_requests; int64_t dropped_requests; +/* @@protoc_insertion_point(struct:grpc_lb_v1_ClientStats) */ } grpc_lb_v1_ClientStats; typedef struct _grpc_lb_v1_Duration { @@ -59,22 +61,26 @@ typedef struct _grpc_lb_v1_Duration { int64_t seconds; bool has_nanos; int32_t nanos; +/* @@protoc_insertion_point(struct:grpc_lb_v1_Duration) */ } grpc_lb_v1_Duration; typedef struct _grpc_lb_v1_InitialLoadBalanceRequest { bool has_name; char name[128]; +/* @@protoc_insertion_point(struct:grpc_lb_v1_InitialLoadBalanceRequest) */ } grpc_lb_v1_InitialLoadBalanceRequest; +typedef PB_BYTES_ARRAY_T(46) grpc_lb_v1_Server_ip_address_t; typedef struct _grpc_lb_v1_Server { bool has_ip_address; - char ip_address[46]; + grpc_lb_v1_Server_ip_address_t ip_address; bool has_port; int32_t port; bool has_load_balance_token; char load_balance_token[64]; bool has_drop_request; bool drop_request; +/* @@protoc_insertion_point(struct:grpc_lb_v1_Server) */ } grpc_lb_v1_Server; typedef struct _grpc_lb_v1_InitialLoadBalanceResponse { @@ -82,6 +88,7 @@ typedef struct _grpc_lb_v1_InitialLoadBalanceResponse { char load_balancer_delegate[64]; bool has_client_stats_report_interval; grpc_lb_v1_Duration client_stats_report_interval; +/* @@protoc_insertion_point(struct:grpc_lb_v1_InitialLoadBalanceResponse) */ } grpc_lb_v1_InitialLoadBalanceResponse; typedef struct _grpc_lb_v1_LoadBalanceRequest { @@ -89,12 +96,14 @@ typedef struct _grpc_lb_v1_LoadBalanceRequest { grpc_lb_v1_InitialLoadBalanceRequest initial_request; bool has_client_stats; grpc_lb_v1_ClientStats client_stats; +/* @@protoc_insertion_point(struct:grpc_lb_v1_LoadBalanceRequest) */ } grpc_lb_v1_LoadBalanceRequest; typedef struct _grpc_lb_v1_ServerList { pb_callback_t servers; bool has_expiration_interval; grpc_lb_v1_Duration expiration_interval; +/* @@protoc_insertion_point(struct:grpc_lb_v1_ServerList) */ } grpc_lb_v1_ServerList; typedef struct _grpc_lb_v1_LoadBalanceResponse { @@ -102,6 +111,7 @@ typedef struct _grpc_lb_v1_LoadBalanceResponse { grpc_lb_v1_InitialLoadBalanceResponse initial_response; bool has_server_list; grpc_lb_v1_ServerList server_list; +/* @@protoc_insertion_point(struct:grpc_lb_v1_LoadBalanceResponse) */ } grpc_lb_v1_LoadBalanceResponse; /* Default values for struct fields */ @@ -114,7 +124,7 @@ typedef struct _grpc_lb_v1_LoadBalanceResponse { #define grpc_lb_v1_LoadBalanceResponse_init_default {false, grpc_lb_v1_InitialLoadBalanceResponse_init_default, false, grpc_lb_v1_ServerList_init_default} #define grpc_lb_v1_InitialLoadBalanceResponse_init_default {false, "", false, grpc_lb_v1_Duration_init_default} #define grpc_lb_v1_ServerList_init_default {{{NULL}, NULL}, false, grpc_lb_v1_Duration_init_default} -#define grpc_lb_v1_Server_init_default {false, "", false, 0, false, "", false, 0} +#define grpc_lb_v1_Server_init_default {false, {0, {0}}, false, 0, false, "", false, 0} #define grpc_lb_v1_Duration_init_zero {false, 0, false, 0} #define grpc_lb_v1_LoadBalanceRequest_init_zero {false, grpc_lb_v1_InitialLoadBalanceRequest_init_zero, false, grpc_lb_v1_ClientStats_init_zero} #define grpc_lb_v1_InitialLoadBalanceRequest_init_zero {false, ""} @@ -122,7 +132,7 @@ typedef struct _grpc_lb_v1_LoadBalanceResponse { #define grpc_lb_v1_LoadBalanceResponse_init_zero {false, grpc_lb_v1_InitialLoadBalanceResponse_init_zero, false, grpc_lb_v1_ServerList_init_zero} #define grpc_lb_v1_InitialLoadBalanceResponse_init_zero {false, "", false, grpc_lb_v1_Duration_init_zero} #define grpc_lb_v1_ServerList_init_zero {{{NULL}, NULL}, false, grpc_lb_v1_Duration_init_zero} -#define grpc_lb_v1_Server_init_zero {false, "", false, 0, false, "", false, 0} +#define grpc_lb_v1_Server_init_zero {false, {0, {0}}, false, 0, false, "", false, 0} /* Field tags (for use in manual encoding/decoding) */ #define grpc_lb_v1_ClientStats_total_requests_tag 1 @@ -135,7 +145,7 @@ typedef struct _grpc_lb_v1_LoadBalanceResponse { #define grpc_lb_v1_Server_port_tag 2 #define grpc_lb_v1_Server_load_balance_token_tag 3 #define grpc_lb_v1_Server_drop_request_tag 4 -#define grpc_lb_v1_InitialLoadBalanceResponse_load_balancer_delegate_tag 2 +#define grpc_lb_v1_InitialLoadBalanceResponse_load_balancer_delegate_tag 1 #define grpc_lb_v1_InitialLoadBalanceResponse_client_stats_report_interval_tag 3 #define grpc_lb_v1_LoadBalanceRequest_initial_request_tag 1 #define grpc_lb_v1_LoadBalanceRequest_client_stats_tag 2 @@ -161,6 +171,7 @@ extern const pb_field_t grpc_lb_v1_Server_fields[5]; #define grpc_lb_v1_ClientStats_size 33 #define grpc_lb_v1_LoadBalanceResponse_size (98 + grpc_lb_v1_ServerList_size) #define grpc_lb_v1_InitialLoadBalanceResponse_size 90 +/* grpc_lb_v1_ServerList_size depends on runtime parameters */ #define grpc_lb_v1_Server_size 127 /* Message IDs (where set with "msgid" option) */ @@ -174,5 +185,6 @@ extern const pb_field_t grpc_lb_v1_Server_fields[5]; #ifdef __cplusplus } /* extern "C" */ #endif +/* @@protoc_insertion_point(eof) */ #endif diff --git a/src/proto/grpc/lb/v1/load_balancer.proto b/src/proto/grpc/lb/v1/load_balancer.proto index 1bcad0b1d4b..b4a33f33389 100644 --- a/src/proto/grpc/lb/v1/load_balancer.proto +++ b/src/proto/grpc/lb/v1/load_balancer.proto @@ -32,7 +32,6 @@ syntax = "proto3"; package grpc.lb.v1; message Duration { - // Signed seconds of the span of time. Must be from -315,576,000,000 // to +315,576,000,000 inclusive. int64 seconds = 1; @@ -93,16 +92,11 @@ message LoadBalanceResponse { } message InitialLoadBalanceResponse { - oneof initial_response_type { - // TODO(zhangkun83): ClientConfig not yet defined - //ClientConfig client_config = 1; - - // This is an application layer redirect that indicates the client should - // use the specified server for load balancing. When this field is set in - // the response, the client should open a separate connection to the - // load_balancer_delegate and call the BalanceLoad method. - string load_balancer_delegate = 2; - } + // This is an application layer redirect that indicates the client should use + // the specified server for load balancing. When this field is non-empty in + // the response, the client should open a separate connection to the + // load_balancer_delegate and call the BalanceLoad method. + string load_balancer_delegate = 1; // This interval defines how often the client should send the client stats // to the load balancer. Stats should only be reported when the duration is @@ -125,14 +119,17 @@ message ServerList { } message Server { - // A resolved address and port for the server. The IP address string may + // A resolved address for the server, serialized in network-byte-order. It may // either be an IPv4 or IPv6 address. - string ip_address = 1; + bytes ip_address = 1; + + // A resolved port number for the server. int32 port = 2; - // An opaque token that is passed from the client to the server in metadata. - // The server may expect this token to indicate that the request from the - // client was load balanced. + // An opaque but printable token given to the frontend for each pick. All + // frontend requests for that pick must include the token in its initial + // metadata. The token is used by the backend to verify the request and to + // allow the backend to report load to the gRPC LB system. string load_balance_token = 3; // Indicates whether this particular request should be dropped by the client diff --git a/third_party/nanopb b/third_party/nanopb index f8ac4637662..68a86e96481 160000 --- a/third_party/nanopb +++ b/third_party/nanopb @@ -1 +1 @@ -Subproject commit f8ac463766281625ad710900479130c7fcb4d63b +Subproject commit 68a86e96481e6bea987df8de47027847b30c325b diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index d2ab5b01cc1..4dac74ba9f5 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -44,7 +44,7 @@ cat << EOF | awk '{ print $1 }' | sort > $want_submodules c880e42ba1c8032d4cdde2aba0541d8a9d9fa2e9 third_party/boringssl (version_for_cocoapods_2.0-100-gc880e42) 05b155ff59114735ec8cd089f669c4c3d8f59029 third_party/gflags (v2.1.0-45-g05b155f) c99458533a9b4c743ed51537e25989ea55944908 third_party/googletest (release-1.7.0) - f8ac463766281625ad710900479130c7fcb4d63b third_party/nanopb (nanopb-0.3.4-29-gf8ac463) + 68a86e96481e6bea987df8de47027847b30c325b third_party/nanopb (nanopb-0.3.6-6-g68a86e9) e8ae137c96444ea313485ed1118c5e43b2099cf1 third_party/protobuf (v3.0.0-beta-4-74-ge8ae137) 50893291621658f355bc5b4d450a8d06a563053d third_party/zlib (v1.2.8) bcad91771b7f0bff28a1cac1981d7ef2b9bcef3c third_party/thrift From 88ece5876413941b5354280abca06cff4e0ad6cb Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Mon, 22 Aug 2016 15:03:12 -0700 Subject: [PATCH 07/25] Update protobuf version to 3.0.0 --- tools/codegen/core/gen_nano_proto.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/codegen/core/gen_nano_proto.sh b/tools/codegen/core/gen_nano_proto.sh index c880fc23a23..df107c208f6 100755 --- a/tools/codegen/core/gen_nano_proto.sh +++ b/tools/codegen/core/gen_nano_proto.sh @@ -123,7 +123,7 @@ popd # this should be the same version as the submodule we compile against # ideally we'd update this as a template to ensure that -pip install protobuf==3.0.0b2 +pip install protobuf==3.0.0 pushd "$(dirname $INPUT_PROTO)" > /dev/null From 592c28dbbae120f4a6cedb16042e5d10c35395f6 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Mon, 22 Aug 2016 15:04:47 -0700 Subject: [PATCH 08/25] LB nanopb options updated: - ip addresses max size, now that they are binary encoded - token size increased in 1 (to 65) to accommodate final \0 --- .../lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h | 6 +++--- src/proto/grpc/lb/v1/load_balancer.options | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h index 3c11074b229..832c851ca3b 100644 --- a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h +++ b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h @@ -70,14 +70,14 @@ typedef struct _grpc_lb_v1_InitialLoadBalanceRequest { /* @@protoc_insertion_point(struct:grpc_lb_v1_InitialLoadBalanceRequest) */ } grpc_lb_v1_InitialLoadBalanceRequest; -typedef PB_BYTES_ARRAY_T(46) grpc_lb_v1_Server_ip_address_t; +typedef PB_BYTES_ARRAY_T(16) grpc_lb_v1_Server_ip_address_t; typedef struct _grpc_lb_v1_Server { bool has_ip_address; grpc_lb_v1_Server_ip_address_t ip_address; bool has_port; int32_t port; bool has_load_balance_token; - char load_balance_token[64]; + char load_balance_token[65]; bool has_drop_request; bool drop_request; /* @@protoc_insertion_point(struct:grpc_lb_v1_Server) */ @@ -172,7 +172,7 @@ extern const pb_field_t grpc_lb_v1_Server_fields[5]; #define grpc_lb_v1_LoadBalanceResponse_size (98 + grpc_lb_v1_ServerList_size) #define grpc_lb_v1_InitialLoadBalanceResponse_size 90 /* grpc_lb_v1_ServerList_size depends on runtime parameters */ -#define grpc_lb_v1_Server_size 127 +#define grpc_lb_v1_Server_size 98 /* Message IDs (where set with "msgid" option) */ #ifdef PB_MSGID diff --git a/src/proto/grpc/lb/v1/load_balancer.options b/src/proto/grpc/lb/v1/load_balancer.options index d90366996e7..a9398d5f474 100644 --- a/src/proto/grpc/lb/v1/load_balancer.options +++ b/src/proto/grpc/lb/v1/load_balancer.options @@ -1,6 +1,6 @@ grpc.lb.v1.InitialLoadBalanceRequest.name max_size:128 grpc.lb.v1.InitialLoadBalanceResponse.client_config max_size:64 grpc.lb.v1.InitialLoadBalanceResponse.load_balancer_delegate max_size:64 -grpc.lb.v1.Server.ip_address max_size:46 -grpc.lb.v1.Server.load_balance_token max_size:64 +grpc.lb.v1.Server.ip_address max_size:16 +grpc.lb.v1.Server.load_balance_token max_size:65 load_balancer.proto no_unions:true From 18bc43b4eaf476b41a1a1af1c3c02d8cf70f0f52 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Mon, 22 Aug 2016 15:05:32 -0700 Subject: [PATCH 09/25] Print nanopb error messages in load_balancer_api.c --- src/core/ext/lb_policy/grpclb/load_balancer_api.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/core/ext/lb_policy/grpclb/load_balancer_api.c b/src/core/ext/lb_policy/grpclb/load_balancer_api.c index f4720a13455..a8881004a06 100644 --- a/src/core/ext/lb_policy/grpclb/load_balancer_api.c +++ b/src/core/ext/lb_policy/grpclb/load_balancer_api.c @@ -57,6 +57,7 @@ static bool decode_serverlist(pb_istream_t *stream, const pb_field_t *field, if (dec_arg->first_pass) { /* count how many server do we have */ grpc_grpclb_server server; if (!pb_decode(stream, grpc_lb_v1_Server_fields, &server)) { + gpr_log(GPR_ERROR, "nanopb error: %s", PB_GET_ERROR(stream)); return false; } dec_arg->num_servers++; @@ -69,6 +70,7 @@ static bool decode_serverlist(pb_istream_t *stream, const pb_field_t *field, gpr_malloc(sizeof(grpc_grpclb_server *) * dec_arg->num_servers); } if (!pb_decode(stream, grpc_lb_v1_Server_fields, server)) { + gpr_log(GPR_ERROR, "nanopb error: %s", PB_GET_ERROR(stream)); return false; } dec_arg->servers[dec_arg->decoding_idx++] = server; @@ -118,6 +120,7 @@ grpc_grpclb_initial_response *grpc_grpclb_initial_response_parse( grpc_grpclb_response res; memset(&res, 0, sizeof(grpc_grpclb_response)); if (!pb_decode(&stream, grpc_lb_v1_LoadBalanceResponse_fields, &res)) { + gpr_log(GPR_ERROR, "nanopb error: %s", PB_GET_ERROR(&stream)); return NULL; } grpc_grpclb_initial_response *initial_res = @@ -145,6 +148,7 @@ grpc_grpclb_serverlist *grpc_grpclb_response_parse_serverlist( arg.first_pass = true; status = pb_decode(&stream, grpc_lb_v1_LoadBalanceResponse_fields, &res); if (!status) { + gpr_log(GPR_ERROR, "nanopb error: %s", PB_GET_ERROR(&stream)); return NULL; } @@ -152,6 +156,7 @@ grpc_grpclb_serverlist *grpc_grpclb_response_parse_serverlist( status = pb_decode(&stream_at_start, grpc_lb_v1_LoadBalanceResponse_fields, &res); if (!status) { + gpr_log(GPR_ERROR, "nanopb error: %s", PB_GET_ERROR(&stream)); return NULL; } From 8a81aa12fbdb6ccb0f4c80dfc1f8cdf263097780 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Mon, 22 Aug 2016 15:06:49 -0700 Subject: [PATCH 10/25] Changes to grpclb and tests for binary ip addresses. --- src/core/ext/lb_policy/grpclb/grpclb.c | 79 +++++++++++-------- .../ext/lb_policy/grpclb/load_balancer_api.h | 1 + test/cpp/grpclb/grpclb_test.cc | 42 ++++++---- 3 files changed, 71 insertions(+), 51 deletions(-) diff --git a/src/core/ext/lb_policy/grpclb/grpclb.c b/src/core/ext/lb_policy/grpclb/grpclb.c index 43674504b53..d0316e648d7 100644 --- a/src/core/ext/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/lb_policy/grpclb/grpclb.c @@ -96,6 +96,9 @@ * - Implement LB service forwarding (point 2c. in the doc's diagram). */ +#include +#include + #include #include @@ -285,17 +288,7 @@ struct rr_connectivity_data { static grpc_lb_policy *create_rr(grpc_exec_ctx *exec_ctx, const grpc_grpclb_serverlist *serverlist, glb_lb_policy *glb_policy) { - /* TODO(dgq): support mixed ip version */ GPR_ASSERT(serverlist != NULL && serverlist->num_servers > 0); - char **host_ports = gpr_malloc(sizeof(char *) * serverlist->num_servers); - for (size_t i = 0; i < serverlist->num_servers; ++i) { - gpr_join_host_port(&host_ports[i], serverlist->servers[i]->ip_address, - serverlist->servers[i]->port); - } - - size_t uri_path_len; - char *concat_ipports = gpr_strjoin_sep( - (const char **)host_ports, serverlist->num_servers, ",", &uri_path_len); grpc_lb_policy_args args; memset(&args, 0, sizeof(args)); @@ -305,38 +298,56 @@ static grpc_lb_policy *create_rr(grpc_exec_ctx *exec_ctx, args.addresses = gpr_malloc(sizeof(grpc_resolved_addresses)); args.addresses->addrs = gpr_malloc(sizeof(grpc_resolved_address) * serverlist->num_servers); - size_t out_addrs_idx = 0; + size_t addr_idx = 0; for (size_t i = 0; i < serverlist->num_servers; ++i) { - grpc_uri uri; + const grpc_grpclb_server *const server = serverlist->servers[i]; + /* a minimal of error checking */ + if (server->port >> 16 != 0) { + gpr_log(GPR_ERROR, "Invalid port '%d'. Ignoring server list index %zu", + server->port, i); + continue; + } + const uint16_t netorder_port = htons((uint16_t)server->port); + /* the addresses are given in binary format (a in(6)_addr struct) in + * server->ip_address.bytes. */ + const grpc_grpclb_ip_address *ip = &server->ip_address; struct sockaddr_storage sa; - size_t sa_len; - uri.path = host_ports[i]; - if (parse_ipv4(&uri, &sa, &sa_len)) { /* TODO(dgq): add support for ipv6 */ - memcpy(args.addresses->addrs[out_addrs_idx].addr, &sa, sa_len); - args.addresses->addrs[out_addrs_idx].len = sa_len; - ++out_addrs_idx; - const size_t token_max_size = - GPR_ARRAY_SIZE(serverlist->servers[i]->load_balance_token); - serverlist->servers[i]->load_balance_token[token_max_size - 1] = '\0'; - args.tokens[i].token_size = - strlen(serverlist->servers[i]->load_balance_token); - args.tokens[i].token = gpr_malloc(args.tokens[i].token_size); - memcpy(args.tokens[i].token, serverlist->servers[i]->load_balance_token, - args.tokens[i].token_size); + size_t sa_len = 0; + if (ip->size == 4) { + struct sockaddr_in *addr4 = (struct sockaddr_in *)&sa; + memset(addr4, 0, sizeof(struct sockaddr_in)); + sa_len = sizeof(struct sockaddr_in); + addr4->sin_family = AF_INET; + memcpy(&addr4->sin_addr, ip->bytes, ip->size); + addr4->sin_port = netorder_port; + } else if (ip->size == 6) { + struct sockaddr_in *addr6 = (struct sockaddr_in *)&sa; + memset(addr6, 0, sizeof(struct sockaddr_in)); + sa_len = sizeof(struct sockaddr_in); + addr6->sin_family = AF_INET; + memcpy(&addr6->sin_addr, ip->bytes, ip->size); + addr6->sin_port = netorder_port; } else { - gpr_log(GPR_ERROR, "Invalid LB service address '%s', ignoring.", - host_ports[i]); + gpr_log(GPR_ERROR, + "Expected IP to be 4 or 16 bytes. Got %d. Ignoring server list " + "index %zu", + ip->size, i); + continue; } + GPR_ASSERT(sa_len > 0); + memcpy(args.addresses->addrs[addr_idx].addr, &sa, sa_len); + args.addresses->addrs[addr_idx].len = sa_len; + ++addr_idx; + + args.tokens[i].token_size = GPR_ARRAY_SIZE(server->load_balance_token) - 1; + args.tokens[i].token = gpr_malloc(args.tokens[i].token_size); + memcpy(args.tokens[i].token, server->load_balance_token, + args.tokens[i].token_size); } - args.addresses->naddrs = out_addrs_idx; + args.addresses->naddrs = addr_idx; grpc_lb_policy *rr = grpc_lb_policy_create(exec_ctx, "round_robin", &args); - gpr_free(concat_ipports); - for (size_t i = 0; i < serverlist->num_servers; i++) { - gpr_free(host_ports[i]); - } - gpr_free(host_ports); gpr_free(args.addresses->addrs); gpr_free(args.addresses); gpr_free(args.tokens); diff --git a/src/core/ext/lb_policy/grpclb/load_balancer_api.h b/src/core/ext/lb_policy/grpclb/load_balancer_api.h index 9726c87a37f..c1e73d08eff 100644 --- a/src/core/ext/lb_policy/grpclb/load_balancer_api.h +++ b/src/core/ext/lb_policy/grpclb/load_balancer_api.h @@ -45,6 +45,7 @@ extern "C" { #define GRPC_GRPCLB_SERVICE_NAME_MAX_LENGTH 128 +typedef grpc_lb_v1_Server_ip_address_t grpc_grpclb_ip_address; typedef grpc_lb_v1_LoadBalanceRequest grpc_grpclb_request; typedef grpc_lb_v1_InitialLoadBalanceResponse grpc_grpclb_initial_response; typedef grpc_lb_v1_Server grpc_grpclb_server; diff --git a/test/cpp/grpclb/grpclb_test.cc b/test/cpp/grpclb/grpclb_test.cc index 487ef0c26d7..1e906403130 100644 --- a/test/cpp/grpclb/grpclb_test.cc +++ b/test/cpp/grpclb/grpclb_test.cc @@ -37,7 +37,8 @@ #include #include -extern "C" { +#include + #include #include #include @@ -48,6 +49,9 @@ extern "C" { #include #include +#include + +extern "C" { #include "src/core/ext/client_config/client_channel.h" #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/support/string.h" @@ -107,8 +111,8 @@ static gpr_slice build_response_payload_slice( int64_t expiration_interval_secs, int32_t expiration_interval_nanos) { // server_list { // servers { - // ip_address: "127.0.0.1" - // port: ... + // ip_address: + // port: <16 bit uint> // load_balance_token: "token..." // } // ... @@ -127,21 +131,21 @@ static gpr_slice build_response_payload_slice( } for (size_t i = 0; i < nports; i++) { auto *server = serverlist->add_servers(); - server->set_ip_address(host); + // TODO(dgq): test ipv6 + struct in_addr ip4; + GPR_ASSERT(inet_pton(AF_INET, host, &ip4) == 1); + server->set_ip_address( + grpc::string(reinterpret_cast(&ip4), sizeof(ip4))); server->set_port(ports[i]); // The following long long int cast is meant to work around the // disfunctional implementation of std::to_string in gcc 4.4, which doesn't // have a version for int but does have one for long long int. - server->set_load_balance_token("token" + - std::to_string((long long int)ports[i])); + string token_data = "token" + std::to_string((long long int)ports[i]); + token_data.resize(64, '-'); + server->set_load_balance_token(token_data); } - - gpr_log(GPR_INFO, "generating response: %s", - response.ShortDebugString().c_str()); - - const gpr_slice response_slice = - gpr_slice_from_copied_string(response.SerializeAsString().c_str()); - return response_slice; + const grpc::string &enc_resp = response.SerializeAsString(); + return gpr_slice_from_copied_buffer(enc_resp.data(), enc_resp.size()); } static void drain_cq(grpc_completion_queue *cq) { @@ -321,11 +325,15 @@ static void start_backend_server(server_fixture *sf) { return; } GPR_ASSERT(ev.type == GRPC_OP_COMPLETE); - char *expected_token; - GPR_ASSERT(gpr_asprintf(&expected_token, "token%d", sf->port) > 0); + + // The following long long int cast is meant to work around the + // disfunctional implementation of std::to_string in gcc 4.4, which doesn't + // have a version for int but does have one for long long int. + string expected_token = "token" + std::to_string((long long int)sf->port); + expected_token.resize(64, '-'); GPR_ASSERT(contains_metadata(&request_metadata_recv, - "load-reporting-initial", expected_token)); - gpr_free(expected_token); + "load-reporting-initial", + expected_token.c_str())); gpr_log(GPR_INFO, "Server[%s] after tag 100", sf->servers_hostport); From bc33491a8d57c9c8d6b2f4ce87fabd7201311a98 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Mon, 22 Aug 2016 16:57:20 -0700 Subject: [PATCH 11/25] Fixed include of arpa/inet.h --- src/core/ext/lb_policy/grpclb/grpclb.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/core/ext/lb_policy/grpclb/grpclb.c b/src/core/ext/lb_policy/grpclb/grpclb.c index d0316e648d7..6587a934526 100644 --- a/src/core/ext/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/lb_policy/grpclb/grpclb.c @@ -96,7 +96,6 @@ * - Implement LB service forwarding (point 2c. in the doc's diagram). */ -#include #include #include @@ -112,7 +111,7 @@ #include "src/core/ext/client_config/parse_address.h" #include "src/core/ext/lb_policy/grpclb/grpclb.h" #include "src/core/ext/lb_policy/grpclb/load_balancer_api.h" -#include "src/core/lib/iomgr/sockaddr_utils.h" +#include "src/core/lib/iomgr/sockaddr.h" #include "src/core/lib/support/string.h" #include "src/core/lib/surface/call.h" #include "src/core/lib/surface/channel.h" From a50a754ce84b23c2d8587a90719b49cd60a716f4 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 23 Aug 2016 11:29:23 -0700 Subject: [PATCH 12/25] added missing include for sockaddr_utils.h --- src/core/ext/lb_policy/grpclb/grpclb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/ext/lb_policy/grpclb/grpclb.c b/src/core/ext/lb_policy/grpclb/grpclb.c index 6587a934526..778e6a1293c 100644 --- a/src/core/ext/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/lb_policy/grpclb/grpclb.c @@ -112,6 +112,7 @@ #include "src/core/ext/lb_policy/grpclb/grpclb.h" #include "src/core/ext/lb_policy/grpclb/load_balancer_api.h" #include "src/core/lib/iomgr/sockaddr.h" +#include "src/core/lib/iomgr/sockaddr_utils.h" #include "src/core/lib/support/string.h" #include "src/core/lib/surface/call.h" #include "src/core/lib/surface/channel.h" From b8b384a9f685337ed3c6e7accc974df72b1963da Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 23 Aug 2016 21:10:29 -0700 Subject: [PATCH 13/25] Small grpclb.c refactoring for readability --- src/core/ext/lb_policy/grpclb/grpclb.c | 78 ++++++++++++++------------ 1 file changed, 42 insertions(+), 36 deletions(-) diff --git a/src/core/ext/lb_policy/grpclb/grpclb.c b/src/core/ext/lb_policy/grpclb/grpclb.c index 6587a934526..6fecc7cd586 100644 --- a/src/core/ext/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/lb_policy/grpclb/grpclb.c @@ -112,6 +112,7 @@ #include "src/core/ext/lb_policy/grpclb/grpclb.h" #include "src/core/ext/lb_policy/grpclb/load_balancer_api.h" #include "src/core/lib/iomgr/sockaddr.h" +#include "src/core/lib/iomgr/sockaddr_utils.h" #include "src/core/lib/support/string.h" #include "src/core/lib/surface/call.h" #include "src/core/lib/surface/channel.h" @@ -284,6 +285,39 @@ struct rr_connectivity_data { glb_lb_policy *glb_policy; }; +static bool process_serverlist(const grpc_grpclb_server *server, + struct sockaddr_storage *sa, size_t *sa_len) { + if (server->port >> 16 != 0) { + gpr_log(GPR_ERROR, "Invalid port '%d'.", server->port); + return false; + } + const uint16_t netorder_port = htons((uint16_t)server->port); + /* the addresses are given in binary format (a in(6)_addr struct) in + * server->ip_address.bytes. */ + const grpc_grpclb_ip_address *ip = &server->ip_address; + *sa_len = 0; + if (ip->size == 4) { + struct sockaddr_in *addr4 = (struct sockaddr_in *)sa; + *sa_len = sizeof(struct sockaddr_in); + memset(addr4, 0, *sa_len); + addr4->sin_family = AF_INET; + memcpy(&addr4->sin_addr, ip->bytes, ip->size); + addr4->sin_port = netorder_port; + } else if (ip->size == 6) { + struct sockaddr_in *addr6 = (struct sockaddr_in *)sa; + *sa_len = sizeof(struct sockaddr_in); + memset(addr6, 0, *sa_len); + addr6->sin_family = AF_INET; + memcpy(&addr6->sin_addr, ip->bytes, ip->size); + addr6->sin_port = netorder_port; + } else { + gpr_log(GPR_ERROR, "Expected IP to be 4 or 16 bytes. Got %d.", ip->size); + return false; + } + GPR_ASSERT(*sa_len > 0); + return true; +} + static grpc_lb_policy *create_rr(grpc_exec_ctx *exec_ctx, const grpc_grpclb_serverlist *serverlist, glb_lb_policy *glb_policy) { @@ -299,45 +333,17 @@ static grpc_lb_policy *create_rr(grpc_exec_ctx *exec_ctx, gpr_malloc(sizeof(grpc_resolved_address) * serverlist->num_servers); size_t addr_idx = 0; for (size_t i = 0; i < serverlist->num_servers; ++i) { - const grpc_grpclb_server *const server = serverlist->servers[i]; - /* a minimal of error checking */ - if (server->port >> 16 != 0) { - gpr_log(GPR_ERROR, "Invalid port '%d'. Ignoring server list index %zu", - server->port, i); + const grpc_grpclb_server *server = serverlist->servers[i]; + grpc_resolved_address *raddr = &args.addresses->addrs[addr_idx]; + if (!process_serverlist(server, (struct sockaddr_storage *)raddr->addr, + &raddr->len)) { + gpr_log(GPR_INFO, + "Problem processing server at index %zu of received serverlist, " + "ignoring.", + i); continue; } - const uint16_t netorder_port = htons((uint16_t)server->port); - /* the addresses are given in binary format (a in(6)_addr struct) in - * server->ip_address.bytes. */ - const grpc_grpclb_ip_address *ip = &server->ip_address; - struct sockaddr_storage sa; - size_t sa_len = 0; - if (ip->size == 4) { - struct sockaddr_in *addr4 = (struct sockaddr_in *)&sa; - memset(addr4, 0, sizeof(struct sockaddr_in)); - sa_len = sizeof(struct sockaddr_in); - addr4->sin_family = AF_INET; - memcpy(&addr4->sin_addr, ip->bytes, ip->size); - addr4->sin_port = netorder_port; - } else if (ip->size == 6) { - struct sockaddr_in *addr6 = (struct sockaddr_in *)&sa; - memset(addr6, 0, sizeof(struct sockaddr_in)); - sa_len = sizeof(struct sockaddr_in); - addr6->sin_family = AF_INET; - memcpy(&addr6->sin_addr, ip->bytes, ip->size); - addr6->sin_port = netorder_port; - } else { - gpr_log(GPR_ERROR, - "Expected IP to be 4 or 16 bytes. Got %d. Ignoring server list " - "index %zu", - ip->size, i); - continue; - } - GPR_ASSERT(sa_len > 0); - memcpy(args.addresses->addrs[addr_idx].addr, &sa, sa_len); - args.addresses->addrs[addr_idx].len = sa_len; ++addr_idx; - args.tokens[i].token_size = GPR_ARRAY_SIZE(server->load_balance_token) - 1; args.tokens[i].token = gpr_malloc(args.tokens[i].token_size); memcpy(args.tokens[i].token, server->load_balance_token, From 041f9776f70d3f5683eb23d418cbd31358a39c00 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 24 Aug 2016 06:52:25 -0700 Subject: [PATCH 14/25] Fixed wrong processing of ipv6 IPs. --- src/core/ext/lb_policy/grpclb/grpclb.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/core/ext/lb_policy/grpclb/grpclb.c b/src/core/ext/lb_policy/grpclb/grpclb.c index 6fecc7cd586..c3294b7988f 100644 --- a/src/core/ext/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/lb_policy/grpclb/grpclb.c @@ -304,12 +304,12 @@ static bool process_serverlist(const grpc_grpclb_server *server, memcpy(&addr4->sin_addr, ip->bytes, ip->size); addr4->sin_port = netorder_port; } else if (ip->size == 6) { - struct sockaddr_in *addr6 = (struct sockaddr_in *)sa; - *sa_len = sizeof(struct sockaddr_in); + struct sockaddr_in6 *addr6 = (struct sockaddr_in6 *)sa; + *sa_len = sizeof(struct sockaddr_in6); memset(addr6, 0, *sa_len); - addr6->sin_family = AF_INET; - memcpy(&addr6->sin_addr, ip->bytes, ip->size); - addr6->sin_port = netorder_port; + addr6->sin6_family = AF_INET; + memcpy(&addr6->sin6_addr, ip->bytes, ip->size); + addr6->sin6_port = netorder_port; } else { gpr_log(GPR_ERROR, "Expected IP to be 4 or 16 bytes. Got %d.", ip->size); return false; From 880064586d0bf718ae6de139e155d6100d51ea9a Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 24 Aug 2016 06:53:23 -0700 Subject: [PATCH 15/25] Removed direct include of arpa/inet.h --- test/cpp/grpclb/grpclb_test.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/cpp/grpclb/grpclb_test.cc b/test/cpp/grpclb/grpclb_test.cc index 1e906403130..29b3bfecda9 100644 --- a/test/cpp/grpclb/grpclb_test.cc +++ b/test/cpp/grpclb/grpclb_test.cc @@ -37,8 +37,6 @@ #include #include -#include - #include #include #include @@ -54,6 +52,7 @@ extern "C" { #include "src/core/ext/client_config/client_channel.h" #include "src/core/lib/channel/channel_stack.h" +#include "src/core/lib/iomgr/sockaddr.h" #include "src/core/lib/support/string.h" #include "src/core/lib/support/tmpfile.h" #include "src/core/lib/surface/channel.h" From ba641f48d74cc02a12318cb825dc5d6923054a27 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 24 Aug 2016 06:53:43 -0700 Subject: [PATCH 16/25] Updated grpclb API test. --- test/cpp/grpclb/grpclb_api_test.cc | 44 +++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/test/cpp/grpclb/grpclb_api_test.cc b/test/cpp/grpclb/grpclb_api_test.cc index 33de1ee93c6..e67189c69ec 100644 --- a/test/cpp/grpclb/grpclb_api_test.cc +++ b/test/cpp/grpclb/grpclb_api_test.cc @@ -31,10 +31,12 @@ * */ +#include #include -#include #include "src/core/ext/lb_policy/grpclb/load_balancer_api.h" +#include "src/core/lib/iomgr/sockaddr.h" +#include "src/core/lib/iomgr/sockaddr_utils.h" #include "src/proto/grpc/lb/v1/load_balancer.pb.h" // C++ version namespace grpc { @@ -45,8 +47,28 @@ using grpc::lb::v1::LoadBalanceResponse; class GrpclbTest : public ::testing::Test {}; +grpc::string Ip4ToPackedString(const char* ip_str) { + struct in_addr ip4; + GPR_ASSERT(inet_pton(AF_INET, ip_str, &ip4) == 1); + return grpc::string(reinterpret_cast(&ip4), sizeof(ip4)); +} + +grpc::string PackedStringToIp(const grpc_grpclb_ip_address& pb_ip) { + char ip_str[46] = {0}; + int af = -1; + if (pb_ip.size == 4) { + af = AF_INET; + } else if (pb_ip.size == 16) { + af = AF_INET6; + } else { + abort(); + } + GPR_ASSERT(inet_ntop(af, pb_ip.bytes, ip_str, 46) != NULL); + return ip_str; +} + TEST_F(GrpclbTest, CreateRequest) { - const std::string service_name = "AServiceName"; + const grpc::string service_name = "AServiceName"; LoadBalanceRequest request; grpc_grpclb_request* c_req = grpc_grpclb_request_create(service_name.c_str()); gpr_slice slice = grpc_grpclb_request_encode(c_req); @@ -65,7 +87,7 @@ TEST_F(GrpclbTest, ParseInitialResponse) { initial_response->mutable_client_stats_report_interval(); client_stats_report_interval->set_seconds(123); client_stats_report_interval->set_nanos(456); - const std::string encoded_response = response.SerializeAsString(); + const grpc::string encoded_response = response.SerializeAsString(); gpr_slice encoded_slice = gpr_slice_from_copied_string(encoded_response.c_str()); @@ -82,29 +104,31 @@ TEST_F(GrpclbTest, ParseResponseServerList) { LoadBalanceResponse response; auto* serverlist = response.mutable_server_list(); auto* server = serverlist->add_servers(); - server->set_ip_address("127.0.0.1"); + server->set_ip_address(Ip4ToPackedString("127.0.0.1")); server->set_port(12345); server->set_drop_request(true); server = response.mutable_server_list()->add_servers(); - server->set_ip_address("10.0.0.1"); + server->set_ip_address(Ip4ToPackedString("10.0.0.1")); server->set_port(54321); server->set_drop_request(false); auto* expiration_interval = serverlist->mutable_expiration_interval(); expiration_interval->set_seconds(888); expiration_interval->set_nanos(999); - const std::string encoded_response = response.SerializeAsString(); - gpr_slice encoded_slice = - gpr_slice_from_copied_string(encoded_response.c_str()); + const grpc::string encoded_response = response.SerializeAsString(); + const gpr_slice encoded_slice = gpr_slice_from_copied_buffer( + encoded_response.data(), encoded_response.size()); grpc_grpclb_serverlist* c_serverlist = grpc_grpclb_response_parse_serverlist(encoded_slice); ASSERT_EQ(c_serverlist->num_servers, 2ul); EXPECT_TRUE(c_serverlist->servers[0]->has_ip_address); - EXPECT_TRUE(strcmp(c_serverlist->servers[0]->ip_address, "127.0.0.1") == 0); + EXPECT_EQ(PackedStringToIp(c_serverlist->servers[0]->ip_address), + "127.0.0.1"); EXPECT_EQ(c_serverlist->servers[0]->port, 12345); EXPECT_TRUE(c_serverlist->servers[0]->drop_request); EXPECT_TRUE(c_serverlist->servers[1]->has_ip_address); - EXPECT_TRUE(strcmp(c_serverlist->servers[1]->ip_address, "10.0.0.1") == 0); + + EXPECT_EQ(PackedStringToIp(c_serverlist->servers[1]->ip_address), "10.0.0.1"); EXPECT_EQ(c_serverlist->servers[1]->port, 54321); EXPECT_FALSE(c_serverlist->servers[1]->drop_request); From 9946f9064fd958a92aec4a71044fd1e7f72c617c Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Mon, 29 Aug 2016 17:05:17 -0700 Subject: [PATCH 17/25] fixed generated load_balancer.pb.h formatting --- .../ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h index 832c851ca3b..4f1031ec7bc 100644 --- a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h +++ b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h @@ -172,7 +172,7 @@ extern const pb_field_t grpc_lb_v1_Server_fields[5]; #define grpc_lb_v1_LoadBalanceResponse_size (98 + grpc_lb_v1_ServerList_size) #define grpc_lb_v1_InitialLoadBalanceResponse_size 90 /* grpc_lb_v1_ServerList_size depends on runtime parameters */ -#define grpc_lb_v1_Server_size 98 +#define grpc_lb_v1_Server_size 98 /* Message IDs (where set with "msgid" option) */ #ifdef PB_MSGID From 42adfb89ce3f842bf7a3d24e9f04b32d46644457 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Fri, 2 Sep 2016 18:43:24 +0200 Subject: [PATCH 18/25] typo --- src/core/ext/client_config/lb_policy_factory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/ext/client_config/lb_policy_factory.h b/src/core/ext/client_config/lb_policy_factory.h index cfe6f1afa3a..2125eaba702 100644 --- a/src/core/ext/client_config/lb_policy_factory.h +++ b/src/core/ext/client_config/lb_policy_factory.h @@ -54,7 +54,7 @@ typedef struct grpc_lb_policy_address_token { typedef struct grpc_lb_policy_args { grpc_resolved_addresses *addresses; - /* It not NULL, array of load balancing tokens associated with \a addresses, + /* If not NULL, array of load balancing tokens associated with \a addresses, * on a 1:1 correspondence. Some indices may be NULL for missing tokens. */ grpc_lb_policy_address_token *tokens; grpc_client_channel_factory *client_channel_factory; From 331b9c02f9372ef832e612d48b90991c181ba8a7 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Mon, 12 Sep 2016 18:37:05 -0700 Subject: [PATCH 19/25] Moved LB token changes solely into grpclb.c --- src/core/ext/client_config/client_channel.c | 2 +- src/core/ext/client_config/lb_policy.c | 5 +- src/core/ext/client_config/lb_policy.h | 20 +- .../ext/client_config/lb_policy_factory.h | 25 +- src/core/ext/lb_policy/grpclb/grpclb.c | 240 ++++++++++++------ .../ext/lb_policy/pick_first/pick_first.c | 18 +- .../ext/lb_policy/round_robin/round_robin.c | 89 +++---- .../ext/resolver/dns/native/dns_resolver.c | 9 +- .../ext/resolver/sockaddr/sockaddr_resolver.c | 10 +- src/core/lib/transport/metadata.c | 18 +- src/core/lib/transport/metadata.h | 2 +- 11 files changed, 269 insertions(+), 169 deletions(-) diff --git a/src/core/ext/client_config/client_channel.c b/src/core/ext/client_config/client_channel.c index 8f1c821ebb0..58ef629be06 100644 --- a/src/core/ext/client_config/client_channel.c +++ b/src/core/ext/client_config/client_channel.c @@ -572,7 +572,7 @@ static bool pick_subchannel(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, initial_metadata_flags, &calld->lb_token_mdelem}; r = grpc_lb_policy_pick(exec_ctx, lb_policy, &inputs, connected_subchannel, - on_ready); + NULL, on_ready); GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "pick_subchannel"); GPR_TIMER_END("pick_subchannel", 0); return r; diff --git a/src/core/ext/client_config/lb_policy.c b/src/core/ext/client_config/lb_policy.c index 71170f5655b..903563ef6b5 100644 --- a/src/core/ext/client_config/lb_policy.c +++ b/src/core/ext/client_config/lb_policy.c @@ -101,9 +101,10 @@ void grpc_lb_policy_weak_unref(grpc_exec_ctx *exec_ctx, int grpc_lb_policy_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, const grpc_lb_policy_pick_args *pick_args, - grpc_connected_subchannel **target, + grpc_connected_subchannel **target, void **user_data, grpc_closure *on_complete) { - return policy->vtable->pick(exec_ctx, policy, pick_args, target, on_complete); + return policy->vtable->pick(exec_ctx, policy, pick_args, target, user_data, + on_complete); } void grpc_lb_policy_cancel_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, diff --git a/src/core/ext/client_config/lb_policy.h b/src/core/ext/client_config/lb_policy.h index 6f133a29488..734b3bff970 100644 --- a/src/core/ext/client_config/lb_policy.h +++ b/src/core/ext/client_config/lb_policy.h @@ -72,7 +72,8 @@ struct grpc_lb_policy_vtable { /** \see grpc_lb_policy_pick */ int (*pick)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, const grpc_lb_policy_pick_args *pick_args, - grpc_connected_subchannel **target, grpc_closure *on_complete); + grpc_connected_subchannel **target, void **user_data, + grpc_closure *on_complete); /** \see grpc_lb_policy_cancel_pick */ void (*cancel_pick)(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, @@ -138,14 +139,19 @@ void grpc_lb_policy_init(grpc_lb_policy *policy, const grpc_lb_policy_vtable *vtable); /** Find an appropriate target for this call, based on \a pick_args. - Upon completion \a on_complete will be called, with \a *target set to an - appropriate connected subchannel if the pick was successful or NULL - otherwise. - Picking can be asynchronous. Any IO should be done under \a - pick_args->pollent. */ + Picking can be synchronous or asynchronous. In the synchronous case, when a + pick is readily available, it'll be returned in \a target and a non-zero + value will be returned. + In the asynchronous case, zero is returned and \a on_complete will be called + once \a target and \a user_data have been set. Any IO should be done under + \a + pick_args->pollent. + The opaque \a user_data output argument corresponds to information that may + need be propagated from the LB policy. It may be NULL. + Errors are signaled by receiving a NULL \a *target. */ int grpc_lb_policy_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, const grpc_lb_policy_pick_args *pick_args, - grpc_connected_subchannel **target, + grpc_connected_subchannel **target, void **user_data, grpc_closure *on_complete); /** Perform a connected subchannel ping (see \a grpc_connected_subchannel_ping) diff --git a/src/core/ext/client_config/lb_policy_factory.h b/src/core/ext/client_config/lb_policy_factory.h index 2125eaba702..e1d67633b42 100644 --- a/src/core/ext/client_config/lb_policy_factory.h +++ b/src/core/ext/client_config/lb_policy_factory.h @@ -47,16 +47,25 @@ struct grpc_lb_policy_factory { const grpc_lb_policy_factory_vtable *vtable; }; -typedef struct grpc_lb_policy_address_token { - uint8_t *token; - size_t token_size; -} grpc_lb_policy_address_token; +/** A resolved address alongside any LB related information associated with it. + * \a user_data, if not \a NULL, is opaque and meant to be consumed by the gRPC + * LB policy. Anywhere else, refer to the functions in \a + * grpc_lb_policy_user_data_vtable to operate with it */ +typedef struct grpc_lb_address { + grpc_resolved_address *resolved_address; + void *user_data; +} grpc_lb_address; + +/** Functions acting upon the opaque \a grpc_lb_address.user_data */ +typedef struct grpc_lb_policy_user_data_vtable { + void *(*copy)(void *); + void (*destroy)(void *); +} grpc_lb_policy_user_data_vtable; typedef struct grpc_lb_policy_args { - grpc_resolved_addresses *addresses; - /* If not NULL, array of load balancing tokens associated with \a addresses, - * on a 1:1 correspondence. Some indices may be NULL for missing tokens. */ - grpc_lb_policy_address_token *tokens; + grpc_lb_address *lb_addresses; + size_t num_addresses; + grpc_lb_policy_user_data_vtable user_data_vtable; grpc_client_channel_factory *client_channel_factory; } grpc_lb_policy_args; diff --git a/src/core/ext/lb_policy/grpclb/grpclb.c b/src/core/ext/lb_policy/grpclb/grpclb.c index c3294b7988f..ad070e458a7 100644 --- a/src/core/ext/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/lb_policy/grpclb/grpclb.c @@ -116,14 +116,50 @@ #include "src/core/lib/support/string.h" #include "src/core/lib/surface/call.h" #include "src/core/lib/surface/channel.h" +#include "src/core/lib/transport/static_metadata.h" int grpc_lb_glb_trace = 0; +static void *user_data_copy(void *user_data) { + if (user_data == NULL) return NULL; + return GRPC_MDELEM_REF(user_data); +} + +static void user_data_destroy(void *user_data) { + if (user_data == NULL) return; + GRPC_MDELEM_UNREF(user_data); +} + +/* add lb_token of selected subchannel (address) to the call's initial + * metadata */ +static void initial_metadata_add_lb_token( + grpc_metadata_batch *initial_metadata, + grpc_linked_mdelem *lb_token_mdelem_storage, grpc_mdelem *lb_token) { + GPR_ASSERT(lb_token_mdelem_storage != NULL); + GPR_ASSERT(lb_token != NULL); + grpc_metadata_batch_add_tail(initial_metadata, lb_token_mdelem_storage, + lb_token); +} + typedef struct wrapped_rr_closure_arg { /* the original closure. Usually a on_complete/notify cb for pick() and ping() * calls against the internal RR instance, respectively. */ grpc_closure *wrapped_closure; + /* the pick's initial metadata, kept in order to append the LB token for the + * pick */ + grpc_metadata_batch *initial_metadata; + + /* the picked target, used to determine which LB token to add to the pick's + * initial metadata */ + grpc_connected_subchannel **target; + + /* the LB token associated with the pick */ + grpc_mdelem *lb_token; + + /* storage for the lb token initial metadata mdelem */ + grpc_linked_mdelem *lb_token_mdelem_storage; + /* The RR instance related to the closure */ grpc_lb_policy *rr_policy; @@ -146,6 +182,11 @@ static void wrapped_rr_closure(grpc_exec_ctx *exec_ctx, void *arg, GRPC_LB_POLICY_UNREF(exec_ctx, wc_arg->rr_policy, "wrapped_rr_closure"); } GPR_ASSERT(wc_arg->wrapped_closure != NULL); + + initial_metadata_add_lb_token(wc_arg->initial_metadata, + wc_arg->lb_token_mdelem_storage, + wc_arg->lb_token); + grpc_exec_ctx_sched(exec_ctx, wc_arg->wrapped_closure, error, NULL); gpr_free(wc_arg->owning_pending_node); } @@ -194,12 +235,15 @@ static void add_pending_pick(pending_pick **root, memset(pp, 0, sizeof(pending_pick)); memset(&pp->wrapped_on_complete_arg, 0, sizeof(wrapped_rr_closure_arg)); pp->next = *root; - pp->lb_token_mdelem_storage = pick_args->lb_token_mdelem_storage; pp->pollent = pick_args->pollent; pp->target = target; pp->initial_metadata = pick_args->initial_metadata; pp->initial_metadata_flags = pick_args->initial_metadata_flags; + pp->lb_token_mdelem_storage = pick_args->lb_token_mdelem_storage; pp->wrapped_on_complete_arg.wrapped_closure = on_complete; + pp->wrapped_on_complete_arg.initial_metadata = pick_args->initial_metadata; + pp->wrapped_on_complete_arg.lb_token_mdelem_storage = + pick_args->lb_token_mdelem_storage; grpc_closure_init(&pp->wrapped_on_complete, wrapped_rr_closure, &pp->wrapped_on_complete_arg); *root = pp; @@ -285,37 +329,90 @@ struct rr_connectivity_data { glb_lb_policy *glb_policy; }; -static bool process_serverlist(const grpc_grpclb_server *server, - struct sockaddr_storage *sa, size_t *sa_len) { - if (server->port >> 16 != 0) { - gpr_log(GPR_ERROR, "Invalid port '%d'.", server->port); - return false; +/* populate \a addresses according to \a serverlist. Returns the number of + * addresses successfully parsed and added to \a addresses */ +static size_t process_serverlist(const grpc_grpclb_serverlist *serverlist, + grpc_lb_address **lb_addresses) { + size_t num_valid = 0; + /* first pass: count how many are valid in order to allocate the necessary + * memory in a single block */ + for (size_t i = 0; i < serverlist->num_servers; ++i) { + const grpc_grpclb_server *server = serverlist->servers[i]; + const grpc_grpclb_ip_address *ip = &server->ip_address; + + if (server->port >> 16 != 0) { + gpr_log(GPR_ERROR, + "Invalid port '%d' at index %zu of serverlist. Ignoring.", + server->port, i); + continue; + } + + if (ip->size != 4 && ip->size != 16) { + gpr_log(GPR_ERROR, + "Expected IP to be 4 or 16 bytes, got %d at index %zu of " + "serverlist. Ignoring", + ip->size, i); + continue; + } + ++num_valid; } - const uint16_t netorder_port = htons((uint16_t)server->port); - /* the addresses are given in binary format (a in(6)_addr struct) in - * server->ip_address.bytes. */ - const grpc_grpclb_ip_address *ip = &server->ip_address; - *sa_len = 0; - if (ip->size == 4) { - struct sockaddr_in *addr4 = (struct sockaddr_in *)sa; - *sa_len = sizeof(struct sockaddr_in); - memset(addr4, 0, *sa_len); - addr4->sin_family = AF_INET; - memcpy(&addr4->sin_addr, ip->bytes, ip->size); - addr4->sin_port = netorder_port; - } else if (ip->size == 6) { - struct sockaddr_in6 *addr6 = (struct sockaddr_in6 *)sa; - *sa_len = sizeof(struct sockaddr_in6); - memset(addr6, 0, *sa_len); - addr6->sin6_family = AF_INET; - memcpy(&addr6->sin6_addr, ip->bytes, ip->size); - addr6->sin6_port = netorder_port; - } else { - gpr_log(GPR_ERROR, "Expected IP to be 4 or 16 bytes. Got %d.", ip->size); - return false; + if (num_valid == 0) { + return 0; } - GPR_ASSERT(*sa_len > 0); - return true; + + /* allocate the memory block for the "resolved" addresses. */ + grpc_resolved_address *r_addrs_memblock = + gpr_malloc(sizeof(grpc_resolved_address) * num_valid); + memset(r_addrs_memblock, 0, sizeof(grpc_resolved_address) * num_valid); + grpc_lb_address *lb_addrs = gpr_malloc(sizeof(grpc_lb_address) * num_valid); + memset(lb_addrs, 0, sizeof(grpc_lb_address) * num_valid); + + /* second pass: actually populate the addresses and LB tokens (aka user data + * to the outside world) to be read by the RR policy during its creation */ + for (size_t i = 0; i < num_valid; ++i) { + const grpc_grpclb_server *server = serverlist->servers[i]; + grpc_lb_address *const lb_addr = &lb_addrs[i]; + + /* lb token processing */ + if (server->has_load_balance_token) { + const size_t lb_token_size = + GPR_ARRAY_SIZE(server->load_balance_token) - 1; + grpc_mdstr *lb_token_mdstr = grpc_mdstr_from_buffer( + (uint8_t *)server->load_balance_token, lb_token_size); + lb_addr->user_data = grpc_mdelem_from_metadata_strings( + GRPC_MDSTR_LOAD_REPORTING_INITIAL, lb_token_mdstr); + } + + /* address processing */ + const uint16_t netorder_port = htons((uint16_t)server->port); + /* the addresses are given in binary format (a in(6)_addr struct) in + * server->ip_address.bytes. */ + const grpc_grpclb_ip_address *ip = &server->ip_address; + + lb_addr->resolved_address = &r_addrs_memblock[i]; + struct sockaddr_storage *sa = + (struct sockaddr_storage *)lb_addr->resolved_address->addr; + size_t *sa_len = &lb_addr->resolved_address->len; + *sa_len = 0; + if (ip->size == 4) { + struct sockaddr_in *addr4 = (struct sockaddr_in *)sa; + *sa_len = sizeof(struct sockaddr_in); + memset(addr4, 0, *sa_len); + addr4->sin_family = AF_INET; + memcpy(&addr4->sin_addr, ip->bytes, ip->size); + addr4->sin_port = netorder_port; + } else if (ip->size == 16) { + struct sockaddr_in6 *addr6 = (struct sockaddr_in6 *)sa; + *sa_len = sizeof(struct sockaddr_in6); + memset(addr6, 0, *sa_len); + addr6->sin6_family = AF_INET; + memcpy(&addr6->sin6_addr, ip->bytes, ip->size); + addr6->sin6_port = netorder_port; + } + GPR_ASSERT(*sa_len > 0); + } + *lb_addresses = lb_addrs; + return num_valid; } static grpc_lb_policy *create_rr(grpc_exec_ctx *exec_ctx, @@ -326,36 +423,19 @@ static grpc_lb_policy *create_rr(grpc_exec_ctx *exec_ctx, grpc_lb_policy_args args; memset(&args, 0, sizeof(args)); args.client_channel_factory = glb_policy->cc_factory; - args.tokens = gpr_malloc(sizeof(grpc_lb_policy_address_token) * - serverlist->num_servers); - args.addresses = gpr_malloc(sizeof(grpc_resolved_addresses)); - args.addresses->addrs = - gpr_malloc(sizeof(grpc_resolved_address) * serverlist->num_servers); - size_t addr_idx = 0; - for (size_t i = 0; i < serverlist->num_servers; ++i) { - const grpc_grpclb_server *server = serverlist->servers[i]; - grpc_resolved_address *raddr = &args.addresses->addrs[addr_idx]; - if (!process_serverlist(server, (struct sockaddr_storage *)raddr->addr, - &raddr->len)) { - gpr_log(GPR_INFO, - "Problem processing server at index %zu of received serverlist, " - "ignoring.", - i); - continue; - } - ++addr_idx; - args.tokens[i].token_size = GPR_ARRAY_SIZE(server->load_balance_token) - 1; - args.tokens[i].token = gpr_malloc(args.tokens[i].token_size); - memcpy(args.tokens[i].token, server->load_balance_token, - args.tokens[i].token_size); - } - args.addresses->naddrs = addr_idx; + args.num_addresses = process_serverlist(serverlist, &args.lb_addresses); + args.user_data_vtable.copy = user_data_copy; + args.user_data_vtable.destroy = user_data_destroy; grpc_lb_policy *rr = grpc_lb_policy_create(exec_ctx, "round_robin", &args); - - gpr_free(args.addresses->addrs); - gpr_free(args.addresses); - gpr_free(args.tokens); + if (args.num_addresses > 0) { + /* free "resolved" addresses memblock */ + gpr_free(args.lb_addresses->resolved_address); + } + for (size_t i = 0; i < args.num_addresses; ++i) { + args.user_data_vtable.destroy(args.lb_addresses[i].user_data); + } + gpr_free(args.lb_addresses); return rr; } @@ -395,6 +475,7 @@ static void rr_handover(grpc_exec_ctx *exec_ctx, glb_lb_policy *glb_policy, pp->pollent, pp->initial_metadata, pp->initial_metadata_flags, pp->lb_token_mdelem_storage}; grpc_lb_policy_pick(exec_ctx, glb_policy->rr_policy, &pick_args, pp->target, + (void **)&pp->wrapped_on_complete_arg.lb_token, &pp->wrapped_on_complete); pp->wrapped_on_complete_arg.owning_pending_node = pp; } @@ -457,25 +538,26 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, * Create a client channel over them to communicate with a LB service */ glb_policy->cc_factory = args->client_channel_factory; GPR_ASSERT(glb_policy->cc_factory != NULL); - if (args->addresses->naddrs == 0) { + if (args->num_addresses == 0) { return NULL; } - /* construct a target from the args->addresses, in the form + /* construct a target from the addresses in args, given in the form * ipvX://ip1:port1,ip2:port2,... * TODO(dgq): support mixed ip version */ - char **addr_strs = gpr_malloc(sizeof(char *) * args->addresses->naddrs); - addr_strs[0] = - grpc_sockaddr_to_uri((const struct sockaddr *)&args->addresses->addrs[0]); - for (size_t i = 1; i < args->addresses->naddrs; i++) { - GPR_ASSERT(grpc_sockaddr_to_string( - &addr_strs[i], - (const struct sockaddr *)&args->addresses->addrs[i], - true) == 0); + char **addr_strs = gpr_malloc(sizeof(char *) * args->num_addresses); + addr_strs[0] = grpc_sockaddr_to_uri( + (const struct sockaddr *)&args->lb_addresses[0].resolved_address->addr); + for (size_t i = 1; i < args->num_addresses; i++) { + GPR_ASSERT( + grpc_sockaddr_to_string(&addr_strs[i], + (const struct sockaddr *)&args->lb_addresses[i] + .resolved_address->addr, + true) == 0); } size_t uri_path_len; char *target_uri_str = gpr_strjoin_sep( - (const char **)addr_strs, args->addresses->naddrs, ",", &uri_path_len); + (const char **)addr_strs, args->num_addresses, ",", &uri_path_len); /* will pick using pick_first */ glb_policy->lb_channel = grpc_client_channel_factory_create_channel( @@ -483,7 +565,7 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, GRPC_CLIENT_CHANNEL_TYPE_LOAD_BALANCING, NULL); gpr_free(target_uri_str); - for (size_t i = 0; i < args->addresses->naddrs; i++) { + for (size_t i = 0; i < args->num_addresses; i++) { gpr_free(addr_strs[i]); } gpr_free(addr_strs); @@ -635,7 +717,7 @@ static void glb_exit_idle(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { static int glb_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, const grpc_lb_policy_pick_args *pick_args, - grpc_connected_subchannel **target, + grpc_connected_subchannel **target, void **user_data, grpc_closure *on_complete) { glb_lb_policy *glb_policy = (glb_lb_policy *)pol; @@ -662,22 +744,28 @@ static int glb_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, memset(&glb_policy->wc_arg, 0, sizeof(wrapped_rr_closure_arg)); glb_policy->wc_arg.rr_policy = glb_policy->rr_policy; glb_policy->wc_arg.wrapped_closure = on_complete; + glb_policy->wc_arg.lb_token_mdelem_storage = + pick_args->lb_token_mdelem_storage; + glb_policy->wc_arg.initial_metadata = pick_args->initial_metadata; + glb_policy->wc_arg.owning_pending_node = NULL; grpc_closure_init(&glb_policy->wrapped_on_complete, wrapped_rr_closure, &glb_policy->wc_arg); r = grpc_lb_policy_pick(exec_ctx, glb_policy->rr_policy, pick_args, target, + (void **)&glb_policy->wc_arg.lb_token, &glb_policy->wrapped_on_complete); if (r != 0) { - /* the call to grpc_lb_policy_pick has been sychronous. Unreffing the RR - * policy and notify the original callback */ - glb_policy->wc_arg.wrapped_closure = NULL; + /* synchronous grpc_lb_policy_pick call. Unref the RR policy. */ if (grpc_lb_glb_trace) { gpr_log(GPR_INFO, "Unreffing RR (0x%" PRIxPTR ")", (intptr_t)glb_policy->wc_arg.rr_policy); } GRPC_LB_POLICY_UNREF(exec_ctx, glb_policy->wc_arg.rr_policy, "glb_pick"); - grpc_exec_ctx_sched(exec_ctx, glb_policy->wc_arg.wrapped_closure, - GRPC_ERROR_NONE, NULL); + + /* add the load reporting initial metadata */ + initial_metadata_add_lb_token(pick_args->initial_metadata, + pick_args->lb_token_mdelem_storage, + glb_policy->wc_arg.lb_token); } } else { grpc_polling_entity_add_to_pollset_set(exec_ctx, pick_args->pollent, diff --git a/src/core/ext/lb_policy/pick_first/pick_first.c b/src/core/ext/lb_policy/pick_first/pick_first.c index e1277b353fe..21d948033a0 100644 --- a/src/core/ext/lb_policy/pick_first/pick_first.c +++ b/src/core/ext/lb_policy/pick_first/pick_first.c @@ -200,7 +200,7 @@ static void pf_exit_idle(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { static int pf_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, const grpc_lb_policy_pick_args *pick_args, - grpc_connected_subchannel **target, + grpc_connected_subchannel **target, void **user_data, grpc_closure *on_complete) { pick_first_lb_policy *p = (pick_first_lb_policy *)pol; pending_pick *pp; @@ -438,23 +438,23 @@ static void pick_first_factory_unref(grpc_lb_policy_factory *factory) {} static grpc_lb_policy *create_pick_first(grpc_exec_ctx *exec_ctx, grpc_lb_policy_factory *factory, grpc_lb_policy_args *args) { - GPR_ASSERT(args->addresses != NULL); + GPR_ASSERT(args->lb_addresses != NULL); GPR_ASSERT(args->client_channel_factory != NULL); - if (args->addresses->naddrs == 0) return NULL; + if (args->num_addresses == 0) return NULL; pick_first_lb_policy *p = gpr_malloc(sizeof(*p)); memset(p, 0, sizeof(*p)); - p->subchannels = - gpr_malloc(sizeof(grpc_subchannel *) * args->addresses->naddrs); - memset(p->subchannels, 0, sizeof(*p->subchannels) * args->addresses->naddrs); + p->subchannels = gpr_malloc(sizeof(grpc_subchannel *) * args->num_addresses); + memset(p->subchannels, 0, sizeof(*p->subchannels) * args->num_addresses); grpc_subchannel_args sc_args; size_t subchannel_idx = 0; - for (size_t i = 0; i < args->addresses->naddrs; i++) { + for (size_t i = 0; i < args->num_addresses; i++) { memset(&sc_args, 0, sizeof(grpc_subchannel_args)); - sc_args.addr = (struct sockaddr *)(args->addresses->addrs[i].addr); - sc_args.addr_len = (size_t)args->addresses->addrs[i].len; + sc_args.addr = + (struct sockaddr *)(args->lb_addresses[i].resolved_address->addr); + sc_args.addr_len = (size_t)args->lb_addresses[i].resolved_address->len; grpc_subchannel *subchannel = grpc_client_channel_factory_create_subchannel( exec_ctx, args->client_channel_factory, &sc_args); diff --git a/src/core/ext/lb_policy/round_robin/round_robin.c b/src/core/ext/lb_policy/round_robin/round_robin.c index 8fda405fb87..2069dc192c0 100644 --- a/src/core/ext/lb_policy/round_robin/round_robin.c +++ b/src/core/ext/lb_policy/round_robin/round_robin.c @@ -84,8 +84,10 @@ typedef struct pending_pick { /* the initial metadata for the pick. See grpc_lb_policy_pick() */ grpc_metadata_batch *initial_metadata; - /* storage for the lb token initial metadata mdelem */ - grpc_linked_mdelem *lb_token_mdelem_storage; + /* output argument where to store the pick()ed user_data. It'll be NULL if no + * such data is present or there's an error (the definite test for errors is + * \a target being NULL). */ + void **user_data; /* bitmask passed to pick() and used for selective cancelling. See * grpc_lb_policy_cancel_picks() */ @@ -103,7 +105,7 @@ typedef struct pending_pick { typedef struct ready_list { grpc_subchannel *subchannel; /* references namesake entry in subchannel_data */ - grpc_lb_policy_address_token *lb_token; + void *user_data; struct ready_list *next; struct ready_list *prev; } ready_list; @@ -121,8 +123,8 @@ typedef struct { ready_list *ready_list_node; /** last observed connectivity */ grpc_connectivity_state connectivity_state; - /** the subchannel's target LB token */ - grpc_lb_policy_address_token *lb_token; + /** the subchannel's target user data */ + void *user_data; } subchannel_data; struct round_robin_lb_policy { @@ -131,8 +133,10 @@ struct round_robin_lb_policy { /** total number of addresses received at creation time */ size_t num_addresses; - /** load balancing tokens, one per incoming address */ - grpc_lb_policy_address_token *lb_tokens; + /** user data, one per incoming address */ + void **user_data; + /** functions to operate over \a user_data elements */ + grpc_lb_policy_user_data_vtable user_data_vtable; /** all our subchannels */ size_t num_subchannels; @@ -204,7 +208,7 @@ static ready_list *add_connected_sc_locked(round_robin_lb_policy *p, ready_list *new_elem = gpr_malloc(sizeof(ready_list)); memset(new_elem, 0, sizeof(ready_list)); new_elem->subchannel = sd->subchannel; - new_elem->lb_token = sd->lb_token; + new_elem->user_data = sd->user_data; if (p->ready_list.prev == NULL) { /* first element */ new_elem->next = &p->ready_list; @@ -246,7 +250,7 @@ static void remove_disconnected_sc_locked(round_robin_lb_policy *p, } if (grpc_lb_round_robin_trace) { - gpr_log(GPR_DEBUG, "[READYLIST] REMOVED NODE %p (SC %p)", node, + gpr_log(GPR_DEBUG, "[READYLIST] REMOVED NODE %p (SC %p)", (void *)node, (void *)node->subchannel); } @@ -259,9 +263,8 @@ static void remove_disconnected_sc_locked(round_robin_lb_policy *p, static void rr_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { round_robin_lb_policy *p = (round_robin_lb_policy *)pol; - size_t i; ready_list *elem; - for (i = 0; i < p->num_subchannels; i++) { + for (size_t i = 0; i < p->num_subchannels; i++) { subchannel_data *sd = p->subchannels[i]; GRPC_SUBCHANNEL_UNREF(exec_ctx, sd->subchannel, "round_robin"); gpr_free(sd); @@ -282,12 +285,10 @@ static void rr_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { elem = tmp; } - if (p->lb_tokens != NULL) { - for (i = 0; i < p->num_addresses; i++) { - gpr_free(p->lb_tokens[i].token); - } - gpr_free(p->lb_tokens); + for (size_t i = 0; i < p->num_addresses; i++) { + p->user_data_vtable.destroy(p->user_data[i]); } + gpr_free(p->user_data); gpr_free(p); } @@ -397,26 +398,9 @@ static void rr_exit_idle(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { gpr_mu_unlock(&p->mu); } -/* add lb_token of selected subchannel (address) to the call's initial - * metadata */ -static void initial_metadata_add_lb_token( - grpc_metadata_batch *initial_metadata, - grpc_linked_mdelem *lb_token_mdelem_storage, - grpc_lb_policy_address_token *lb_token) { - if (lb_token != NULL && lb_token->token_size > 0) { - GPR_ASSERT(lb_token->token != NULL); - grpc_mdstr *lb_token_mdstr = - grpc_mdstr_from_buffer(lb_token->token, lb_token->token_size); - grpc_metadata_batch_add_tail( - initial_metadata, lb_token_mdelem_storage, - grpc_mdelem_from_metadata_strings(GRPC_MDSTR_LOAD_REPORTING_INITIAL, - lb_token_mdstr)); - } -} - static int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, const grpc_lb_policy_pick_args *pick_args, - grpc_connected_subchannel **target, + grpc_connected_subchannel **target, void **user_data, grpc_closure *on_complete) { round_robin_lb_policy *p = (round_robin_lb_policy *)pol; pending_pick *pp; @@ -426,9 +410,7 @@ static int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, /* readily available, report right away */ gpr_mu_unlock(&p->mu); *target = grpc_subchannel_get_connected_subchannel(selected->subchannel); - initial_metadata_add_lb_token(pick_args->initial_metadata, - pick_args->lb_token_mdelem_storage, - selected->lb_token); + *user_data = p->user_data_vtable.copy(selected->user_data); if (grpc_lb_round_robin_trace) { gpr_log(GPR_DEBUG, "[RR PICK] TARGET <-- CONNECTED SUBCHANNEL %p (NODE %p)", @@ -451,7 +433,7 @@ static int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, pp->on_complete = on_complete; pp->initial_metadata = pick_args->initial_metadata; pp->initial_metadata_flags = pick_args->initial_metadata_flags; - pp->lb_token_mdelem_storage = pick_args->lb_token_mdelem_storage; + pp->user_data = user_data; p->pending_picks = pp; gpr_mu_unlock(&p->mu); return 0; @@ -493,11 +475,9 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, while ((pp = p->pending_picks)) { p->pending_picks = pp->next; - initial_metadata_add_lb_token(pp->initial_metadata, - pp->lb_token_mdelem_storage, - selected->lb_token); *pp->target = grpc_subchannel_get_connected_subchannel(selected->subchannel); + *pp->user_data = p->user_data_vtable.copy(selected->user_data); if (grpc_lb_round_robin_trace) { gpr_log(GPR_DEBUG, "[RR CONN CHANGED] TARGET <-- SUBCHANNEL %p (NODE %p)", @@ -631,30 +611,29 @@ static void round_robin_factory_unref(grpc_lb_policy_factory *factory) {} static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx, grpc_lb_policy_factory *factory, grpc_lb_policy_args *args) { - GPR_ASSERT(args->addresses != NULL); + GPR_ASSERT(args->lb_addresses != NULL); GPR_ASSERT(args->client_channel_factory != NULL); + if (args->num_addresses == 0) return NULL; round_robin_lb_policy *p = gpr_malloc(sizeof(*p)); memset(p, 0, sizeof(*p)); - p->num_addresses = args->addresses->naddrs; - if (args->tokens != NULL) { - /* we need to copy because args contents aren't owned */ - p->lb_tokens = - gpr_malloc(sizeof(grpc_lb_policy_address_token) * p->num_addresses); - memcpy(p->lb_tokens, args->tokens, - sizeof(grpc_lb_policy_address_token) * p->num_addresses); - } - + p->num_addresses = args->num_addresses; p->subchannels = gpr_malloc(sizeof(subchannel_data) * p->num_addresses); memset(p->subchannels, 0, sizeof(*p->subchannels) * p->num_addresses); + p->user_data = gpr_malloc(sizeof(void *) * p->num_addresses); + memset(p->user_data, 0, sizeof(void *) * p->num_addresses); + p->user_data_vtable = args->user_data_vtable; grpc_subchannel_args sc_args; size_t subchannel_idx = 0; for (size_t i = 0; i < p->num_addresses; i++) { memset(&sc_args, 0, sizeof(grpc_subchannel_args)); - sc_args.addr = (struct sockaddr *)(args->addresses->addrs[i].addr); - sc_args.addr_len = (size_t)args->addresses->addrs[i].len; + sc_args.addr = + (struct sockaddr *)args->lb_addresses[i].resolved_address->addr; + sc_args.addr_len = args->lb_addresses[i].resolved_address->len; + + p->user_data[i] = p->user_data_vtable.copy(args->lb_addresses[i].user_data); grpc_subchannel *subchannel = grpc_client_channel_factory_create_subchannel( exec_ctx, args->client_channel_factory, &sc_args); @@ -666,9 +645,7 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx, sd->policy = p; sd->index = subchannel_idx; sd->subchannel = subchannel; - if (p->lb_tokens != NULL) { - sd->lb_token = &p->lb_tokens[i]; - } + sd->user_data = p->user_data[i]; ++subchannel_idx; grpc_closure_init(&sd->connectivity_changed_closure, rr_connectivity_changed, sd); diff --git a/src/core/ext/resolver/dns/native/dns_resolver.c b/src/core/ext/resolver/dns/native/dns_resolver.c index 79682e78b5d..1841a75845d 100644 --- a/src/core/ext/resolver/dns/native/dns_resolver.c +++ b/src/core/ext/resolver/dns/native/dns_resolver.c @@ -175,7 +175,14 @@ static void dns_on_resolved(grpc_exec_ctx *exec_ctx, void *arg, grpc_lb_policy_args lb_policy_args; result = grpc_resolver_result_create(); memset(&lb_policy_args, 0, sizeof(lb_policy_args)); - lb_policy_args.addresses = addresses; + lb_policy_args.num_addresses = addresses->naddrs; + lb_policy_args.lb_addresses = + gpr_malloc(sizeof(grpc_lb_address) * lb_policy_args.num_addresses); + memset(lb_policy_args.lb_addresses, 0, + sizeof(grpc_lb_address) * lb_policy_args.num_addresses); + for (size_t i = 0; i < addresses->naddrs; ++i) { + lb_policy_args.lb_addresses[i].resolved_address = &r->addresses->addrs[i]; + } lb_policy_args.client_channel_factory = r->client_channel_factory; lb_policy = grpc_lb_policy_create(exec_ctx, r->lb_policy_name, &lb_policy_args); diff --git a/src/core/ext/resolver/sockaddr/sockaddr_resolver.c b/src/core/ext/resolver/sockaddr/sockaddr_resolver.c index 3807522d2bd..54a7e1c84c9 100644 --- a/src/core/ext/resolver/sockaddr/sockaddr_resolver.c +++ b/src/core/ext/resolver/sockaddr/sockaddr_resolver.c @@ -125,10 +125,18 @@ static void sockaddr_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx, grpc_resolver_result *result = grpc_resolver_result_create(); grpc_lb_policy_args lb_policy_args; memset(&lb_policy_args, 0, sizeof(lb_policy_args)); - lb_policy_args.addresses = r->addresses; + lb_policy_args.num_addresses = r->addresses->naddrs; + lb_policy_args.lb_addresses = + gpr_malloc(sizeof(grpc_lb_address) * lb_policy_args.num_addresses); + memset(lb_policy_args.lb_addresses, 0, + sizeof(grpc_lb_address) * lb_policy_args.num_addresses); + for (size_t i = 0; i < lb_policy_args.num_addresses; ++i) { + lb_policy_args.lb_addresses[i].resolved_address = &r->addresses->addrs[i]; + } lb_policy_args.client_channel_factory = r->client_channel_factory; grpc_lb_policy *lb_policy = grpc_lb_policy_create(exec_ctx, r->lb_policy_name, &lb_policy_args); + gpr_free(lb_policy_args.lb_addresses); grpc_resolver_result_set_lb_policy(result, lb_policy); GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "sockaddr"); r->published = 1; diff --git a/src/core/lib/transport/metadata.c b/src/core/lib/transport/metadata.c index 0677f297660..4b40c275adb 100644 --- a/src/core/lib/transport/metadata.c +++ b/src/core/lib/transport/metadata.c @@ -278,7 +278,7 @@ static void ref_md_locked(mdtab_shard *shard, internal_metadata *md DEBUG_ARGS) { #ifdef GRPC_METADATA_REFCOUNT_DEBUG gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, - "ELM REF:%p:%d->%d: '%s' = '%s'", md, + "ELM REF:%p:%zu->%zu: '%s' = '%s'", (void *)md, gpr_atm_no_barrier_load(&md->refcnt), gpr_atm_no_barrier_load(&md->refcnt) + 1, grpc_mdstr_as_c_string((grpc_mdstr *)md->key), @@ -566,7 +566,7 @@ grpc_mdelem *grpc_mdelem_from_metadata_strings(grpc_mdstr *mkey, shard->elems[idx] = md; gpr_mu_init(&md->mu_user_data); #ifdef GRPC_METADATA_REFCOUNT_DEBUG - gpr_log(GPR_DEBUG, "ELM NEW:%p:%d: '%s' = '%s'", md, + gpr_log(GPR_DEBUG, "ELM NEW:%p:%zu: '%s' = '%s'", (void *)md, gpr_atm_no_barrier_load(&md->refcnt), grpc_mdstr_as_c_string((grpc_mdstr *)md->key), grpc_mdstr_as_c_string((grpc_mdstr *)md->value)); @@ -639,7 +639,7 @@ grpc_mdelem *grpc_mdelem_ref(grpc_mdelem *gmd DEBUG_ARGS) { if (is_mdelem_static(gmd)) return gmd; #ifdef GRPC_METADATA_REFCOUNT_DEBUG gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, - "ELM REF:%p:%d->%d: '%s' = '%s'", md, + "ELM REF:%p:%zu->%zu: '%s' = '%s'", (void *)md, gpr_atm_no_barrier_load(&md->refcnt), gpr_atm_no_barrier_load(&md->refcnt) + 1, grpc_mdstr_as_c_string((grpc_mdstr *)md->key), @@ -649,7 +649,7 @@ grpc_mdelem *grpc_mdelem_ref(grpc_mdelem *gmd DEBUG_ARGS) { this function - meaning that no adjustment to mdtab_free is necessary, simplifying the logic here to be just an atomic increment */ /* use C assert to have this removed in opt builds */ - assert(gpr_atm_no_barrier_load(&md->refcnt) >= 1); + GPR_ASSERT(gpr_atm_no_barrier_load(&md->refcnt) >= 1); gpr_atm_no_barrier_fetch_add(&md->refcnt, 1); return gmd; } @@ -660,14 +660,16 @@ void grpc_mdelem_unref(grpc_mdelem *gmd DEBUG_ARGS) { if (is_mdelem_static(gmd)) return; #ifdef GRPC_METADATA_REFCOUNT_DEBUG gpr_log(file, line, GPR_LOG_SEVERITY_DEBUG, - "ELM UNREF:%p:%d->%d: '%s' = '%s'", md, + "ELM UNREF:%p:%zu->%zu: '%s' = '%s'", (void *)md, gpr_atm_no_barrier_load(&md->refcnt), gpr_atm_no_barrier_load(&md->refcnt) - 1, grpc_mdstr_as_c_string((grpc_mdstr *)md->key), grpc_mdstr_as_c_string((grpc_mdstr *)md->value)); #endif uint32_t hash = GRPC_MDSTR_KV_HASH(md->key->hash, md->value->hash); - if (1 == gpr_atm_full_fetch_add(&md->refcnt, -1)) { + const gpr_atm prev_refcount = gpr_atm_full_fetch_add(&md->refcnt, -1); + GPR_ASSERT(prev_refcount >= 1); + if (1 == prev_refcount) { /* once the refcount hits zero, some other thread can come along and free md at any time: it's unsafe from this point on to access it */ mdtab_shard *shard = @@ -676,10 +678,12 @@ void grpc_mdelem_unref(grpc_mdelem *gmd DEBUG_ARGS) { } } -const char *grpc_mdstr_as_c_string(grpc_mdstr *s) { +const char *grpc_mdstr_as_c_string(const grpc_mdstr *s) { return (const char *)GPR_SLICE_START_PTR(s->slice); } +size_t grpc_mdstr_length(const grpc_mdstr *s) { return GRPC_MDSTR_LENGTH(s); } + grpc_mdstr *grpc_mdstr_ref(grpc_mdstr *gs DEBUG_ARGS) { internal_string *s = (internal_string *)gs; if (is_mdstr_static(gs)) return gs; diff --git a/src/core/lib/transport/metadata.h b/src/core/lib/transport/metadata.h index 2b0921c8d74..71eff0acf26 100644 --- a/src/core/lib/transport/metadata.h +++ b/src/core/lib/transport/metadata.h @@ -147,7 +147,7 @@ void grpc_mdelem_unref(grpc_mdelem *md); /* Recover a char* from a grpc_mdstr. The returned string is null terminated. Does not promise that the returned string has no embedded nulls however. */ -const char *grpc_mdstr_as_c_string(grpc_mdstr *s); +const char *grpc_mdstr_as_c_string(const grpc_mdstr *s); #define GRPC_MDSTR_LENGTH(s) (GPR_SLICE_LENGTH(s->slice)) From 6cc44fcf3ab68d55eadec5590e35b9268cc2d6a6 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Mon, 12 Sep 2016 23:04:35 -0700 Subject: [PATCH 20/25] PR comments --- src/core/ext/lb_policy/grpclb/grpclb.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/core/ext/lb_policy/grpclb/grpclb.c b/src/core/ext/lb_policy/grpclb/grpclb.c index ad070e458a7..7fdc97dc3d7 100644 --- a/src/core/ext/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/lb_policy/grpclb/grpclb.c @@ -722,13 +722,12 @@ static int glb_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, glb_lb_policy *glb_policy = (glb_lb_policy *)pol; if (pick_args->lb_token_mdelem_storage == NULL) { - /* TODO(dgq): should this be an assert? If storage is NULL, something has - * gone very wrong at the client channel filter */ - gpr_log(GPR_ERROR, - "No mdelem storage for the LB token. Load reporting won't work " - "without it. Failing"); *target = NULL; - grpc_exec_ctx_sched(exec_ctx, on_complete, GRPC_ERROR_NONE, NULL); + grpc_exec_ctx_sched( + exec_ctx, on_complete, + GRPC_ERROR_CREATE("No mdelem storage for the LB token. Load reporting " + "won't work without it. Failing"), + NULL); return 1; } From 35c2aba849e0f6852e3538fc488a6a2afec81c09 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Tue, 13 Sep 2016 15:28:09 -0700 Subject: [PATCH 21/25] More PR comments. Mainly, removed all user_data memory management responsibilities from the round robin policy. --- .../ext/client_config/lb_policy_factory.h | 1 - src/core/ext/lb_policy/grpclb/grpclb.c | 93 +++++++++++++------ .../ext/lb_policy/round_robin/round_robin.c | 19 +--- test/cpp/grpclb/grpclb_test.cc | 1 + 4 files changed, 69 insertions(+), 45 deletions(-) diff --git a/src/core/ext/client_config/lb_policy_factory.h b/src/core/ext/client_config/lb_policy_factory.h index e1d67633b42..d2d1a0f16e0 100644 --- a/src/core/ext/client_config/lb_policy_factory.h +++ b/src/core/ext/client_config/lb_policy_factory.h @@ -65,7 +65,6 @@ typedef struct grpc_lb_policy_user_data_vtable { typedef struct grpc_lb_policy_args { grpc_lb_address *lb_addresses; size_t num_addresses; - grpc_lb_policy_user_data_vtable user_data_vtable; grpc_client_channel_factory *client_channel_factory; } grpc_lb_policy_args; diff --git a/src/core/ext/lb_policy/grpclb/grpclb.c b/src/core/ext/lb_policy/grpclb/grpclb.c index 7fdc97dc3d7..9a176ba0d10 100644 --- a/src/core/ext/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/lb_policy/grpclb/grpclb.c @@ -185,7 +185,7 @@ static void wrapped_rr_closure(grpc_exec_ctx *exec_ctx, void *arg, initial_metadata_add_lb_token(wc_arg->initial_metadata, wc_arg->lb_token_mdelem_storage, - wc_arg->lb_token); + user_data_copy(wc_arg->lb_token)); grpc_exec_ctx_sched(exec_ctx, wc_arg->wrapped_closure, error, NULL); gpr_free(wc_arg->owning_pending_node); @@ -302,6 +302,12 @@ typedef struct glb_lb_policy { * response has arrived. */ grpc_grpclb_serverlist *serverlist; + /** total number of valid addresses received in \a serverlist */ + size_t num_ok_serverlist_addresses; + + /** LB addresses from \a serverlist, \a num_ok_serverlist_addresses of them */ + grpc_lb_address *lb_addresses; + /** list of picks that are waiting on RR's policy connectivity */ pending_pick *pending_picks; @@ -329,32 +335,39 @@ struct rr_connectivity_data { glb_lb_policy *glb_policy; }; -/* populate \a addresses according to \a serverlist. Returns the number of - * addresses successfully parsed and added to \a addresses */ -static size_t process_serverlist(const grpc_grpclb_serverlist *serverlist, - grpc_lb_address **lb_addresses) { - size_t num_valid = 0; - /* first pass: count how many are valid in order to allocate the necessary - * memory in a single block */ - for (size_t i = 0; i < serverlist->num_servers; ++i) { - const grpc_grpclb_server *server = serverlist->servers[i]; - const grpc_grpclb_ip_address *ip = &server->ip_address; - - if (server->port >> 16 != 0) { +static bool is_server_valid(const grpc_grpclb_server *server, size_t idx, + bool log) { + const grpc_grpclb_ip_address *ip = &server->ip_address; + if (server->port >> 16 != 0) { + if (log) { gpr_log(GPR_ERROR, "Invalid port '%d' at index %zu of serverlist. Ignoring.", - server->port, i); - continue; + server->port, idx); } + return false; + } - if (ip->size != 4 && ip->size != 16) { + if (ip->size != 4 && ip->size != 16) { + if (log) { gpr_log(GPR_ERROR, "Expected IP to be 4 or 16 bytes, got %d at index %zu of " "serverlist. Ignoring", - ip->size, i); - continue; + ip->size, idx); } - ++num_valid; + return false; + } + return true; +} + +/* populate \a addresses according to \a serverlist. Returns the number of + * addresses successfully parsed and added to \a addresses */ +static size_t process_serverlist(const grpc_grpclb_serverlist *serverlist, + grpc_lb_address **lb_addresses) { + size_t num_valid = 0; + /* first pass: count how many are valid in order to allocate the necessary + * memory in a single block */ + for (size_t i = 0; i < serverlist->num_servers; ++i) { + if (is_server_valid(serverlist->servers[i], i, true)) ++num_valid; } if (num_valid == 0) { return 0; @@ -368,9 +381,14 @@ static size_t process_serverlist(const grpc_grpclb_serverlist *serverlist, memset(lb_addrs, 0, sizeof(grpc_lb_address) * num_valid); /* second pass: actually populate the addresses and LB tokens (aka user data - * to the outside world) to be read by the RR policy during its creation */ + * to the outside world) to be read by the RR policy during its creation. + * Given that the validity tests are very cheap, they are performed again + * instead of marking the valid ones during the first pass, as this would + * incurr in an allocation due to the arbitrary number of server */ + size_t num_processed = 0; for (size_t i = 0; i < num_valid; ++i) { const grpc_grpclb_server *server = serverlist->servers[i]; + if (!is_server_valid(serverlist->servers[i], i, false)) continue; grpc_lb_address *const lb_addr = &lb_addrs[i]; /* lb token processing */ @@ -410,7 +428,9 @@ static size_t process_serverlist(const grpc_grpclb_serverlist *serverlist, addr6->sin6_port = netorder_port; } GPR_ASSERT(*sa_len > 0); + ++num_processed; } + GPR_ASSERT(num_processed == num_valid); *lb_addresses = lb_addrs; return num_valid; } @@ -423,19 +443,27 @@ static grpc_lb_policy *create_rr(grpc_exec_ctx *exec_ctx, grpc_lb_policy_args args; memset(&args, 0, sizeof(args)); args.client_channel_factory = glb_policy->cc_factory; - args.num_addresses = process_serverlist(serverlist, &args.lb_addresses); - args.user_data_vtable.copy = user_data_copy; - args.user_data_vtable.destroy = user_data_destroy; + const size_t num_ok_addresses = + process_serverlist(serverlist, &args.lb_addresses); + args.num_addresses = num_ok_addresses; grpc_lb_policy *rr = grpc_lb_policy_create(exec_ctx, "round_robin", &args); + + glb_policy->num_ok_serverlist_addresses = num_ok_addresses; + if (glb_policy->lb_addresses != NULL) { + /* dispose of the previous version */ + for (size_t i = 0; i < num_ok_addresses; ++i) { + user_data_destroy(glb_policy->lb_addresses[i].user_data); + } + gpr_free(glb_policy->lb_addresses); + } + + glb_policy->lb_addresses = args.lb_addresses; + if (args.num_addresses > 0) { /* free "resolved" addresses memblock */ gpr_free(args.lb_addresses->resolved_address); } - for (size_t i = 0; i < args.num_addresses; ++i) { - args.user_data_vtable.destroy(args.lb_addresses[i].user_data); - } - gpr_free(args.lb_addresses); return rr; } @@ -601,6 +629,11 @@ static void glb_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { grpc_grpclb_destroy_serverlist(glb_policy->serverlist); } gpr_mu_destroy(&glb_policy->mu); + + for (size_t i = 0; i < glb_policy->num_ok_serverlist_addresses; ++i) { + user_data_destroy(glb_policy->lb_addresses[i].user_data); + } + gpr_free(glb_policy->lb_addresses); gpr_free(glb_policy); } @@ -762,9 +795,9 @@ static int glb_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, GRPC_LB_POLICY_UNREF(exec_ctx, glb_policy->wc_arg.rr_policy, "glb_pick"); /* add the load reporting initial metadata */ - initial_metadata_add_lb_token(pick_args->initial_metadata, - pick_args->lb_token_mdelem_storage, - glb_policy->wc_arg.lb_token); + initial_metadata_add_lb_token( + pick_args->initial_metadata, pick_args->lb_token_mdelem_storage, + user_data_copy(glb_policy->wc_arg.lb_token)); } } else { grpc_polling_entity_add_to_pollset_set(exec_ctx, pick_args->pollent, diff --git a/src/core/ext/lb_policy/round_robin/round_robin.c b/src/core/ext/lb_policy/round_robin/round_robin.c index 2069dc192c0..4d89cef9b6e 100644 --- a/src/core/ext/lb_policy/round_robin/round_robin.c +++ b/src/core/ext/lb_policy/round_robin/round_robin.c @@ -81,9 +81,6 @@ typedef struct pending_pick { /* polling entity for the pick()'s async notification */ grpc_polling_entity *pollent; - /* the initial metadata for the pick. See grpc_lb_policy_pick() */ - grpc_metadata_batch *initial_metadata; - /* output argument where to store the pick()ed user_data. It'll be NULL if no * such data is present or there's an error (the definite test for errors is * \a target being NULL). */ @@ -133,10 +130,9 @@ struct round_robin_lb_policy { /** total number of addresses received at creation time */ size_t num_addresses; - /** user data, one per incoming address */ + /** user data, one per incoming address. This pointer is borrowed and opaque. + * It'll be returned as-is in successful picks. */ void **user_data; - /** functions to operate over \a user_data elements */ - grpc_lb_policy_user_data_vtable user_data_vtable; /** all our subchannels */ size_t num_subchannels; @@ -285,9 +281,6 @@ static void rr_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { elem = tmp; } - for (size_t i = 0; i < p->num_addresses; i++) { - p->user_data_vtable.destroy(p->user_data[i]); - } gpr_free(p->user_data); gpr_free(p); } @@ -410,7 +403,7 @@ static int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, /* readily available, report right away */ gpr_mu_unlock(&p->mu); *target = grpc_subchannel_get_connected_subchannel(selected->subchannel); - *user_data = p->user_data_vtable.copy(selected->user_data); + *user_data = selected->user_data; if (grpc_lb_round_robin_trace) { gpr_log(GPR_DEBUG, "[RR PICK] TARGET <-- CONNECTED SUBCHANNEL %p (NODE %p)", @@ -431,7 +424,6 @@ static int rr_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, pp->pollent = pick_args->pollent; pp->target = target; pp->on_complete = on_complete; - pp->initial_metadata = pick_args->initial_metadata; pp->initial_metadata_flags = pick_args->initial_metadata_flags; pp->user_data = user_data; p->pending_picks = pp; @@ -477,7 +469,7 @@ static void rr_connectivity_changed(grpc_exec_ctx *exec_ctx, void *arg, *pp->target = grpc_subchannel_get_connected_subchannel(selected->subchannel); - *pp->user_data = p->user_data_vtable.copy(selected->user_data); + *pp->user_data = selected->user_data; if (grpc_lb_round_robin_trace) { gpr_log(GPR_DEBUG, "[RR CONN CHANGED] TARGET <-- SUBCHANNEL %p (NODE %p)", @@ -623,7 +615,6 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx, memset(p->subchannels, 0, sizeof(*p->subchannels) * p->num_addresses); p->user_data = gpr_malloc(sizeof(void *) * p->num_addresses); memset(p->user_data, 0, sizeof(void *) * p->num_addresses); - p->user_data_vtable = args->user_data_vtable; grpc_subchannel_args sc_args; size_t subchannel_idx = 0; @@ -633,7 +624,7 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx, (struct sockaddr *)args->lb_addresses[i].resolved_address->addr; sc_args.addr_len = args->lb_addresses[i].resolved_address->len; - p->user_data[i] = p->user_data_vtable.copy(args->lb_addresses[i].user_data); + p->user_data[i] = args->lb_addresses[i].user_data; grpc_subchannel *subchannel = grpc_client_channel_factory_create_subchannel( exec_ctx, args->client_channel_factory, &sc_args); diff --git a/test/cpp/grpclb/grpclb_test.cc b/test/cpp/grpclb/grpclb_test.cc index 76c429872f4..c0442561655 100644 --- a/test/cpp/grpclb/grpclb_test.cc +++ b/test/cpp/grpclb/grpclb_test.cc @@ -70,6 +70,7 @@ extern "C" { // - Send an empty serverlist update and verify that the client request blocks // until a new serverlist with actual contents is available. // - Send identical serverlist update +// - Send a serverlist with faulty ip:port addresses (port > 2^16, etc). // - Test reception of invalid serverlist // - Test pinging // - Test against a non-LB server. That server should return UNIMPLEMENTED and From f47d6fbdccd6e3c64b0ff70b6a63236819886540 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Wed, 14 Sep 2016 12:59:17 -0700 Subject: [PATCH 22/25] More PR comments --- .../ext/client_config/lb_policy_factory.h | 15 +-- src/core/ext/lb_policy/grpclb/grpclb.c | 107 ++++++++++-------- .../ext/lb_policy/pick_first/pick_first.c | 9 +- .../ext/lb_policy/round_robin/round_robin.c | 9 +- .../ext/resolver/dns/native/dns_resolver.c | 7 +- .../ext/resolver/sockaddr/sockaddr_resolver.c | 8 +- test/cpp/grpclb/grpclb_test.cc | 63 ++++++----- 7 files changed, 118 insertions(+), 100 deletions(-) diff --git a/src/core/ext/client_config/lb_policy_factory.h b/src/core/ext/client_config/lb_policy_factory.h index d2d1a0f16e0..7191ca7d89b 100644 --- a/src/core/ext/client_config/lb_policy_factory.h +++ b/src/core/ext/client_config/lb_policy_factory.h @@ -48,22 +48,17 @@ struct grpc_lb_policy_factory { }; /** A resolved address alongside any LB related information associated with it. - * \a user_data, if not \a NULL, is opaque and meant to be consumed by the gRPC - * LB policy. Anywhere else, refer to the functions in \a - * grpc_lb_policy_user_data_vtable to operate with it */ + * \a user_data, if not NULL, contains opaque data meant to be consumed by the + * gRPC LB policy. Note that no all LB policies support \a user_data as input. + * Those who don't will simply ignore it and will correspondingly return NULL in + * their namesake pick() output argument. */ typedef struct grpc_lb_address { grpc_resolved_address *resolved_address; void *user_data; } grpc_lb_address; -/** Functions acting upon the opaque \a grpc_lb_address.user_data */ -typedef struct grpc_lb_policy_user_data_vtable { - void *(*copy)(void *); - void (*destroy)(void *); -} grpc_lb_policy_user_data_vtable; - typedef struct grpc_lb_policy_args { - grpc_lb_address *lb_addresses; + grpc_lb_address *addresses; size_t num_addresses; grpc_client_channel_factory *client_channel_factory; } grpc_lb_policy_args; diff --git a/src/core/ext/lb_policy/grpclb/grpclb.c b/src/core/ext/lb_policy/grpclb/grpclb.c index 9a176ba0d10..11504511163 100644 --- a/src/core/ext/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/lb_policy/grpclb/grpclb.c @@ -125,9 +125,16 @@ static void *user_data_copy(void *user_data) { return GRPC_MDELEM_REF(user_data); } -static void user_data_destroy(void *user_data) { - if (user_data == NULL) return; - GRPC_MDELEM_UNREF(user_data); +static void lb_addrs_destroy(grpc_lb_address *lb_addresses, + size_t num_addresses) { + /* free "resolved" addresses memblock */ + gpr_free(lb_addresses->resolved_address); + for (size_t i = 0; i < num_addresses; ++i) { + if (lb_addresses[i].user_data != NULL) { + GRPC_MDELEM_UNREF(lb_addresses[i].user_data); + } + } + gpr_free(lb_addresses); } /* add lb_token of selected subchannel (address) to the call's initial @@ -385,21 +392,12 @@ static size_t process_serverlist(const grpc_grpclb_serverlist *serverlist, * Given that the validity tests are very cheap, they are performed again * instead of marking the valid ones during the first pass, as this would * incurr in an allocation due to the arbitrary number of server */ - size_t num_processed = 0; - for (size_t i = 0; i < num_valid; ++i) { - const grpc_grpclb_server *server = serverlist->servers[i]; - if (!is_server_valid(serverlist->servers[i], i, false)) continue; - grpc_lb_address *const lb_addr = &lb_addrs[i]; - - /* lb token processing */ - if (server->has_load_balance_token) { - const size_t lb_token_size = - GPR_ARRAY_SIZE(server->load_balance_token) - 1; - grpc_mdstr *lb_token_mdstr = grpc_mdstr_from_buffer( - (uint8_t *)server->load_balance_token, lb_token_size); - lb_addr->user_data = grpc_mdelem_from_metadata_strings( - GRPC_MDSTR_LOAD_REPORTING_INITIAL, lb_token_mdstr); - } + size_t addr_idx = 0; + for (size_t sl_idx = 0; sl_idx < serverlist->num_servers; ++sl_idx) { + GPR_ASSERT(addr_idx < num_valid); + const grpc_grpclb_server *server = serverlist->servers[sl_idx]; + if (!is_server_valid(serverlist->servers[sl_idx], sl_idx, false)) continue; + grpc_lb_address *const lb_addr = &lb_addrs[addr_idx]; /* address processing */ const uint16_t netorder_port = htons((uint16_t)server->port); @@ -407,7 +405,7 @@ static size_t process_serverlist(const grpc_grpclb_serverlist *serverlist, * server->ip_address.bytes. */ const grpc_grpclb_ip_address *ip = &server->ip_address; - lb_addr->resolved_address = &r_addrs_memblock[i]; + lb_addr->resolved_address = &r_addrs_memblock[addr_idx]; struct sockaddr_storage *sa = (struct sockaddr_storage *)lb_addr->resolved_address->addr; size_t *sa_len = &lb_addr->resolved_address->len; @@ -428,9 +426,25 @@ static size_t process_serverlist(const grpc_grpclb_serverlist *serverlist, addr6->sin6_port = netorder_port; } GPR_ASSERT(*sa_len > 0); - ++num_processed; + + /* lb token processing */ + if (server->has_load_balance_token) { + const size_t lb_token_size = + GPR_ARRAY_SIZE(server->load_balance_token) - 1; + grpc_mdstr *lb_token_mdstr = grpc_mdstr_from_buffer( + (uint8_t *)server->load_balance_token, lb_token_size); + lb_addr->user_data = grpc_mdelem_from_metadata_strings( + GRPC_MDSTR_LOAD_REPORTING_INITIAL, lb_token_mdstr); + } else { + gpr_log(GPR_ERROR, + "Missing LB token for backend address '%s'. The empty token will " + "be used instead", + grpc_sockaddr_to_uri((struct sockaddr *)sa)); + lb_addr->user_data = GRPC_MDELEM_LOAD_REPORTING_INITIAL_EMPTY; + } + ++addr_idx; } - GPR_ASSERT(num_processed == num_valid); + GPR_ASSERT(addr_idx == num_valid); *lb_addresses = lb_addrs; return num_valid; } @@ -444,26 +458,19 @@ static grpc_lb_policy *create_rr(grpc_exec_ctx *exec_ctx, memset(&args, 0, sizeof(args)); args.client_channel_factory = glb_policy->cc_factory; const size_t num_ok_addresses = - process_serverlist(serverlist, &args.lb_addresses); + process_serverlist(serverlist, &args.addresses); args.num_addresses = num_ok_addresses; grpc_lb_policy *rr = grpc_lb_policy_create(exec_ctx, "round_robin", &args); - glb_policy->num_ok_serverlist_addresses = num_ok_addresses; if (glb_policy->lb_addresses != NULL) { /* dispose of the previous version */ - for (size_t i = 0; i < num_ok_addresses; ++i) { - user_data_destroy(glb_policy->lb_addresses[i].user_data); - } - gpr_free(glb_policy->lb_addresses); + lb_addrs_destroy(glb_policy->lb_addresses, + glb_policy->num_ok_serverlist_addresses); } + glb_policy->num_ok_serverlist_addresses = num_ok_addresses; + glb_policy->lb_addresses = args.addresses; - glb_policy->lb_addresses = args.lb_addresses; - - if (args.num_addresses > 0) { - /* free "resolved" addresses memblock */ - gpr_free(args.lb_addresses->resolved_address); - } return rr; } @@ -560,7 +567,8 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, memset(glb_policy, 0, sizeof(*glb_policy)); /* All input addresses in args->addresses come from a resolver that claims - * they are LB services. It's the resolver's responsibility to make sure this + * they are LB services. It's the resolver's responsibility to make sure + * this * policy is only instantiated and used in that case. * * Create a client channel over them to communicate with a LB service */ @@ -570,18 +578,24 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, return NULL; } + /* this LB policy doesn't support \a user_data */ + GPR_ASSERT(args->addresses[0].user_data == NULL); + /* construct a target from the addresses in args, given in the form * ipvX://ip1:port1,ip2:port2,... * TODO(dgq): support mixed ip version */ char **addr_strs = gpr_malloc(sizeof(char *) * args->num_addresses); addr_strs[0] = grpc_sockaddr_to_uri( - (const struct sockaddr *)&args->lb_addresses[0].resolved_address->addr); + (const struct sockaddr *)&args->addresses[0].resolved_address->addr); for (size_t i = 1; i < args->num_addresses; i++) { + /* this LB policy doesn't support \a user_data */ + GPR_ASSERT(args->addresses[i].user_data == NULL); + GPR_ASSERT( - grpc_sockaddr_to_string(&addr_strs[i], - (const struct sockaddr *)&args->lb_addresses[i] - .resolved_address->addr, - true) == 0); + grpc_sockaddr_to_string( + &addr_strs[i], + (const struct sockaddr *)&args->addresses[i].resolved_address->addr, + true) == 0); } size_t uri_path_len; char *target_uri_str = gpr_strjoin_sep( @@ -630,10 +644,8 @@ static void glb_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { } gpr_mu_destroy(&glb_policy->mu); - for (size_t i = 0; i < glb_policy->num_ok_serverlist_addresses; ++i) { - user_data_destroy(glb_policy->lb_addresses[i].user_data); - } - gpr_free(glb_policy->lb_addresses); + lb_addrs_destroy(glb_policy->lb_addresses, + glb_policy->num_ok_serverlist_addresses); gpr_free(glb_policy); } @@ -882,7 +894,8 @@ typedef struct lb_client_data { grpc_metadata_array initial_metadata_recv; /* initial MD from LB server */ grpc_metadata_array trailing_metadata_recv; /* trailing MD from LB server */ - /* what's being sent to the LB server. Note that its value may vary if the LB + /* what's being sent to the LB server. Note that its value may vary if the + * LB * server indicates a redirect. */ grpc_byte_buffer *request_payload; @@ -1090,7 +1103,8 @@ static void res_recv_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { * it'll just create the first RR policy instance */ rr_handover(exec_ctx, lb_client->glb_policy, error); } else { - /* unref the RR policy, eventually leading to its substitution with a + /* unref the RR policy, eventually leading to its substitution with + * a * new one constructed from the received serverlist (see * glb_rr_connectivity_changed) */ GRPC_LB_POLICY_UNREF(exec_ctx, lb_client->glb_policy->rr_policy, @@ -1155,7 +1169,8 @@ static void srv_status_rcvd_cb(grpc_exec_ctx *exec_ctx, void *arg, lb_client->status, lb_client->status_details, lb_client->status_details_capacity); } - /* TODO(dgq): deal with stream termination properly (fire up another one? fail + /* TODO(dgq): deal with stream termination properly (fire up another one? + * fail * the original call?) */ } diff --git a/src/core/ext/lb_policy/pick_first/pick_first.c b/src/core/ext/lb_policy/pick_first/pick_first.c index 21d948033a0..d907ddd5d1d 100644 --- a/src/core/ext/lb_policy/pick_first/pick_first.c +++ b/src/core/ext/lb_policy/pick_first/pick_first.c @@ -438,7 +438,7 @@ static void pick_first_factory_unref(grpc_lb_policy_factory *factory) {} static grpc_lb_policy *create_pick_first(grpc_exec_ctx *exec_ctx, grpc_lb_policy_factory *factory, grpc_lb_policy_args *args) { - GPR_ASSERT(args->lb_addresses != NULL); + GPR_ASSERT(args->addresses != NULL); GPR_ASSERT(args->client_channel_factory != NULL); if (args->num_addresses == 0) return NULL; @@ -451,10 +451,13 @@ static grpc_lb_policy *create_pick_first(grpc_exec_ctx *exec_ctx, grpc_subchannel_args sc_args; size_t subchannel_idx = 0; for (size_t i = 0; i < args->num_addresses; i++) { + /* this LB policy doesn't support \a user_data */ + GPR_ASSERT(args->addresses[i].user_data == NULL); + memset(&sc_args, 0, sizeof(grpc_subchannel_args)); sc_args.addr = - (struct sockaddr *)(args->lb_addresses[i].resolved_address->addr); - sc_args.addr_len = (size_t)args->lb_addresses[i].resolved_address->len; + (struct sockaddr *)(args->addresses[i].resolved_address->addr); + sc_args.addr_len = (size_t)args->addresses[i].resolved_address->len; grpc_subchannel *subchannel = grpc_client_channel_factory_create_subchannel( exec_ctx, args->client_channel_factory, &sc_args); diff --git a/src/core/ext/lb_policy/round_robin/round_robin.c b/src/core/ext/lb_policy/round_robin/round_robin.c index 4d89cef9b6e..1351ff0277f 100644 --- a/src/core/ext/lb_policy/round_robin/round_robin.c +++ b/src/core/ext/lb_policy/round_robin/round_robin.c @@ -603,7 +603,7 @@ static void round_robin_factory_unref(grpc_lb_policy_factory *factory) {} static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx, grpc_lb_policy_factory *factory, grpc_lb_policy_args *args) { - GPR_ASSERT(args->lb_addresses != NULL); + GPR_ASSERT(args->addresses != NULL); GPR_ASSERT(args->client_channel_factory != NULL); if (args->num_addresses == 0) return NULL; @@ -620,11 +620,10 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx, size_t subchannel_idx = 0; for (size_t i = 0; i < p->num_addresses; i++) { memset(&sc_args, 0, sizeof(grpc_subchannel_args)); - sc_args.addr = - (struct sockaddr *)args->lb_addresses[i].resolved_address->addr; - sc_args.addr_len = args->lb_addresses[i].resolved_address->len; + sc_args.addr = (struct sockaddr *)args->addresses[i].resolved_address->addr; + sc_args.addr_len = args->addresses[i].resolved_address->len; - p->user_data[i] = args->lb_addresses[i].user_data; + p->user_data[i] = args->addresses[i].user_data; grpc_subchannel *subchannel = grpc_client_channel_factory_create_subchannel( exec_ctx, args->client_channel_factory, &sc_args); diff --git a/src/core/ext/resolver/dns/native/dns_resolver.c b/src/core/ext/resolver/dns/native/dns_resolver.c index 1841a75845d..32e9de69a6c 100644 --- a/src/core/ext/resolver/dns/native/dns_resolver.c +++ b/src/core/ext/resolver/dns/native/dns_resolver.c @@ -176,16 +176,17 @@ static void dns_on_resolved(grpc_exec_ctx *exec_ctx, void *arg, result = grpc_resolver_result_create(); memset(&lb_policy_args, 0, sizeof(lb_policy_args)); lb_policy_args.num_addresses = addresses->naddrs; - lb_policy_args.lb_addresses = + lb_policy_args.addresses = gpr_malloc(sizeof(grpc_lb_address) * lb_policy_args.num_addresses); - memset(lb_policy_args.lb_addresses, 0, + memset(lb_policy_args.addresses, 0, sizeof(grpc_lb_address) * lb_policy_args.num_addresses); for (size_t i = 0; i < addresses->naddrs; ++i) { - lb_policy_args.lb_addresses[i].resolved_address = &r->addresses->addrs[i]; + lb_policy_args.addresses[i].resolved_address = &r->addresses->addrs[i]; } lb_policy_args.client_channel_factory = r->client_channel_factory; lb_policy = grpc_lb_policy_create(exec_ctx, r->lb_policy_name, &lb_policy_args); + gpr_free(lb_policy_args.addresses); if (lb_policy != NULL) { grpc_resolver_result_set_lb_policy(result, lb_policy); GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "construction"); diff --git a/src/core/ext/resolver/sockaddr/sockaddr_resolver.c b/src/core/ext/resolver/sockaddr/sockaddr_resolver.c index 54a7e1c84c9..425285287cf 100644 --- a/src/core/ext/resolver/sockaddr/sockaddr_resolver.c +++ b/src/core/ext/resolver/sockaddr/sockaddr_resolver.c @@ -126,17 +126,17 @@ static void sockaddr_maybe_finish_next_locked(grpc_exec_ctx *exec_ctx, grpc_lb_policy_args lb_policy_args; memset(&lb_policy_args, 0, sizeof(lb_policy_args)); lb_policy_args.num_addresses = r->addresses->naddrs; - lb_policy_args.lb_addresses = + lb_policy_args.addresses = gpr_malloc(sizeof(grpc_lb_address) * lb_policy_args.num_addresses); - memset(lb_policy_args.lb_addresses, 0, + memset(lb_policy_args.addresses, 0, sizeof(grpc_lb_address) * lb_policy_args.num_addresses); for (size_t i = 0; i < lb_policy_args.num_addresses; ++i) { - lb_policy_args.lb_addresses[i].resolved_address = &r->addresses->addrs[i]; + lb_policy_args.addresses[i].resolved_address = &r->addresses->addrs[i]; } lb_policy_args.client_channel_factory = r->client_channel_factory; grpc_lb_policy *lb_policy = grpc_lb_policy_create(exec_ctx, r->lb_policy_name, &lb_policy_args); - gpr_free(lb_policy_args.lb_addresses); + gpr_free(lb_policy_args.addresses); grpc_resolver_result_set_lb_policy(result, lb_policy); GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "sockaddr"); r->published = 1; diff --git a/test/cpp/grpclb/grpclb_test.cc b/test/cpp/grpclb/grpclb_test.cc index c0442561655..95abe38031b 100644 --- a/test/cpp/grpclb/grpclb_test.cc +++ b/test/cpp/grpclb/grpclb_test.cc @@ -37,6 +37,8 @@ #include #include +#include + #include #include #include @@ -76,6 +78,7 @@ extern "C" { // - Test against a non-LB server. That server should return UNIMPLEMENTED and // the call should fail. // - Random LB server closing the stream unexpectedly. +// - Test using DNS-resolvable names (localhost?) namespace grpc { namespace { @@ -612,27 +615,30 @@ static void fork_lb_server(void *arg) { tf->lb_server_update_delay_ms); } -static void setup_test_fixture(test_fixture *tf, - int lb_server_update_delay_ms) { - tf->lb_server_update_delay_ms = lb_server_update_delay_ms; +static test_fixture setup_test_fixture(int lb_server_update_delay_ms) { + test_fixture tf; + memset(&tf, 0, sizeof(tf)); + tf.lb_server_update_delay_ms = lb_server_update_delay_ms; gpr_thd_options options = gpr_thd_options_default(); gpr_thd_options_set_joinable(&options); for (int i = 0; i < NUM_BACKENDS; ++i) { - setup_server("127.0.0.1", &tf->lb_backends[i]); - gpr_thd_new(&tf->lb_backends[i].tid, fork_backend_server, - &tf->lb_backends[i], &options); + setup_server("127.0.0.1", &tf.lb_backends[i]); + gpr_thd_new(&tf.lb_backends[i].tid, fork_backend_server, &tf.lb_backends[i], + &options); } - setup_server("127.0.0.1", &tf->lb_server); - gpr_thd_new(&tf->lb_server.tid, fork_lb_server, &tf->lb_server, &options); + setup_server("127.0.0.1", &tf.lb_server); + gpr_thd_new(&tf.lb_server.tid, fork_lb_server, &tf.lb_server, &options); char *server_uri; gpr_asprintf(&server_uri, "ipv4:%s?lb_policy=grpclb&lb_enabled=1", - tf->lb_server.servers_hostport); - setup_client(server_uri, &tf->client); + tf.lb_server.servers_hostport); + setup_client(server_uri, &tf.client); gpr_free(server_uri); + + return tf; } static void teardown_test_fixture(test_fixture *tf) { @@ -643,19 +649,13 @@ static void teardown_test_fixture(test_fixture *tf) { teardown_server(&tf->lb_server); } -// The LB server will send two updates: batch 1 and batch 2. Each batch -// contains -// two addresses, both of a valid and running backend server. Batch 1 is -// readily -// available and provided as soon as the client establishes the streaming -// call. -// Batch 2 is sent after a delay of \a lb_server_update_delay_ms -// milliseconds. +// The LB server will send two updates: batch 1 and batch 2. Each batch contains +// two addresses, both of a valid and running backend server. Batch 1 is readily +// available and provided as soon as the client establishes the streaming call. +// Batch 2 is sent after a delay of \a lb_server_update_delay_ms milliseconds. static test_fixture test_update(int lb_server_update_delay_ms) { gpr_log(GPR_INFO, "start %s(%d)", __func__, lb_server_update_delay_ms); - test_fixture tf; - memset(&tf, 0, sizeof(tf)); - setup_test_fixture(&tf, lb_server_update_delay_ms); + test_fixture tf = setup_test_fixture(lb_server_update_delay_ms); perform_request( &tf.client); // "consumes" 1st backend server of 1st serverlist perform_request( @@ -671,13 +671,7 @@ static test_fixture test_update(int lb_server_update_delay_ms) { return tf; } -} // namespace -} // namespace grpc - -int main(int argc, char **argv) { - grpc_test_init(argc, argv); - grpc_init(); - +TEST(GrpclbTest, Updates) { grpc::test_fixture tf_result; // Clients take a bit over one second to complete a call (the last part of the // call sleeps for 1 second while verifying the client's completion queue is @@ -712,7 +706,18 @@ int main(int argc, char **argv) { GPR_ASSERT(tf_result.lb_backends[1].num_calls_serviced > 0); GPR_ASSERT(tf_result.lb_backends[2].num_calls_serviced > 0); GPR_ASSERT(tf_result.lb_backends[3].num_calls_serviced == 0); +} + +TEST(GrpclbTest, InvalidAddressInServerlist) {} +} // namespace +} // namespace grpc + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + grpc_test_init(argc, argv); + grpc_init(); + const auto result = RUN_ALL_TESTS(); grpc_shutdown(); - return 0; + return result; } From 8424fdcb70c5a316a66c55401550209a180e838d Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Thu, 15 Sep 2016 08:35:59 -0700 Subject: [PATCH 23/25] PR comments --- src/core/ext/client_config/lb_policy.h | 8 +++----- src/core/ext/lb_policy/grpclb/grpclb.c | 18 +++++------------- .../ext/lb_policy/round_robin/round_robin.c | 12 ++++++------ 3 files changed, 14 insertions(+), 24 deletions(-) diff --git a/src/core/ext/client_config/lb_policy.h b/src/core/ext/client_config/lb_policy.h index 734b3bff970..37c93d707c1 100644 --- a/src/core/ext/client_config/lb_policy.h +++ b/src/core/ext/client_config/lb_policy.h @@ -144,11 +144,9 @@ void grpc_lb_policy_init(grpc_lb_policy *policy, value will be returned. In the asynchronous case, zero is returned and \a on_complete will be called once \a target and \a user_data have been set. Any IO should be done under - \a - pick_args->pollent. - The opaque \a user_data output argument corresponds to information that may - need be propagated from the LB policy. It may be NULL. - Errors are signaled by receiving a NULL \a *target. */ + \a pick_args->pollent. The opaque \a user_data output argument corresponds + to information that may need be propagated from the LB policy. It may be + NULL. Errors are signaled by receiving a NULL \a *target. */ int grpc_lb_policy_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *policy, const grpc_lb_policy_pick_args *pick_args, grpc_connected_subchannel **target, void **user_data, diff --git a/src/core/ext/lb_policy/grpclb/grpclb.c b/src/core/ext/lb_policy/grpclb/grpclb.c index 11504511163..3ce3910b30e 100644 --- a/src/core/ext/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/lb_policy/grpclb/grpclb.c @@ -120,11 +120,6 @@ int grpc_lb_glb_trace = 0; -static void *user_data_copy(void *user_data) { - if (user_data == NULL) return NULL; - return GRPC_MDELEM_REF(user_data); -} - static void lb_addrs_destroy(grpc_lb_address *lb_addresses, size_t num_addresses) { /* free "resolved" addresses memblock */ @@ -192,7 +187,7 @@ static void wrapped_rr_closure(grpc_exec_ctx *exec_ctx, void *arg, initial_metadata_add_lb_token(wc_arg->initial_metadata, wc_arg->lb_token_mdelem_storage, - user_data_copy(wc_arg->lb_token)); + GRPC_MDELEM_REF(wc_arg->lb_token)); grpc_exec_ctx_sched(exec_ctx, wc_arg->wrapped_closure, error, NULL); gpr_free(wc_arg->owning_pending_node); @@ -809,7 +804,7 @@ static int glb_pick(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol, /* add the load reporting initial metadata */ initial_metadata_add_lb_token( pick_args->initial_metadata, pick_args->lb_token_mdelem_storage, - user_data_copy(glb_policy->wc_arg.lb_token)); + GRPC_MDELEM_REF(glb_policy->wc_arg.lb_token)); } } else { grpc_polling_entity_add_to_pollset_set(exec_ctx, pick_args->pollent, @@ -894,8 +889,7 @@ typedef struct lb_client_data { grpc_metadata_array initial_metadata_recv; /* initial MD from LB server */ grpc_metadata_array trailing_metadata_recv; /* trailing MD from LB server */ - /* what's being sent to the LB server. Note that its value may vary if the - * LB + /* what's being sent to the LB server. Note that its value may vary if the LB * server indicates a redirect. */ grpc_byte_buffer *request_payload; @@ -1103,8 +1097,7 @@ static void res_recv_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { * it'll just create the first RR policy instance */ rr_handover(exec_ctx, lb_client->glb_policy, error); } else { - /* unref the RR policy, eventually leading to its substitution with - * a + /* unref the RR policy, eventually leading to its substitution with a * new one constructed from the received serverlist (see * glb_rr_connectivity_changed) */ GRPC_LB_POLICY_UNREF(exec_ctx, lb_client->glb_policy->rr_policy, @@ -1170,8 +1163,7 @@ static void srv_status_rcvd_cb(grpc_exec_ctx *exec_ctx, void *arg, lb_client->status_details_capacity); } /* TODO(dgq): deal with stream termination properly (fire up another one? - * fail - * the original call?) */ + * fail the original call?) */ } /* Code wiring the policy with the rest of the core */ diff --git a/src/core/ext/lb_policy/round_robin/round_robin.c b/src/core/ext/lb_policy/round_robin/round_robin.c index 1351ff0277f..f87d8f080d1 100644 --- a/src/core/ext/lb_policy/round_robin/round_robin.c +++ b/src/core/ext/lb_policy/round_robin/round_robin.c @@ -132,7 +132,7 @@ struct round_robin_lb_policy { size_t num_addresses; /** user data, one per incoming address. This pointer is borrowed and opaque. * It'll be returned as-is in successful picks. */ - void **user_data; + void **user_data_pointers; /** all our subchannels */ size_t num_subchannels; @@ -281,7 +281,7 @@ static void rr_destroy(grpc_exec_ctx *exec_ctx, grpc_lb_policy *pol) { elem = tmp; } - gpr_free(p->user_data); + gpr_free(p->user_data_pointers); gpr_free(p); } @@ -613,8 +613,8 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx, p->num_addresses = args->num_addresses; p->subchannels = gpr_malloc(sizeof(subchannel_data) * p->num_addresses); memset(p->subchannels, 0, sizeof(*p->subchannels) * p->num_addresses); - p->user_data = gpr_malloc(sizeof(void *) * p->num_addresses); - memset(p->user_data, 0, sizeof(void *) * p->num_addresses); + p->user_data_pointers = gpr_malloc(sizeof(void *) * p->num_addresses); + memset(p->user_data_pointers, 0, sizeof(void *) * p->num_addresses); grpc_subchannel_args sc_args; size_t subchannel_idx = 0; @@ -623,7 +623,7 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx, sc_args.addr = (struct sockaddr *)args->addresses[i].resolved_address->addr; sc_args.addr_len = args->addresses[i].resolved_address->len; - p->user_data[i] = args->addresses[i].user_data; + p->user_data_pointers[i] = args->addresses[i].user_data; grpc_subchannel *subchannel = grpc_client_channel_factory_create_subchannel( exec_ctx, args->client_channel_factory, &sc_args); @@ -635,7 +635,7 @@ static grpc_lb_policy *round_robin_create(grpc_exec_ctx *exec_ctx, sd->policy = p; sd->index = subchannel_idx; sd->subchannel = subchannel; - sd->user_data = p->user_data[i]; + sd->user_data = p->user_data_pointers[i]; ++subchannel_idx; grpc_closure_init(&sd->connectivity_changed_closure, rr_connectivity_changed, sd); From 6eb48f10732e456301d841d79dd30cdd15995572 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Thu, 15 Sep 2016 08:41:44 -0700 Subject: [PATCH 24/25] Fix comment --- src/core/ext/lb_policy/round_robin/round_robin.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/core/ext/lb_policy/round_robin/round_robin.c b/src/core/ext/lb_policy/round_robin/round_robin.c index f87d8f080d1..4434165ff95 100644 --- a/src/core/ext/lb_policy/round_robin/round_robin.c +++ b/src/core/ext/lb_policy/round_robin/round_robin.c @@ -130,8 +130,9 @@ struct round_robin_lb_policy { /** total number of addresses received at creation time */ size_t num_addresses; - /** user data, one per incoming address. This pointer is borrowed and opaque. - * It'll be returned as-is in successful picks. */ + /** array holding the borrowed and opaque pointers to incoming user data, one + * per incoming address. These individual pointers will be returned as-is in + * successful picks. */ void **user_data_pointers; /** all our subchannels */ From 5ebb7af9ef074e69fb0173e938fddaa1b6b88913 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Thu, 15 Sep 2016 10:02:16 -0700 Subject: [PATCH 25/25] Don't assert on use of unsupported user_data for LB policies --- src/core/ext/lb_policy/grpclb/grpclb.c | 12 ++++++++---- src/core/ext/lb_policy/pick_first/pick_first.c | 6 ++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/core/ext/lb_policy/grpclb/grpclb.c b/src/core/ext/lb_policy/grpclb/grpclb.c index 3ce3910b30e..cf326583331 100644 --- a/src/core/ext/lb_policy/grpclb/grpclb.c +++ b/src/core/ext/lb_policy/grpclb/grpclb.c @@ -573,8 +573,10 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, return NULL; } - /* this LB policy doesn't support \a user_data */ - GPR_ASSERT(args->addresses[0].user_data == NULL); + if (args->addresses[0].user_data != NULL) { + gpr_log(GPR_ERROR, + "This LB policy doesn't support user data. It will be ignored"); + } /* construct a target from the addresses in args, given in the form * ipvX://ip1:port1,ip2:port2,... @@ -583,8 +585,10 @@ static grpc_lb_policy *glb_create(grpc_exec_ctx *exec_ctx, addr_strs[0] = grpc_sockaddr_to_uri( (const struct sockaddr *)&args->addresses[0].resolved_address->addr); for (size_t i = 1; i < args->num_addresses; i++) { - /* this LB policy doesn't support \a user_data */ - GPR_ASSERT(args->addresses[i].user_data == NULL); + if (args->addresses[i].user_data != NULL) { + gpr_log(GPR_ERROR, + "This LB policy doesn't support user data. It will be ignored"); + } GPR_ASSERT( grpc_sockaddr_to_string( diff --git a/src/core/ext/lb_policy/pick_first/pick_first.c b/src/core/ext/lb_policy/pick_first/pick_first.c index d907ddd5d1d..9513078dce6 100644 --- a/src/core/ext/lb_policy/pick_first/pick_first.c +++ b/src/core/ext/lb_policy/pick_first/pick_first.c @@ -451,8 +451,10 @@ static grpc_lb_policy *create_pick_first(grpc_exec_ctx *exec_ctx, grpc_subchannel_args sc_args; size_t subchannel_idx = 0; for (size_t i = 0; i < args->num_addresses; i++) { - /* this LB policy doesn't support \a user_data */ - GPR_ASSERT(args->addresses[i].user_data == NULL); + if (args->addresses[i].user_data != NULL) { + gpr_log(GPR_ERROR, + "This LB policy doesn't support user data. It will be ignored"); + } memset(&sc_args, 0, sizeof(grpc_subchannel_args)); sc_args.addr =