From 6227144964d2893e2da6f09d7685ddcf60d82a51 Mon Sep 17 00:00:00 2001 From: Ashitha Santhosh Date: Tue, 10 Mar 2020 14:29:49 -0700 Subject: [PATCH 01/73] Implemented Frame Size Negotiation in ALTS for gRPC C++. --- .../alts/alts_security_connector.cc | 25 +++++++++--- .../alts/handshaker/alts_handshaker_client.cc | 9 ++++- .../alts/handshaker/alts_handshaker_client.h | 12 ++++-- .../alts/handshaker/alts_tsi_handshaker.cc | 39 ++++++++++++++++++- .../tsi/alts/handshaker/alts_tsi_handshaker.h | 5 ++- 5 files changed, 77 insertions(+), 13 deletions(-) diff --git a/src/core/lib/security/security_connector/alts/alts_security_connector.cc b/src/core/lib/security/security_connector/alts/alts_security_connector.cc index 1274edb6e6e..6bfe6ea74c3 100644 --- a/src/core/lib/security/security_connector/alts/alts_security_connector.cc +++ b/src/core/lib/security/security_connector/alts/alts_security_connector.cc @@ -82,10 +82,17 @@ class grpc_alts_channel_security_connector final tsi_handshaker* handshaker = nullptr; const grpc_alts_credentials* creds = static_cast(channel_creds()); - GPR_ASSERT(alts_tsi_handshaker_create(creds->options(), target_name_, - creds->handshaker_service_url(), true, - interested_parties, - &handshaker) == TSI_OK); + size_t user_specified_max_frame_size = 0; + const grpc_arg* arg = + grpc_channel_args_find(args, GRPC_ARG_TSI_MAX_FRAME_SIZE); + if (arg != nullptr && arg->type == GRPC_ARG_INTEGER) { + user_specified_max_frame_size = grpc_channel_arg_get_integer( + arg, {0, 0, std::numeric_limits::max()}); + } + GPR_ASSERT(alts_tsi_handshaker_create( + creds->options(), target_name_, + creds->handshaker_service_url(), true, interested_parties, + &handshaker, user_specified_max_frame_size) == TSI_OK); handshake_manager->Add( grpc_core::SecurityHandshakerCreate(handshaker, this, args)); } @@ -140,9 +147,17 @@ class grpc_alts_server_security_connector final tsi_handshaker* handshaker = nullptr; const grpc_alts_server_credentials* creds = static_cast(server_creds()); + size_t user_specified_max_frame_size = 0; + const grpc_arg* arg = + grpc_channel_args_find(args, GRPC_ARG_TSI_MAX_FRAME_SIZE); + if (arg != nullptr && arg->type == GRPC_ARG_INTEGER) { + user_specified_max_frame_size = grpc_channel_arg_get_integer( + arg, {0, 0, std::numeric_limits::max()}); + } GPR_ASSERT(alts_tsi_handshaker_create( creds->options(), nullptr, creds->handshaker_service_url(), - false, interested_parties, &handshaker) == TSI_OK); + false, interested_parties, &handshaker, + user_specified_max_frame_size) == TSI_OK); handshake_manager->Add( grpc_core::SecurityHandshakerCreate(handshaker, this, args)); } diff --git a/src/core/tsi/alts/handshaker/alts_handshaker_client.cc b/src/core/tsi/alts/handshaker/alts_handshaker_client.cc index 2592763e5a2..5c5a407f54f 100644 --- a/src/core/tsi/alts/handshaker/alts_handshaker_client.cc +++ b/src/core/tsi/alts/handshaker/alts_handshaker_client.cc @@ -102,6 +102,8 @@ typedef struct alts_grpc_handshaker_client { bool receive_status_finished; /* if non-null, contains arguments to complete a TSI next callback. */ recv_message_result* pending_recv_message_result; + /* Maximum frame size used by frame protector. */ + size_t max_frame_size; } alts_grpc_handshaker_client; static void handshaker_client_send_buffer_destroy( @@ -506,6 +508,8 @@ static grpc_byte_buffer* get_serialized_start_client( upb_strview_makez(ptr->data)); ptr = ptr->next; } + grpc_gcp_StartClientHandshakeReq_set_max_frame_size(start_client, + client->max_frame_size); return get_serialized_handshaker_req(req, arena.ptr()); } @@ -565,6 +569,8 @@ static grpc_byte_buffer* get_serialized_start_server( arena.ptr()); grpc_gcp_RpcProtocolVersions_assign_from_struct( server_version, arena.ptr(), &client->options->rpc_versions); + grpc_gcp_StartServerHandshakeReq_set_max_frame_size(start_server, + client->max_frame_size); return get_serialized_handshaker_req(req, arena.ptr()); } @@ -674,7 +680,7 @@ alts_handshaker_client* alts_grpc_handshaker_client_create( grpc_alts_credentials_options* options, const grpc_slice& target_name, grpc_iomgr_cb_func grpc_cb, tsi_handshaker_on_next_done_cb cb, void* user_data, alts_handshaker_client_vtable* vtable_for_testing, - bool is_client) { + bool is_client, size_t max_frame_size) { if (channel == nullptr || handshaker_service_url == nullptr) { gpr_log(GPR_ERROR, "Invalid arguments to alts_handshaker_client_create()"); return nullptr; @@ -694,6 +700,7 @@ alts_handshaker_client* alts_grpc_handshaker_client_create( client->recv_bytes = grpc_empty_slice(); grpc_metadata_array_init(&client->recv_initial_metadata); client->is_client = is_client; + client->max_frame_size = max_frame_size; client->buffer_size = TSI_ALTS_INITIAL_BUFFER_SIZE; client->buffer = static_cast(gpr_zalloc(client->buffer_size)); grpc_slice slice = grpc_slice_from_copied_string(handshaker_service_url); diff --git a/src/core/tsi/alts/handshaker/alts_handshaker_client.h b/src/core/tsi/alts/handshaker/alts_handshaker_client.h index 319a23c88c7..b3af548b405 100644 --- a/src/core/tsi/alts/handshaker/alts_handshaker_client.h +++ b/src/core/tsi/alts/handshaker/alts_handshaker_client.h @@ -117,7 +117,7 @@ void alts_handshaker_client_destroy(alts_handshaker_client* client); * This method creates an ALTS handshaker client. * * - handshaker: ALTS TSI handshaker to which the created handshaker client - * belongs to. + * belongs to. * - channel: grpc channel to ALTS handshaker service. * - handshaker_service_url: address of ALTS handshaker service in the format of * "host:port". @@ -132,8 +132,12 @@ void alts_handshaker_client_destroy(alts_handshaker_client* client); * - vtable_for_testing: ALTS handshaker client vtable instance used for * testing purpose. * - is_client: a boolean value indicating if the created handshaker client is - * used at the client (is_client = true) or server (is_client = false) side. It - * returns the created ALTS handshaker client on success, and NULL on failure. + * used at the client (is_client = true) or server (is_client = false) side. + * - max_frame_size: Maximum frame size used by frame protector (User specified + * maximum frame size if present or default max frame size). + * + * It returns the created ALTS handshaker client on success, and NULL + * on failure. */ alts_handshaker_client* alts_grpc_handshaker_client_create( alts_tsi_handshaker* handshaker, grpc_channel* channel, @@ -141,7 +145,7 @@ alts_handshaker_client* alts_grpc_handshaker_client_create( grpc_alts_credentials_options* options, const grpc_slice& target_name, grpc_iomgr_cb_func grpc_cb, tsi_handshaker_on_next_done_cb cb, void* user_data, alts_handshaker_client_vtable* vtable_for_testing, - bool is_client); + bool is_client, size_t max_frame_size); /** * This method handles handshaker response returned from ALTS handshaker diff --git a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc index 0c700306d8f..c9daaecbce6 100644 --- a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc +++ b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc @@ -41,6 +41,11 @@ #include "src/core/tsi/alts/handshaker/alts_tsi_utils.h" #include "src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.h" +// Frame size negotiation extends send frame size range to +// [kMinFrameSize, kMaxFrameSize] +constexpr size_t kMinFrameSize = 16 * 1024; +constexpr size_t kMaxFrameSize = 128 * 1024; + /* Main struct for ALTS TSI handshaker. */ struct alts_tsi_handshaker { tsi_handshaker base; @@ -63,6 +68,8 @@ struct alts_tsi_handshaker { // shutdown effectively follows base.handshake_shutdown, // but is synchronized by the mutex of this object. bool shutdown; + // Maximum frame size used by frame protector. + size_t max_frame_size; }; /* Main struct for ALTS TSI handshaker result. */ @@ -75,6 +82,8 @@ typedef struct alts_tsi_handshaker_result { grpc_slice rpc_versions; bool is_client; grpc_slice serialized_context; + // Peer's maximum frame size. + size_t max_frame_size; } alts_tsi_handshaker_result; static tsi_result handshaker_result_extract_peer( @@ -156,6 +165,26 @@ static tsi_result handshaker_result_create_zero_copy_grpc_protector( alts_tsi_handshaker_result* result = reinterpret_cast( const_cast(self)); + + // In case the peer does not send max frame size (e.g. peer is gRPC Go or + // peer uses an old binary), the negotiated frame size is set to + // kMinFrameSize (ignoring max_output_protected_frame_size value if + // present). Otherwise, it is based on peer and user specified max frame + // size (if present). + size_t max_frame_size = kMinFrameSize; + if (result->max_frame_size) { + size_t peer_max_frame_size = result->max_frame_size; + max_frame_size = std::min(peer_max_frame_size, + max_output_protected_frame_size == nullptr + ? kMaxFrameSize + : *max_output_protected_frame_size); + max_frame_size = std::max(max_frame_size, kMinFrameSize); + } + max_output_protected_frame_size = &max_frame_size; + gpr_log(GPR_DEBUG, + "After Frame Size Negotiation, maximum frame size used by frame " + "protector equals %zu", + *max_output_protected_frame_size); tsi_result ok = alts_zero_copy_grpc_protector_create( reinterpret_cast(result->key_data), kAltsAes128GcmRekeyKeyLength, /*is_rekey=*/true, result->is_client, @@ -288,6 +317,7 @@ tsi_result alts_tsi_handshaker_result_create(grpc_gcp_HandshakerResp* resp, static_cast(gpr_zalloc(peer_service_account.size + 1)); memcpy(result->peer_identity, peer_service_account.data, peer_service_account.size); + result->max_frame_size = grpc_gcp_HandshakerResult_max_frame_size(hresult); upb::Arena rpc_versions_arena; bool serialized = grpc_gcp_rpc_protocol_versions_encode( peer_rpc_version, rpc_versions_arena.ptr(), &result->rpc_versions); @@ -374,7 +404,8 @@ static tsi_result alts_tsi_handshaker_continue_handshaker_next( handshaker, channel, handshaker->handshaker_service_url, handshaker->interested_parties, handshaker->options, handshaker->target_name, grpc_cb, cb, user_data, - handshaker->client_vtable_for_testing, handshaker->is_client); + handshaker->client_vtable_for_testing, handshaker->is_client, + handshaker->max_frame_size); if (client == nullptr) { gpr_log(GPR_ERROR, "Failed to create ALTS handshaker client"); return TSI_FAILED_PRECONDITION; @@ -570,7 +601,8 @@ bool alts_tsi_handshaker_has_shutdown(alts_tsi_handshaker* handshaker) { tsi_result alts_tsi_handshaker_create( const grpc_alts_credentials_options* options, const char* target_name, const char* handshaker_service_url, bool is_client, - grpc_pollset_set* interested_parties, tsi_handshaker** self) { + grpc_pollset_set* interested_parties, tsi_handshaker** self, + size_t user_specified_max_frame_size) { if (handshaker_service_url == nullptr || self == nullptr || options == nullptr || (is_client && target_name == nullptr)) { gpr_log(GPR_ERROR, "Invalid arguments to alts_tsi_handshaker_create()"); @@ -590,6 +622,9 @@ tsi_result alts_tsi_handshaker_create( handshaker->has_created_handshaker_client = false; handshaker->handshaker_service_url = gpr_strdup(handshaker_service_url); handshaker->options = grpc_alts_credentials_options_copy(options); + handshaker->max_frame_size = user_specified_max_frame_size != 0 + ? user_specified_max_frame_size + : kMaxFrameSize; handshaker->base.vtable = handshaker->use_dedicated_cq ? &handshaker_vtable_dedicated : &handshaker_vtable; diff --git a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h index 5bace9affa8..ec3f167af5e 100644 --- a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h +++ b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h @@ -54,6 +54,8 @@ typedef struct alts_tsi_handshaker alts_tsi_handshaker; * - interested_parties: set of pollsets interested in this connection. * - self: address of ALTS TSI handshaker instance to be returned from the * method. + * - user_specified_max_frame_size: Determines the maximum frame size used by + * frame protector that is specified via user. If unspecified, the value is 0. * * It returns TSI_OK on success and an error status code on failure. Note that * if interested_parties is nullptr, a dedicated TSI thread will be created and @@ -62,7 +64,8 @@ typedef struct alts_tsi_handshaker alts_tsi_handshaker; tsi_result alts_tsi_handshaker_create( const grpc_alts_credentials_options* options, const char* target_name, const char* handshaker_service_url, bool is_client, - grpc_pollset_set* interested_parties, tsi_handshaker** self); + grpc_pollset_set* interested_parties, tsi_handshaker** self, + size_t user_specified_max_frame_size); /** * This method creates an ALTS TSI handshaker result instance. From 72112d034e40b95d3f015ec94cf189cac0f56983 Mon Sep 17 00:00:00 2001 From: Ashitha Santhosh Date: Tue, 17 Mar 2020 23:37:20 -0700 Subject: [PATCH 02/73] Added tests. --- .../alts/handshaker/alts_handshaker_client.h | 2 +- .../tsi/alts/handshaker/alts_tsi_handshaker.h | 2 +- .../handshaker/alts_handshaker_client_test.cc | 20 ++++++++---- .../handshaker/alts_tsi_handshaker_test.cc | 32 ++++++++++++++++++- 4 files changed, 46 insertions(+), 10 deletions(-) diff --git a/src/core/tsi/alts/handshaker/alts_handshaker_client.h b/src/core/tsi/alts/handshaker/alts_handshaker_client.h index b3af548b405..d8669da01cb 100644 --- a/src/core/tsi/alts/handshaker/alts_handshaker_client.h +++ b/src/core/tsi/alts/handshaker/alts_handshaker_client.h @@ -134,7 +134,7 @@ void alts_handshaker_client_destroy(alts_handshaker_client* client); * - is_client: a boolean value indicating if the created handshaker client is * used at the client (is_client = true) or server (is_client = false) side. * - max_frame_size: Maximum frame size used by frame protector (User specified - * maximum frame size if present or default max frame size). + * maximum frame size if present or default max frame size). * * It returns the created ALTS handshaker client on success, and NULL * on failure. diff --git a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h index ec3f167af5e..0c9a5732c48 100644 --- a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h +++ b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h @@ -55,7 +55,7 @@ typedef struct alts_tsi_handshaker alts_tsi_handshaker; * - self: address of ALTS TSI handshaker instance to be returned from the * method. * - user_specified_max_frame_size: Determines the maximum frame size used by - * frame protector that is specified via user. If unspecified, the value is 0. + * frame protector that is specified via user. If unspecified, the value is 0. * * It returns TSI_OK on success and an error status code on failure. Note that * if interested_parties is nullptr, a dedicated TSI thread will be created and diff --git a/test/core/tsi/alts/handshaker/alts_handshaker_client_test.cc b/test/core/tsi/alts/handshaker/alts_handshaker_client_test.cc index 0e1ab006728..e28bda1d059 100644 --- a/test/core/tsi/alts/handshaker/alts_handshaker_client_test.cc +++ b/test/core/tsi/alts/handshaker/alts_handshaker_client_test.cc @@ -31,6 +31,7 @@ #define ALTS_HANDSHAKER_CLIENT_TEST_TARGET_NAME "bigtable.google.api.com" #define ALTS_HANDSHAKER_CLIENT_TEST_TARGET_SERVICE_ACCOUNT1 "A@google.com" #define ALTS_HANDSHAKER_CLIENT_TEST_TARGET_SERVICE_ACCOUNT2 "B@google.com" +#define ALTS_HANDSHAKER_CLIENT_TEST_MAX_FRAME_SIZE 64 * 1024 const size_t kHandshakerClientOpNum = 4; const size_t kMaxRpcVersionMajor = 3; @@ -155,8 +156,8 @@ static grpc_call_error check_must_not_be_called(grpc_call* /*call*/, /** * A mock grpc_caller used to check correct execution of client_start operation. * It checks if the client_start handshaker request is populated with correct - * handshake_security_protocol, application_protocol, and record_protocol, and - * op is correctly populated. + * handshake_security_protocol, application_protocol, record_protocol and + * max_frame_size, and op is correctly populated. */ static grpc_call_error check_client_start_success(grpc_call* /*call*/, const grpc_op* op, @@ -196,7 +197,8 @@ static grpc_call_error check_client_start_success(grpc_call* /*call*/, GPR_ASSERT(upb_strview_eql( grpc_gcp_StartClientHandshakeReq_target_name(client_start), upb_strview_makez(ALTS_HANDSHAKER_CLIENT_TEST_TARGET_NAME))); - + GPR_ASSERT(grpc_gcp_StartClientHandshakeReq_max_frame_size( + client_start, ALTS_HANDSHAKER_CLIENT_TEST_MAX_FRAME_SIZE)); GPR_ASSERT(validate_op(client, op, nops, true /* is_start */)); return GRPC_CALL_OK; } @@ -204,8 +206,8 @@ static grpc_call_error check_client_start_success(grpc_call* /*call*/, /** * A mock grpc_caller used to check correct execution of server_start operation. * It checks if the server_start handshaker request is populated with correct - * handshake_security_protocol, application_protocol, and record_protocol, and - * op is correctly populated. + * handshake_security_protocol, application_protocol, record_protocol and + * max_frame_size, and op is correctly populated. */ static grpc_call_error check_server_start_success(grpc_call* /*call*/, const grpc_op* op, @@ -245,6 +247,8 @@ static grpc_call_error check_server_start_success(grpc_call* /*call*/, upb_strview_makez(ALTS_RECORD_PROTOCOL))); validate_rpc_protocol_versions( grpc_gcp_StartServerHandshakeReq_rpc_versions(server_start)); + GPR_ASSERT(grpc_gcp_StartServerHandshakeReq_max_frame_size( + server_start, ALTS_HANDSHAKER_CLIENT_TEST_MAX_FRAME_SIZE)); GPR_ASSERT(validate_op(client, op, nops, true /* is_start */)); return GRPC_CALL_OK; } @@ -321,12 +325,14 @@ static alts_handshaker_client_test_config* create_config() { nullptr, config->channel, ALTS_HANDSHAKER_SERVICE_URL_FOR_TESTING, nullptr, server_options, grpc_slice_from_static_string(ALTS_HANDSHAKER_CLIENT_TEST_TARGET_NAME), - nullptr, nullptr, nullptr, nullptr, false); + nullptr, nullptr, nullptr, nullptr, false, + ALTS_HANDSHAKER_CLIENT_TEST_MAX_FRAME_SIZE); config->client = alts_grpc_handshaker_client_create( nullptr, config->channel, ALTS_HANDSHAKER_SERVICE_URL_FOR_TESTING, nullptr, client_options, grpc_slice_from_static_string(ALTS_HANDSHAKER_CLIENT_TEST_TARGET_NAME), - nullptr, nullptr, nullptr, nullptr, true); + nullptr, nullptr, nullptr, nullptr, true, + ALTS_HANDSHAKER_CLIENT_TEST_MAX_FRAME_SIZE); GPR_ASSERT(config->client != nullptr); GPR_ASSERT(config->server != nullptr); grpc_alts_credentials_options_destroy(client_options); diff --git a/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc b/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc index 5dd76d82fdc..bbbf2ae11b8 100644 --- a/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc +++ b/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc @@ -49,6 +49,9 @@ #define ALTS_TSI_HANDSHAKER_TEST_APPLICATION_PROTOCOL \ "test application protocol" #define ALTS_TSI_HANDSHAKER_TEST_RECORD_PROTOCOL "test record protocol" +#define ALTS_TSI_HANDSHAKER_TEST_MAX_FRAME_SIZE 256 * 1024 +#define ALTS_TSI_HANDSHAKER_TEST_DEFAULT_MAX_FRAME_SIZE 128 * 1024 +#define ALTS_TSI_HANDSHAKER_TEST_MIN_MAX_FRAME_SIZE 16 * 1024 using grpc_core::internal::alts_handshaker_client_check_fields_for_testing; using grpc_core::internal::alts_handshaker_client_get_handshaker_for_testing; @@ -164,6 +167,8 @@ static grpc_byte_buffer* generate_handshaker_response( upb_strview_makez(ALTS_TSI_HANDSHAKER_TEST_APPLICATION_PROTOCOL)); grpc_gcp_HandshakerResult_set_record_protocol( result, upb_strview_makez(ALTS_TSI_HANDSHAKER_TEST_RECORD_PROTOCOL)); + grpc_gcp_HandshakerResult_set_max_frame_size( + result, ALTS_TSI_HANDSHAKER_TEST_MAX_FRAME_SIZE); break; case SERVER_NEXT: grpc_gcp_HandshakerResp_set_bytes_consumed( @@ -283,6 +288,17 @@ static void on_client_next_success_cb(tsi_result status, void* user_data, GPR_ASSERT(memcmp(bytes_to_send, ALTS_TSI_HANDSHAKER_TEST_OUT_FRAME, bytes_to_send_size) == 0); GPR_ASSERT(result != nullptr); + // Validate max frame size value after Frame Size Negotiation. Here peer max + // frame size is greater than default value, and user specified max frame size + // is absent. + tsi_zero_copy_grpc_protector* zero_copy_protector; + GPR_ASSERT(tsi_handshaker_result_create_zero_copy_frame_protector( + result, nullptr, &zero_copy_protector) == TSI_OK); + size_t actual_max_frame_size; + tsi_zero_copy_grpc_protector_max_frame_size(zero_copy_protector, + &actual_max_frame_size); + GPR_ASSERT(actual_max_frame_size, + ALTS_TSI_HANDSHAKER_TEST_DEFAULT_MAX_FRAME_SIZE); /* Validate peer identity. */ tsi_peer peer; GPR_ASSERT(tsi_handshaker_result_extract_peer(result, &peer) == TSI_OK); @@ -343,6 +359,20 @@ static void on_server_next_success_cb(tsi_result status, void* user_data, GPR_ASSERT(bytes_to_send_size == 0); GPR_ASSERT(bytes_to_send == nullptr); GPR_ASSERT(result != nullptr); + // Validate max frame size value after Frame Size Negotiation. The negotiated + // frame size value equals minimum send frame size, due to the absence of peer + // max frame size. + tsi_zero_copy_grpc_protector* zero_copy_protector; + size_t user_specified_max_frame_size = + ALTS_TSI_HANDSHAKER_TEST_MAX_FRAME_SIZE; + GPR_ASSERT(tsi_handshaker_result_create_zero_copy_frame_protector( + result, &user_specified_max_frame_size, + &zero_copy_protector) == TSI_OK); + size_t actual_max_frame_size; + tsi_zero_copy_grpc_protector_max_frame_size(zero_copy_protector, + &actual_max_frame_size); + GPR_ASSERT(actual_max_frame_size, + ALTS_TSI_HANDSHAKER_TEST_MIN_MAX_FRAME_SIZE); /* Validate peer identity. */ tsi_peer peer; GPR_ASSERT(tsi_handshaker_result_extract_peer(result, &peer) == TSI_OK); @@ -478,7 +508,7 @@ static tsi_handshaker* create_test_handshaker(bool is_client) { grpc_alts_credentials_client_options_create(); alts_tsi_handshaker_create(options, "target_name", ALTS_HANDSHAKER_SERVICE_URL_FOR_TESTING, is_client, - nullptr, &handshaker); + nullptr, &handshaker, 0); alts_tsi_handshaker* alts_handshaker = reinterpret_cast(handshaker); alts_tsi_handshaker_set_client_vtable_for_testing(alts_handshaker, &vtable); From 6fbf67ed9c1589b013fd0439ea5bf3e26b6a4c7c Mon Sep 17 00:00:00 2001 From: Ashitha Santhosh Date: Wed, 18 Mar 2020 14:11:23 -0700 Subject: [PATCH 03/73] Modified test variable name --- test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc b/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc index bbbf2ae11b8..a58e36df62f 100644 --- a/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc +++ b/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc @@ -49,9 +49,9 @@ #define ALTS_TSI_HANDSHAKER_TEST_APPLICATION_PROTOCOL \ "test application protocol" #define ALTS_TSI_HANDSHAKER_TEST_RECORD_PROTOCOL "test record protocol" +#define ALTS_TSI_HANDSHAKER_TEST_MIN_FRAME_SIZE 16 * 1024 #define ALTS_TSI_HANDSHAKER_TEST_MAX_FRAME_SIZE 256 * 1024 #define ALTS_TSI_HANDSHAKER_TEST_DEFAULT_MAX_FRAME_SIZE 128 * 1024 -#define ALTS_TSI_HANDSHAKER_TEST_MIN_MAX_FRAME_SIZE 16 * 1024 using grpc_core::internal::alts_handshaker_client_check_fields_for_testing; using grpc_core::internal::alts_handshaker_client_get_handshaker_for_testing; @@ -371,8 +371,7 @@ static void on_server_next_success_cb(tsi_result status, void* user_data, size_t actual_max_frame_size; tsi_zero_copy_grpc_protector_max_frame_size(zero_copy_protector, &actual_max_frame_size); - GPR_ASSERT(actual_max_frame_size, - ALTS_TSI_HANDSHAKER_TEST_MIN_MAX_FRAME_SIZE); + GPR_ASSERT(actual_max_frame_size, ALTS_TSI_HANDSHAKER_TEST_MIN_FRAME_SIZE); /* Validate peer identity. */ tsi_peer peer; GPR_ASSERT(tsi_handshaker_result_extract_peer(result, &peer) == TSI_OK); From 53fc4e5c133104c019b2511e321e9b64286a0d20 Mon Sep 17 00:00:00 2001 From: Ashitha Santhosh Date: Sun, 22 Mar 2020 01:10:43 -0700 Subject: [PATCH 04/73] Fixed error --- .../tsi/alts/handshaker/alts_handshaker_client_test.cc | 8 ++++---- .../core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc | 9 +++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/test/core/tsi/alts/handshaker/alts_handshaker_client_test.cc b/test/core/tsi/alts/handshaker/alts_handshaker_client_test.cc index e28bda1d059..5f9a4b2d745 100644 --- a/test/core/tsi/alts/handshaker/alts_handshaker_client_test.cc +++ b/test/core/tsi/alts/handshaker/alts_handshaker_client_test.cc @@ -197,8 +197,8 @@ static grpc_call_error check_client_start_success(grpc_call* /*call*/, GPR_ASSERT(upb_strview_eql( grpc_gcp_StartClientHandshakeReq_target_name(client_start), upb_strview_makez(ALTS_HANDSHAKER_CLIENT_TEST_TARGET_NAME))); - GPR_ASSERT(grpc_gcp_StartClientHandshakeReq_max_frame_size( - client_start, ALTS_HANDSHAKER_CLIENT_TEST_MAX_FRAME_SIZE)); + GPR_ASSERT(grpc_gcp_StartClientHandshakeReq_max_frame_size(client_start) == + ALTS_HANDSHAKER_CLIENT_TEST_MAX_FRAME_SIZE); GPR_ASSERT(validate_op(client, op, nops, true /* is_start */)); return GRPC_CALL_OK; } @@ -247,8 +247,8 @@ static grpc_call_error check_server_start_success(grpc_call* /*call*/, upb_strview_makez(ALTS_RECORD_PROTOCOL))); validate_rpc_protocol_versions( grpc_gcp_StartServerHandshakeReq_rpc_versions(server_start)); - GPR_ASSERT(grpc_gcp_StartServerHandshakeReq_max_frame_size( - server_start, ALTS_HANDSHAKER_CLIENT_TEST_MAX_FRAME_SIZE)); + GPR_ASSERT(grpc_gcp_StartServerHandshakeReq_max_frame_size(server_start) == + ALTS_HANDSHAKER_CLIENT_TEST_MAX_FRAME_SIZE); GPR_ASSERT(validate_op(client, op, nops, true /* is_start */)); return GRPC_CALL_OK; } diff --git a/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc b/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc index a58e36df62f..0abab1f8ae2 100644 --- a/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc +++ b/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc @@ -27,6 +27,7 @@ #include "src/core/tsi/alts/handshaker/alts_shared_resource.h" #include "src/core/tsi/alts/handshaker/alts_tsi_handshaker.h" #include "src/core/tsi/alts/handshaker/alts_tsi_handshaker_private.h" +#include "src/core/tsi/transport_security_grpc.h" #include "src/proto/grpc/gcp/altscontext.upb.h" #include "test/core/tsi/alts/handshaker/alts_handshaker_service_api_test_lib.h" #include "test/core/util/test_config.h" @@ -291,13 +292,13 @@ static void on_client_next_success_cb(tsi_result status, void* user_data, // Validate max frame size value after Frame Size Negotiation. Here peer max // frame size is greater than default value, and user specified max frame size // is absent. - tsi_zero_copy_grpc_protector* zero_copy_protector; + tsi_zero_copy_grpc_protector* zero_copy_protector = nullptr; GPR_ASSERT(tsi_handshaker_result_create_zero_copy_frame_protector( result, nullptr, &zero_copy_protector) == TSI_OK); size_t actual_max_frame_size; tsi_zero_copy_grpc_protector_max_frame_size(zero_copy_protector, &actual_max_frame_size); - GPR_ASSERT(actual_max_frame_size, + GPR_ASSERT(actual_max_frame_size == ALTS_TSI_HANDSHAKER_TEST_DEFAULT_MAX_FRAME_SIZE); /* Validate peer identity. */ tsi_peer peer; @@ -362,7 +363,7 @@ static void on_server_next_success_cb(tsi_result status, void* user_data, // Validate max frame size value after Frame Size Negotiation. The negotiated // frame size value equals minimum send frame size, due to the absence of peer // max frame size. - tsi_zero_copy_grpc_protector* zero_copy_protector; + tsi_zero_copy_grpc_protector* zero_copy_protector = nullptr; size_t user_specified_max_frame_size = ALTS_TSI_HANDSHAKER_TEST_MAX_FRAME_SIZE; GPR_ASSERT(tsi_handshaker_result_create_zero_copy_frame_protector( @@ -371,7 +372,7 @@ static void on_server_next_success_cb(tsi_result status, void* user_data, size_t actual_max_frame_size; tsi_zero_copy_grpc_protector_max_frame_size(zero_copy_protector, &actual_max_frame_size); - GPR_ASSERT(actual_max_frame_size, ALTS_TSI_HANDSHAKER_TEST_MIN_FRAME_SIZE); + GPR_ASSERT(actual_max_frame_size == ALTS_TSI_HANDSHAKER_TEST_MIN_FRAME_SIZE); /* Validate peer identity. */ tsi_peer peer; GPR_ASSERT(tsi_handshaker_result_extract_peer(result, &peer) == TSI_OK); From 3651424ea4208624e43c46e443a1d3fecd2369c0 Mon Sep 17 00:00:00 2001 From: Ashitha Santhosh Date: Wed, 25 Mar 2020 03:01:07 -0700 Subject: [PATCH 05/73] Moved k[Min/Max]FrameSize variables to header file. --- src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc | 6 ++---- src/core/tsi/alts/handshaker/alts_tsi_handshaker.h | 5 +++++ test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc | 7 ++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc index c9daaecbce6..b2687825d17 100644 --- a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc +++ b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc @@ -41,10 +41,8 @@ #include "src/core/tsi/alts/handshaker/alts_tsi_utils.h" #include "src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.h" -// Frame size negotiation extends send frame size range to -// [kMinFrameSize, kMaxFrameSize] -constexpr size_t kMinFrameSize = 16 * 1024; -constexpr size_t kMaxFrameSize = 128 * 1024; +const size_t kMinFrameSize = 16 * 1024; +const size_t kMaxFrameSize = 128 * 1024; /* Main struct for ALTS TSI handshaker. */ struct alts_tsi_handshaker { diff --git a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h index 0c9a5732c48..6959c563a5e 100644 --- a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h +++ b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h @@ -38,6 +38,11 @@ const size_t kTsiAltsNumOfPeerProperties = 5; +// Frame size negotiation extends send frame size range to +// [kMinFrameSize, kMaxFrameSize] +ABSL_CONST_INIT extern const size_t kMinFrameSize; +ABSL_CONST_INIT extern const size_t kMaxFrameSize; + typedef struct alts_tsi_handshaker alts_tsi_handshaker; /** diff --git a/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc b/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc index 0abab1f8ae2..1037313bbec 100644 --- a/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc +++ b/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc @@ -50,9 +50,7 @@ #define ALTS_TSI_HANDSHAKER_TEST_APPLICATION_PROTOCOL \ "test application protocol" #define ALTS_TSI_HANDSHAKER_TEST_RECORD_PROTOCOL "test record protocol" -#define ALTS_TSI_HANDSHAKER_TEST_MIN_FRAME_SIZE 16 * 1024 #define ALTS_TSI_HANDSHAKER_TEST_MAX_FRAME_SIZE 256 * 1024 -#define ALTS_TSI_HANDSHAKER_TEST_DEFAULT_MAX_FRAME_SIZE 128 * 1024 using grpc_core::internal::alts_handshaker_client_check_fields_for_testing; using grpc_core::internal::alts_handshaker_client_get_handshaker_for_testing; @@ -298,8 +296,7 @@ static void on_client_next_success_cb(tsi_result status, void* user_data, size_t actual_max_frame_size; tsi_zero_copy_grpc_protector_max_frame_size(zero_copy_protector, &actual_max_frame_size); - GPR_ASSERT(actual_max_frame_size == - ALTS_TSI_HANDSHAKER_TEST_DEFAULT_MAX_FRAME_SIZE); + GPR_ASSERT(actual_max_frame_size == kMaxFrameSize); /* Validate peer identity. */ tsi_peer peer; GPR_ASSERT(tsi_handshaker_result_extract_peer(result, &peer) == TSI_OK); @@ -372,7 +369,7 @@ static void on_server_next_success_cb(tsi_result status, void* user_data, size_t actual_max_frame_size; tsi_zero_copy_grpc_protector_max_frame_size(zero_copy_protector, &actual_max_frame_size); - GPR_ASSERT(actual_max_frame_size == ALTS_TSI_HANDSHAKER_TEST_MIN_FRAME_SIZE); + GPR_ASSERT(actual_max_frame_size == kMinFrameSize); /* Validate peer identity. */ tsi_peer peer; GPR_ASSERT(tsi_handshaker_result_extract_peer(result, &peer) == TSI_OK); From cf19300da25f07e4fc99070f53fc51f9bc986b60 Mon Sep 17 00:00:00 2001 From: Ashitha Santhosh Date: Wed, 25 Mar 2020 18:34:57 -0700 Subject: [PATCH 06/73] Corrected method name. --- test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc b/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc index 1037313bbec..7a866816c6e 100644 --- a/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc +++ b/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc @@ -291,7 +291,7 @@ static void on_client_next_success_cb(tsi_result status, void* user_data, // frame size is greater than default value, and user specified max frame size // is absent. tsi_zero_copy_grpc_protector* zero_copy_protector = nullptr; - GPR_ASSERT(tsi_handshaker_result_create_zero_copy_frame_protector( + GPR_ASSERT(tsi_handshaker_result_create_zero_copy_grpc_protector( result, nullptr, &zero_copy_protector) == TSI_OK); size_t actual_max_frame_size; tsi_zero_copy_grpc_protector_max_frame_size(zero_copy_protector, @@ -363,7 +363,7 @@ static void on_server_next_success_cb(tsi_result status, void* user_data, tsi_zero_copy_grpc_protector* zero_copy_protector = nullptr; size_t user_specified_max_frame_size = ALTS_TSI_HANDSHAKER_TEST_MAX_FRAME_SIZE; - GPR_ASSERT(tsi_handshaker_result_create_zero_copy_frame_protector( + GPR_ASSERT(tsi_handshaker_result_create_zero_copy_grpc_protector( result, &user_specified_max_frame_size, &zero_copy_protector) == TSI_OK); size_t actual_max_frame_size; From 3cc7aae8eec96839ed40f66f535bb2988a2eb275 Mon Sep 17 00:00:00 2001 From: Ashitha Santhosh Date: Thu, 26 Mar 2020 10:56:20 -0700 Subject: [PATCH 07/73] Fixed memory leak. --- test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc b/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc index 7a866816c6e..8f38640ef67 100644 --- a/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc +++ b/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc @@ -297,6 +297,7 @@ static void on_client_next_success_cb(tsi_result status, void* user_data, tsi_zero_copy_grpc_protector_max_frame_size(zero_copy_protector, &actual_max_frame_size); GPR_ASSERT(actual_max_frame_size == kMaxFrameSize); + tsi_zero_copy_grpc_protector_destroy(zero_copy_protector); /* Validate peer identity. */ tsi_peer peer; GPR_ASSERT(tsi_handshaker_result_extract_peer(result, &peer) == TSI_OK); @@ -370,6 +371,7 @@ static void on_server_next_success_cb(tsi_result status, void* user_data, tsi_zero_copy_grpc_protector_max_frame_size(zero_copy_protector, &actual_max_frame_size); GPR_ASSERT(actual_max_frame_size == kMinFrameSize); + tsi_zero_copy_grpc_protector_destroy(zero_copy_protector); /* Validate peer identity. */ tsi_peer peer; GPR_ASSERT(tsi_handshaker_result_extract_peer(result, &peer) == TSI_OK); From 0f47005ad61f6e932769c424f1ebe3be222d674d Mon Sep 17 00:00:00 2001 From: Nicolas Noble Date: Thu, 2 Apr 2020 12:47:01 -0700 Subject: [PATCH 08/73] Removing obsolete comment. --- tools/bazel.rc | 3 --- 1 file changed, 3 deletions(-) diff --git a/tools/bazel.rc b/tools/bazel.rc index 6dbcc273223..b27d9cb0675 100644 --- a/tools/bazel.rc +++ b/tools/bazel.rc @@ -1,7 +1,4 @@ # bazelrc file -# bazel >= 0.18 looks for %workspace%/.bazelrc (which redirects here) -# Older bazel versions look for %workspace%/tools/bazel.rc (this file) -# See https://github.com/bazelbuild/bazel/issues/6319 build --client_env=CC=clang build --copt=-DGRPC_BAZEL_BUILD From 23c32621d6336fa2540d1872e607b80d7594e059 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Thu, 2 Apr 2020 15:36:07 -0700 Subject: [PATCH 09/73] Fix all_the_cpythons. Add 3.5 to defaults --- .../Dockerfile.template | 6 ++++- third_party/abseil-cpp | 2 +- .../python_stretch_default_x64/Dockerfile | 23 +++++++++++++++++++ tools/run_tests/run_tests.py | 1 + 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/templates/tools/dockerfile/test/python_stretch_default_x64/Dockerfile.template b/templates/tools/dockerfile/test/python_stretch_default_x64/Dockerfile.template index ccb88e75302..bb4a9f29c01 100644 --- a/templates/tools/dockerfile/test/python_stretch_default_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/python_stretch_default_x64/Dockerfile.template @@ -16,7 +16,11 @@ <%include file="../../python_stretch.include"/> <%include file="../../compile_python_36.include"/> - + <%include file="../../compile_python_38.include"/> + + RUN apt-get update && apt-get install -y python3.5 python3.5-dev + RUN curl https://bootstrap.pypa.io/get-pip.py | python3.5 + RUN apt-get update && apt-get -t buster install -y python3.7 python3-all-dev RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7 diff --git a/third_party/abseil-cpp b/third_party/abseil-cpp index df3ea785d8c..b832dce8489 160000 --- a/third_party/abseil-cpp +++ b/third_party/abseil-cpp @@ -1 +1 @@ -Subproject commit df3ea785d8c30a9503321a3d35ee7d35808f190d +Subproject commit b832dce8489ef7b6231384909fd9b68d5a5ff2b7 diff --git a/tools/dockerfile/test/python_stretch_default_x64/Dockerfile b/tools/dockerfile/test/python_stretch_default_x64/Dockerfile index 9a1d6c09deb..713fd6fe230 100644 --- a/tools/dockerfile/test/python_stretch_default_x64/Dockerfile +++ b/tools/dockerfile/test/python_stretch_default_x64/Dockerfile @@ -84,6 +84,29 @@ RUN cd /tmp && \ RUN python3.6 -m ensurepip && \ python3.6 -m pip install coverage +#================= +# Compile CPython 3.8.0b4 from source + +RUN apt-get update && apt-get install -y zlib1g-dev libssl-dev +RUN apt-get update && apt-get install -y jq build-essential libffi-dev + +RUN cd /tmp && \ + wget -q https://www.python.org/ftp/python/3.8.0/Python-3.8.0b4.tgz && \ + tar xzvf Python-3.8.0b4.tgz && \ + cd Python-3.8.0b4 && \ + ./configure && \ + make install + +RUN cd /tmp && \ + echo "b8f4f897df967014ddb42033b90c3058 Python-3.8.0b4.tgz" > checksum.md5 && \ + md5sum -c checksum.md5 + +RUN python3.8 -m ensurepip && \ + python3.8 -m pip install coverage + + +RUN apt-get update && apt-get install -y python3.5 python3.5-dev +RUN curl https://bootstrap.pypa.io/get-pip.py | python3.5 RUN apt-get update && apt-get -t buster install -y python3.7 python3-all-dev RUN curl https://bootstrap.pypa.io/get-pip.py | python3.7 diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index a3d255110f0..8ed4c88f683 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -869,6 +869,7 @@ class PythonLanguage(object): else: return ( python27_config, + python35_config, python36_config, python37_config, ) From 4ce897eddc994509f54ece9a478831ea276a2871 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Thu, 2 Apr 2020 18:54:18 -0700 Subject: [PATCH 10/73] C++ize message compress filter --- .../message_compress_filter.cc | 458 ++++++++++-------- 1 file changed, 244 insertions(+), 214 deletions(-) diff --git a/src/core/ext/filters/http/message_compress/message_compress_filter.cc b/src/core/ext/filters/http/message_compress/message_compress_filter.cc index 27f0333bee4..fcee54ebfe3 100644 --- a/src/core/ext/filters/http/message_compress/message_compress_filter.cc +++ b/src/core/ext/filters/http/message_compress/message_compress_filter.cc @@ -40,94 +40,153 @@ #include "src/core/lib/surface/call.h" #include "src/core/lib/transport/static_metadata.h" -static void start_send_message_batch(void* arg, grpc_error* unused); -static void send_message_on_complete(void* arg, grpc_error* error); -static void on_send_message_next_done(void* arg, grpc_error* error); - namespace { -struct channel_data { +class ChannelData { + public: + ChannelData(grpc_channel_element_args* args) { + // Get the enabled and the default algorithms from channel args. + enabled_compression_algorithms_bitset_ = + grpc_channel_args_compression_algorithm_get_states(args->channel_args); + default_compression_algorithm_ = + grpc_channel_args_get_channel_default_compression_algorithm( + args->channel_args); + // Make sure the default is enabled. + if (!GPR_BITGET(enabled_compression_algorithms_bitset_, + default_compression_algorithm_)) { + const char* name; + GPR_ASSERT(grpc_compression_algorithm_name(default_compression_algorithm_, + &name) == 1); + gpr_log(GPR_ERROR, + "default compression algorithm %s not enabled: switching to none", + name); + default_compression_algorithm_ = GRPC_COMPRESS_NONE; + } + enabled_message_compression_algorithms_bitset_ = + grpc_compression_bitset_to_message_bitset( + enabled_compression_algorithms_bitset_); + enabled_stream_compression_algorithms_bitset_ = + grpc_compression_bitset_to_stream_bitset( + enabled_compression_algorithms_bitset_); + GPR_ASSERT(!args->is_last); + } + + grpc_compression_algorithm default_compression_algorithm() const { + return default_compression_algorithm_; + } + + uint32_t enabled_compression_algorithms_bitset() const { + return enabled_compression_algorithms_bitset_; + } + + uint32_t enabled_message_compression_algorithms_bitset() const { + return enabled_message_compression_algorithms_bitset_; + } + + uint32_t enabled_stream_compression_algorithms_bitset() const { + return enabled_stream_compression_algorithms_bitset_; + } + + private: /** The default, channel-level, compression algorithm */ - grpc_compression_algorithm default_compression_algorithm; + grpc_compression_algorithm default_compression_algorithm_; /** Bitset of enabled compression algorithms */ - uint32_t enabled_compression_algorithms_bitset; + uint32_t enabled_compression_algorithms_bitset_; /** Bitset of enabled message compression algorithms */ - uint32_t enabled_message_compression_algorithms_bitset; + uint32_t enabled_message_compression_algorithms_bitset_; /** Bitset of enabled stream compression algorithms */ - uint32_t enabled_stream_compression_algorithms_bitset; + uint32_t enabled_stream_compression_algorithms_bitset_; }; -struct call_data { - call_data(grpc_call_element* elem, const grpc_call_element_args& args) - : call_combiner(args.call_combiner) { - channel_data* channeld = static_cast(elem->channel_data); +class CallData { + public: + CallData(grpc_call_element* elem, const grpc_call_element_args& args) + : elem_(elem), call_combiner_(args.call_combiner) { + ChannelData* channeld = static_cast(elem_->channel_data); // The call's message compression algorithm is set to channel's default // setting. It can be overridden later by initial metadata. - if (GPR_LIKELY(GPR_BITGET(channeld->enabled_compression_algorithms_bitset, - channeld->default_compression_algorithm))) { - message_compression_algorithm = + if (GPR_LIKELY(GPR_BITGET(channeld->enabled_compression_algorithms_bitset(), + channeld->default_compression_algorithm()))) { + message_compression_algorithm_ = grpc_compression_algorithm_to_message_compression_algorithm( - channeld->default_compression_algorithm); + channeld->default_compression_algorithm()); } - GRPC_CLOSURE_INIT(&start_send_message_batch_in_call_combiner, - start_send_message_batch, elem, - grpc_schedule_on_exec_ctx); + GRPC_CLOSURE_INIT(&start_send_message_batch_in_call_combiner_, + StartSendMessageBatch, this, grpc_schedule_on_exec_ctx); } - ~call_data() { - if (state_initialized) { - grpc_slice_buffer_destroy_internal(&slices); + ~CallData() { + if (state_initialized_) { + grpc_slice_buffer_destroy_internal(&slices_); } - GRPC_ERROR_UNREF(cancel_error); + GRPC_ERROR_UNREF(cancel_error_); } - grpc_core::CallCombiner* call_combiner; - grpc_message_compression_algorithm message_compression_algorithm = + static void CompressStartTransportStreamOpBatch( + grpc_call_element* elem, grpc_transport_stream_op_batch* batch); + + bool SkipMessageCompression(); + void InitializeState(); + + grpc_error* ProcessSendInitialMetadata(grpc_metadata_batch* initial_metadata); + + // Methods for processing a send_message batch + static void StartSendMessageBatch(void* arg, grpc_error* unused); + static void OnSendMessageNextDone(void* arg, grpc_error* error); + grpc_error* PullSliceFromSendMessage(); + void ContinueReadingSendMessage(); + void FinishSendMessage(); + void SendMessageBatchContinue(); + static void FailSendMessageBatchInCallCombiner(void* arg, grpc_error* error); + + static void SendMessageOnComplete(void* arg, grpc_error* error); + + private: + grpc_call_element* elem_ = nullptr; + grpc_core::CallCombiner* call_combiner_ = nullptr; + grpc_message_compression_algorithm message_compression_algorithm_ = GRPC_MESSAGE_COMPRESS_NONE; - grpc_error* cancel_error = GRPC_ERROR_NONE; - grpc_transport_stream_op_batch* send_message_batch = nullptr; - bool seen_initial_metadata = false; + grpc_error* cancel_error_ = GRPC_ERROR_NONE; + grpc_transport_stream_op_batch* send_message_batch_ = nullptr; + bool seen_initial_metadata_ = false; /* Set to true, if the fields below are initialized. */ - bool state_initialized = false; - grpc_closure start_send_message_batch_in_call_combiner; + bool state_initialized_ = false; + grpc_closure start_send_message_batch_in_call_combiner_; /* The fields below are only initialized when we compress the payload. * Keep them at the bottom of the struct, so they don't pollute the * cache-lines. */ - grpc_linked_mdelem message_compression_algorithm_storage; - grpc_linked_mdelem stream_compression_algorithm_storage; - grpc_linked_mdelem accept_encoding_storage; - grpc_linked_mdelem accept_stream_encoding_storage; - grpc_slice_buffer slices; /**< Buffers up input slices to be compressed */ + grpc_linked_mdelem message_compression_algorithm_storage_; + grpc_linked_mdelem stream_compression_algorithm_storage_; + grpc_linked_mdelem accept_encoding_storage_; + grpc_linked_mdelem accept_stream_encoding_storage_; + grpc_slice_buffer slices_; /**< Buffers up input slices to be compressed */ grpc_core::ManualConstructor - replacement_stream; - grpc_closure* original_send_message_on_complete; - grpc_closure send_message_on_complete; - grpc_closure on_send_message_next_done; + replacement_stream_; + grpc_closure* original_send_message_on_complete_ = nullptr; + grpc_closure send_message_on_complete_; + grpc_closure on_send_message_next_done_; }; -} // namespace - // Returns true if we should skip message compression for the current message. -static bool skip_message_compression(grpc_call_element* elem) { - call_data* calld = static_cast(elem->call_data); +bool CallData::SkipMessageCompression() { // If the flags of this message indicate that it shouldn't be compressed, we // skip message compression. uint32_t flags = - calld->send_message_batch->payload->send_message.send_message->flags(); + send_message_batch_->payload->send_message.send_message->flags(); if (flags & (GRPC_WRITE_NO_COMPRESS | GRPC_WRITE_INTERNAL_COMPRESS)) { return true; } // If this call doesn't have any message compression algorithm set, skip // message compression. - return calld->message_compression_algorithm == GRPC_MESSAGE_COMPRESS_NONE; + return message_compression_algorithm_ == GRPC_MESSAGE_COMPRESS_NONE; } // Determines the compression algorithm from the initial metadata and the // channel's default setting. -static grpc_compression_algorithm find_compression_algorithm( - grpc_metadata_batch* initial_metadata, channel_data* channeld) { +grpc_compression_algorithm FindCompressionAlgorithm( + grpc_metadata_batch* initial_metadata, ChannelData* channeld) { if (initial_metadata->idx.named.grpc_internal_encoding_request == nullptr) { - return channeld->default_compression_algorithm; + return channeld->default_compression_algorithm(); } grpc_compression_algorithm compression_algorithm; // Parse the compression algorithm from the initial metadata. @@ -143,7 +202,7 @@ static grpc_compression_algorithm find_compression_algorithm( // enabled. // TODO(juanlishen): Maybe use channel default or abort() if the algorithm // from the initial metadata is disabled. - if (GPR_LIKELY(GPR_BITGET(channeld->enabled_compression_algorithms_bitset, + if (GPR_LIKELY(GPR_BITGET(channeld->enabled_compression_algorithms_bitset(), compression_algorithm))) { return compression_algorithm; } @@ -158,30 +217,24 @@ static grpc_compression_algorithm find_compression_algorithm( return GRPC_COMPRESS_NONE; } -static void initialize_state(grpc_call_element* elem, call_data* calld) { - GPR_DEBUG_ASSERT(!calld->state_initialized); - calld->state_initialized = true; - grpc_slice_buffer_init(&calld->slices); - GRPC_CLOSURE_INIT(&calld->send_message_on_complete, - ::send_message_on_complete, elem, +void CallData::InitializeState() { + GPR_DEBUG_ASSERT(!state_initialized_); + state_initialized_ = true; + grpc_slice_buffer_init(&slices_); + GRPC_CLOSURE_INIT(&send_message_on_complete_, SendMessageOnComplete, this, grpc_schedule_on_exec_ctx); - GRPC_CLOSURE_INIT(&calld->on_send_message_next_done, - ::on_send_message_next_done, elem, + GRPC_CLOSURE_INIT(&on_send_message_next_done_, OnSendMessageNextDone, this, grpc_schedule_on_exec_ctx); } -static grpc_error* process_send_initial_metadata( - grpc_call_element* elem, - grpc_metadata_batch* initial_metadata) GRPC_MUST_USE_RESULT; -static grpc_error* process_send_initial_metadata( - grpc_call_element* elem, grpc_metadata_batch* initial_metadata) { - call_data* calld = static_cast(elem->call_data); - channel_data* channeld = static_cast(elem->channel_data); +grpc_error* CallData::ProcessSendInitialMetadata( + grpc_metadata_batch* initial_metadata) { + ChannelData* channeld = static_cast(elem_->channel_data); // Find the compression algorithm. grpc_compression_algorithm compression_algorithm = - find_compression_algorithm(initial_metadata, channeld); + FindCompressionAlgorithm(initial_metadata, channeld); // Note that at most one of the following algorithms can be set. - calld->message_compression_algorithm = + message_compression_algorithm_ = grpc_compression_algorithm_to_message_compression_algorithm( compression_algorithm); grpc_stream_compression_algorithm stream_compression_algorithm = @@ -189,257 +242,253 @@ static grpc_error* process_send_initial_metadata( compression_algorithm); // Hint compression algorithm. grpc_error* error = GRPC_ERROR_NONE; - if (calld->message_compression_algorithm != GRPC_MESSAGE_COMPRESS_NONE) { - initialize_state(elem, calld); + if (message_compression_algorithm_ != GRPC_MESSAGE_COMPRESS_NONE) { + InitializeState(); error = grpc_metadata_batch_add_tail( - initial_metadata, &calld->message_compression_algorithm_storage, + initial_metadata, &message_compression_algorithm_storage_, grpc_message_compression_encoding_mdelem( - calld->message_compression_algorithm), + message_compression_algorithm_), GRPC_BATCH_GRPC_ENCODING); } else if (stream_compression_algorithm != GRPC_STREAM_COMPRESS_NONE) { - initialize_state(elem, calld); + InitializeState(); error = grpc_metadata_batch_add_tail( - initial_metadata, &calld->stream_compression_algorithm_storage, + initial_metadata, &stream_compression_algorithm_storage_, grpc_stream_compression_encoding_mdelem(stream_compression_algorithm), GRPC_BATCH_CONTENT_ENCODING); } if (error != GRPC_ERROR_NONE) return error; // Convey supported compression algorithms. error = grpc_metadata_batch_add_tail( - initial_metadata, &calld->accept_encoding_storage, + initial_metadata, &accept_encoding_storage_, GRPC_MDELEM_ACCEPT_ENCODING_FOR_ALGORITHMS( - channeld->enabled_message_compression_algorithms_bitset), + channeld->enabled_message_compression_algorithms_bitset()), GRPC_BATCH_GRPC_ACCEPT_ENCODING); if (error != GRPC_ERROR_NONE) return error; // Do not overwrite accept-encoding header if it already presents (e.g., added // by some proxy). if (!initial_metadata->idx.named.accept_encoding) { error = grpc_metadata_batch_add_tail( - initial_metadata, &calld->accept_stream_encoding_storage, + initial_metadata, &accept_stream_encoding_storage_, GRPC_MDELEM_ACCEPT_STREAM_ENCODING_FOR_ALGORITHMS( - channeld->enabled_stream_compression_algorithms_bitset), + channeld->enabled_stream_compression_algorithms_bitset()), GRPC_BATCH_ACCEPT_ENCODING); } return error; } -static void send_message_on_complete(void* arg, grpc_error* error) { - grpc_call_element* elem = static_cast(arg); - call_data* calld = static_cast(elem->call_data); - grpc_slice_buffer_reset_and_unref_internal(&calld->slices); +void CallData::SendMessageOnComplete(void* arg, grpc_error* error) { + CallData* calld = static_cast(arg); + grpc_slice_buffer_reset_and_unref_internal(&calld->slices_); grpc_core::Closure::Run(DEBUG_LOCATION, - calld->original_send_message_on_complete, + calld->original_send_message_on_complete_, GRPC_ERROR_REF(error)); } -static void send_message_batch_continue(grpc_call_element* elem) { - call_data* calld = static_cast(elem->call_data); +void CallData::SendMessageBatchContinue() { // Note: The call to grpc_call_next_op() results in yielding the - // call combiner, so we need to clear calld->send_message_batch + // call combiner, so we need to clear calld->send_message_batch_ // before we do that. grpc_transport_stream_op_batch* send_message_batch = - calld->send_message_batch; - calld->send_message_batch = nullptr; - grpc_call_next_op(elem, send_message_batch); + this->send_message_batch_; + send_message_batch_ = nullptr; + grpc_call_next_op(elem_, send_message_batch); } -static void finish_send_message(grpc_call_element* elem) { - call_data* calld = static_cast(elem->call_data); - GPR_DEBUG_ASSERT(calld->message_compression_algorithm != +void CallData::FinishSendMessage() { + GPR_DEBUG_ASSERT(message_compression_algorithm_ != GRPC_MESSAGE_COMPRESS_NONE); // Compress the data if appropriate. grpc_slice_buffer tmp; grpc_slice_buffer_init(&tmp); uint32_t send_flags = - calld->send_message_batch->payload->send_message.send_message->flags(); - bool did_compress = grpc_msg_compress(calld->message_compression_algorithm, - &calld->slices, &tmp); + send_message_batch_->payload->send_message.send_message->flags(); + bool did_compress = + grpc_msg_compress(message_compression_algorithm_, &slices_, &tmp); if (did_compress) { if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) { const char* algo_name; - const size_t before_size = calld->slices.length; + const size_t before_size = slices_.length; const size_t after_size = tmp.length; const float savings_ratio = 1.0f - static_cast(after_size) / static_cast(before_size); GPR_ASSERT(grpc_message_compression_algorithm_name( - calld->message_compression_algorithm, &algo_name)); + message_compression_algorithm_, &algo_name)); gpr_log(GPR_INFO, "Compressed[%s] %" PRIuPTR " bytes vs. %" PRIuPTR " bytes (%.2f%% savings)", algo_name, before_size, after_size, 100 * savings_ratio); } - grpc_slice_buffer_swap(&calld->slices, &tmp); + grpc_slice_buffer_swap(&slices_, &tmp); send_flags |= GRPC_WRITE_INTERNAL_COMPRESS; } else { if (GRPC_TRACE_FLAG_ENABLED(grpc_compression_trace)) { const char* algo_name; GPR_ASSERT(grpc_message_compression_algorithm_name( - calld->message_compression_algorithm, &algo_name)); + message_compression_algorithm_, &algo_name)); gpr_log(GPR_INFO, "Algorithm '%s' enabled but decided not to compress. Input size: " "%" PRIuPTR, - algo_name, calld->slices.length); + algo_name, slices_.length); } } grpc_slice_buffer_destroy_internal(&tmp); // Swap out the original byte stream with our new one and send the // batch down. - calld->replacement_stream.Init(&calld->slices, send_flags); - calld->send_message_batch->payload->send_message.send_message.reset( - calld->replacement_stream.get()); - calld->original_send_message_on_complete = - calld->send_message_batch->on_complete; - calld->send_message_batch->on_complete = &calld->send_message_on_complete; - send_message_batch_continue(elem); + replacement_stream_.Init(&slices_, send_flags); + send_message_batch_->payload->send_message.send_message.reset( + replacement_stream_.get()); + original_send_message_on_complete_ = send_message_batch_->on_complete; + send_message_batch_->on_complete = &send_message_on_complete_; + SendMessageBatchContinue(); } -static void fail_send_message_batch_in_call_combiner(void* arg, - grpc_error* error) { - call_data* calld = static_cast(arg); - if (calld->send_message_batch != nullptr) { +void CallData::FailSendMessageBatchInCallCombiner(void* arg, + grpc_error* error) { + CallData* calld = static_cast(arg); + if (calld->send_message_batch_ != nullptr) { grpc_transport_stream_op_batch_finish_with_failure( - calld->send_message_batch, GRPC_ERROR_REF(error), calld->call_combiner); - calld->send_message_batch = nullptr; + calld->send_message_batch_, GRPC_ERROR_REF(error), + calld->call_combiner_); + calld->send_message_batch_ = nullptr; } } -// Pulls a slice from the send_message byte stream and adds it to calld->slices. -static grpc_error* pull_slice_from_send_message(call_data* calld) { +// Pulls a slice from the send_message byte stream and adds it to +// calld->slices_. +grpc_error* CallData::PullSliceFromSendMessage() { grpc_slice incoming_slice; grpc_error* error = - calld->send_message_batch->payload->send_message.send_message->Pull( + send_message_batch_->payload->send_message.send_message->Pull( &incoming_slice); if (error == GRPC_ERROR_NONE) { - grpc_slice_buffer_add(&calld->slices, incoming_slice); + grpc_slice_buffer_add(&slices_, incoming_slice); } return error; } // Reads as many slices as possible from the send_message byte stream. -// If all data has been read, invokes finish_send_message(). Otherwise, +// If all data has been read, invokes FinishSendMessage(). Otherwise, // an async call to ByteStream::Next() has been started, which will // eventually result in calling on_send_message_next_done(). -static void continue_reading_send_message(grpc_call_element* elem) { - call_data* calld = static_cast(elem->call_data); - if (calld->slices.length == - calld->send_message_batch->payload->send_message.send_message->length()) { - finish_send_message(elem); +void CallData::ContinueReadingSendMessage() { + if (slices_.length == + send_message_batch_->payload->send_message.send_message->length()) { + FinishSendMessage(); return; } - while (calld->send_message_batch->payload->send_message.send_message->Next( - ~static_cast(0), &calld->on_send_message_next_done)) { - grpc_error* error = pull_slice_from_send_message(calld); + while (send_message_batch_->payload->send_message.send_message->Next( + ~static_cast(0), &on_send_message_next_done_)) { + grpc_error* error = PullSliceFromSendMessage(); if (error != GRPC_ERROR_NONE) { // Closure callback; does not take ownership of error. - fail_send_message_batch_in_call_combiner(calld, error); + FailSendMessageBatchInCallCombiner(this, error); GRPC_ERROR_UNREF(error); return; } - if (calld->slices.length == calld->send_message_batch->payload->send_message - .send_message->length()) { - finish_send_message(elem); + if (slices_.length == + send_message_batch_->payload->send_message.send_message->length()) { + FinishSendMessage(); break; } } } // Async callback for ByteStream::Next(). -static void on_send_message_next_done(void* arg, grpc_error* error) { - grpc_call_element* elem = static_cast(arg); - call_data* calld = static_cast(elem->call_data); +void CallData::OnSendMessageNextDone(void* arg, grpc_error* error) { + CallData* calld = static_cast(arg); if (error != GRPC_ERROR_NONE) { // Closure callback; does not take ownership of error. - fail_send_message_batch_in_call_combiner(calld, error); + FailSendMessageBatchInCallCombiner(calld, error); return; } - error = pull_slice_from_send_message(calld); + error = calld->PullSliceFromSendMessage(); if (error != GRPC_ERROR_NONE) { // Closure callback; does not take ownership of error. - fail_send_message_batch_in_call_combiner(calld, error); + FailSendMessageBatchInCallCombiner(calld, error); GRPC_ERROR_UNREF(error); return; } - if (calld->slices.length == - calld->send_message_batch->payload->send_message.send_message->length()) { - finish_send_message(elem); + if (calld->slices_.length == calld->send_message_batch_->payload->send_message + .send_message->length()) { + calld->FinishSendMessage(); } else { - continue_reading_send_message(elem); + calld->ContinueReadingSendMessage(); } } -static void start_send_message_batch(void* arg, grpc_error* /*unused*/) { - grpc_call_element* elem = static_cast(arg); - if (skip_message_compression(elem)) { - send_message_batch_continue(elem); +void CallData::StartSendMessageBatch(void* arg, grpc_error* /*unused*/) { + CallData* calld = static_cast(arg); + if (calld->SkipMessageCompression()) { + calld->SendMessageBatchContinue(); } else { - continue_reading_send_message(elem); + calld->ContinueReadingSendMessage(); } } -static void compress_start_transport_stream_op_batch( +void CallData::CompressStartTransportStreamOpBatch( grpc_call_element* elem, grpc_transport_stream_op_batch* batch) { GPR_TIMER_SCOPE("compress_start_transport_stream_op_batch", 0); - call_data* calld = static_cast(elem->call_data); + CallData* calld = static_cast(elem->call_data); // Handle cancel_stream. if (batch->cancel_stream) { - GRPC_ERROR_UNREF(calld->cancel_error); - calld->cancel_error = + GRPC_ERROR_UNREF(calld->cancel_error_); + calld->cancel_error_ = GRPC_ERROR_REF(batch->payload->cancel_stream.cancel_error); - if (calld->send_message_batch != nullptr) { - if (!calld->seen_initial_metadata) { + if (calld->send_message_batch_ != nullptr) { + if (!calld->seen_initial_metadata_) { GRPC_CALL_COMBINER_START( - calld->call_combiner, - GRPC_CLOSURE_CREATE(fail_send_message_batch_in_call_combiner, calld, + calld->call_combiner_, + GRPC_CLOSURE_CREATE(FailSendMessageBatchInCallCombiner, calld, grpc_schedule_on_exec_ctx), - GRPC_ERROR_REF(calld->cancel_error), "failing send_message op"); + GRPC_ERROR_REF(calld->cancel_error_), "failing send_message op"); } else { - calld->send_message_batch->payload->send_message.send_message->Shutdown( - GRPC_ERROR_REF(calld->cancel_error)); + calld->send_message_batch_->payload->send_message.send_message + ->Shutdown(GRPC_ERROR_REF(calld->cancel_error_)); } } - } else if (calld->cancel_error != GRPC_ERROR_NONE) { + } else if (calld->cancel_error_ != GRPC_ERROR_NONE) { grpc_transport_stream_op_batch_finish_with_failure( - batch, GRPC_ERROR_REF(calld->cancel_error), calld->call_combiner); + batch, GRPC_ERROR_REF(calld->cancel_error_), calld->call_combiner_); return; } // Handle send_initial_metadata. if (batch->send_initial_metadata) { - GPR_ASSERT(!calld->seen_initial_metadata); - grpc_error* error = process_send_initial_metadata( - elem, batch->payload->send_initial_metadata.send_initial_metadata); + GPR_ASSERT(!calld->seen_initial_metadata_); + grpc_error* error = calld->ProcessSendInitialMetadata( + batch->payload->send_initial_metadata.send_initial_metadata); if (error != GRPC_ERROR_NONE) { grpc_transport_stream_op_batch_finish_with_failure(batch, error, - calld->call_combiner); + calld->call_combiner_); return; } - calld->seen_initial_metadata = true; + calld->seen_initial_metadata_ = true; // If we had previously received a batch containing a send_message op, // handle it now. Note that we need to re-enter the call combiner // for this, since we can't send two batches down while holding the // call combiner, since the connected_channel filter (at the bottom of // the call stack) will release the call combiner for each batch it sees. - if (calld->send_message_batch != nullptr) { + if (calld->send_message_batch_ != nullptr) { GRPC_CALL_COMBINER_START( - calld->call_combiner, - &calld->start_send_message_batch_in_call_combiner, GRPC_ERROR_NONE, + calld->call_combiner_, + &calld->start_send_message_batch_in_call_combiner_, GRPC_ERROR_NONE, "starting send_message after send_initial_metadata"); } } // Handle send_message. if (batch->send_message) { - GPR_ASSERT(calld->send_message_batch == nullptr); - calld->send_message_batch = batch; + GPR_ASSERT(calld->send_message_batch_ == nullptr); + calld->send_message_batch_ = batch; // If we have not yet seen send_initial_metadata, then we have to // wait. We save the batch in calld and then drop the call // combiner, which we'll have to pick up again later when we get // send_initial_metadata. - if (!calld->seen_initial_metadata) { + if (!calld->seen_initial_metadata_) { GRPC_CALL_COMBINER_STOP( - calld->call_combiner, + calld->call_combiner_, "send_message batch pending send_initial_metadata"); return; } - start_send_message_batch(elem, GRPC_ERROR_NONE); + StartSendMessageBatch(calld, GRPC_ERROR_NONE); } else { // Pass control down the stack. grpc_call_next_op(elem, batch); @@ -447,63 +496,44 @@ static void compress_start_transport_stream_op_batch( } /* Constructor for call_data */ -static grpc_error* compress_init_call_elem(grpc_call_element* elem, - const grpc_call_element_args* args) { - new (elem->call_data) call_data(elem, *args); +static grpc_error* CompressInitCallElem(grpc_call_element* elem, + const grpc_call_element_args* args) { + new (elem->call_data) CallData(elem, *args); return GRPC_ERROR_NONE; } /* Destructor for call_data */ -static void compress_destroy_call_elem( - grpc_call_element* elem, const grpc_call_final_info* /*final_info*/, - grpc_closure* /*ignored*/) { - call_data* calld = static_cast(elem->call_data); - calld->~call_data(); +static void CompressDestroyCallElem(grpc_call_element* elem, + const grpc_call_final_info* /*final_info*/, + grpc_closure* /*ignored*/) { + CallData* calld = static_cast(elem->call_data); + calld->~CallData(); } -/* Constructor for channel_data */ -static grpc_error* compress_init_channel_elem(grpc_channel_element* elem, - grpc_channel_element_args* args) { - channel_data* channeld = static_cast(elem->channel_data); - // Get the enabled and the default algorithms from channel args. - channeld->enabled_compression_algorithms_bitset = - grpc_channel_args_compression_algorithm_get_states(args->channel_args); - channeld->default_compression_algorithm = - grpc_channel_args_get_channel_default_compression_algorithm( - args->channel_args); - // Make sure the default is enabled. - if (!GPR_BITGET(channeld->enabled_compression_algorithms_bitset, - channeld->default_compression_algorithm)) { - const char* name; - GPR_ASSERT(grpc_compression_algorithm_name( - channeld->default_compression_algorithm, &name) == 1); - gpr_log(GPR_ERROR, - "default compression algorithm %s not enabled: switching to none", - name); - channeld->default_compression_algorithm = GRPC_COMPRESS_NONE; - } - channeld->enabled_message_compression_algorithms_bitset = - grpc_compression_bitset_to_message_bitset( - channeld->enabled_compression_algorithms_bitset); - channeld->enabled_stream_compression_algorithms_bitset = - grpc_compression_bitset_to_stream_bitset( - channeld->enabled_compression_algorithms_bitset); - GPR_ASSERT(!args->is_last); +/* Constructor for ChannelData */ +static grpc_error* CompressInitChannelElem(grpc_channel_element* elem, + grpc_channel_element_args* args) { + new (elem->channel_data) ChannelData(args); return GRPC_ERROR_NONE; } /* Destructor for channel data */ -static void compress_destroy_channel_elem(grpc_channel_element* /*elem*/) {} +void CompressDestroyChannelElem(grpc_channel_element* elem) { + ChannelData* channeld = static_cast(elem->channel_data); + channeld->~ChannelData(); +} + +} // namespace const grpc_channel_filter grpc_message_compress_filter = { - compress_start_transport_stream_op_batch, + CallData::CompressStartTransportStreamOpBatch, grpc_channel_next_op, - sizeof(call_data), - compress_init_call_elem, + sizeof(CallData), + CompressInitCallElem, grpc_call_stack_ignore_set_pollset_or_pollset_set, - compress_destroy_call_elem, - sizeof(channel_data), - compress_init_channel_elem, - compress_destroy_channel_elem, + CompressDestroyCallElem, + sizeof(ChannelData), + CompressInitChannelElem, + CompressDestroyChannelElem, grpc_channel_next_get_info, "message_compress"}; From 1e5aa76adf015c8bb14bba3863a5b762033b3d06 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Thu, 2 Apr 2020 22:07:04 -0700 Subject: [PATCH 11/73] Some notes on new filter implementation --- src/core/lib/channel/channel_stack.h | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/core/lib/channel/channel_stack.h b/src/core/lib/channel/channel_stack.h index f9272871931..87e675951b3 100644 --- a/src/core/lib/channel/channel_stack.h +++ b/src/core/lib/channel/channel_stack.h @@ -31,7 +31,18 @@ chains are linear, then channel stacks provide a mechanism to minimize allocations for that chain. Call stacks are created by channel stacks and represent the per-call data - for that stack. */ + for that stack. + + Implementations should take care of the following details for a batch - + 1. Synchronization is achieved with a CallCombiner. View + src/core/lib/iomgr/call_combiner.h for more details. + 2. If the filter wants to inject an error on the way down, it needs to call + grpc_transport_stream_op_batch_finish_with_failure from within the call + combiner. This will cause any batch callbacks to be called with that error. + 3. If the filter wants to inject an error on the way up (from a callback), it + should also inject that error in the recv_trailing_metadata callback so that + it can have an effect on the call status. +*/ #include From 576f79dff0e425b509283addb1f2a87209c9eab3 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 3 Apr 2020 07:34:54 +0200 Subject: [PATCH 12/73] cleanup gen_build_yaml.py for end2end tests --- test/core/end2end/README | 2 +- test/core/end2end/gen_build_yaml.py | 402 +-------------------- tools/buildgen/generate_build_additions.sh | 2 +- 3 files changed, 17 insertions(+), 389 deletions(-) diff --git a/test/core/end2end/README b/test/core/end2end/README index a18172a7a1d..51cc144039d 100644 --- a/test/core/end2end/README +++ b/test/core/end2end/README @@ -3,5 +3,5 @@ forms a complete end-to-end test. To add a new test or fixture: - add the code to the relevant directory -- update gen_build_yaml.py to reflect the change +- update generate_tests.bzl to reflect the change - regenerate projects diff --git a/test/core/end2end/gen_build_yaml.py b/test/core/end2end/gen_build_yaml.py index d1e78cc6490..3cb31686619 100755 --- a/test/core/end2end/gen_build_yaml.py +++ b/test/core/end2end/gen_build_yaml.py @@ -11,408 +11,36 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Generates the appropriate build.json data for all the end2end tests.""" - -from __future__ import print_function +"""Generates the list of end2end test cases from generate_tests.bzl""" +import os +import sys import yaml -import collections -import hashlib - -FixtureOptions = collections.namedtuple( - 'FixtureOptions', - 'fullstack includes_proxy dns_resolver name_resolution secure platforms ci_mac tracing exclude_configs exclude_iomgrs large_writes enables_compression supports_compression is_inproc is_http2 supports_proxy_auth supports_write_buffering client_channel' -) -default_unsecure_fixture_options = FixtureOptions( - True, False, True, True, False, ['windows', 'linux', 'mac', 'posix'], True, - False, [], [], True, False, True, False, True, False, True, True) -socketpair_unsecure_fixture_options = default_unsecure_fixture_options._replace( - fullstack=False, dns_resolver=False, client_channel=False) -default_secure_fixture_options = default_unsecure_fixture_options._replace( - secure=True) -uds_fixture_options = default_unsecure_fixture_options._replace( - dns_resolver=False, - platforms=['linux', 'mac', 'posix'], - exclude_iomgrs=['uv']) -local_fixture_options = default_secure_fixture_options._replace( - dns_resolver=False, - platforms=['linux', 'mac', 'posix'], - exclude_iomgrs=['uv']) -fd_unsecure_fixture_options = default_unsecure_fixture_options._replace( - dns_resolver=False, - fullstack=False, - platforms=['linux', 'mac', 'posix'], - exclude_iomgrs=['uv'], - client_channel=False) -inproc_fixture_options = default_secure_fixture_options._replace( - dns_resolver=False, - fullstack=False, - name_resolution=False, - supports_compression=False, - is_inproc=True, - is_http2=False, - supports_write_buffering=False, - client_channel=False) - -# maps fixture name to whether it requires the security library -END2END_FIXTURES = { - 'h2_compress': - default_unsecure_fixture_options._replace(enables_compression=True), - 'h2_census': - default_unsecure_fixture_options, - # This cmake target is disabled for now because it depends on OpenCensus, - # which is Bazel-only. - # 'h2_load_reporting': default_unsecure_fixture_options, - 'h2_fakesec': - default_secure_fixture_options._replace(ci_mac=False), - 'h2_fd': - fd_unsecure_fixture_options, - 'h2_full': - default_unsecure_fixture_options, - 'h2_full+pipe': - default_unsecure_fixture_options._replace(platforms=['linux'], - exclude_iomgrs=['uv']), - 'h2_full+trace': - default_unsecure_fixture_options._replace(tracing=True), - 'h2_full+workarounds': - default_unsecure_fixture_options, - 'h2_http_proxy': - default_unsecure_fixture_options._replace(ci_mac=False, - exclude_iomgrs=['uv'], - supports_proxy_auth=True), - 'h2_oauth2': - default_secure_fixture_options._replace(ci_mac=False, - exclude_iomgrs=['uv']), - 'h2_proxy': - default_unsecure_fixture_options._replace(includes_proxy=True, - ci_mac=False, - exclude_iomgrs=['uv']), - 'h2_sockpair_1byte': - socketpair_unsecure_fixture_options._replace(ci_mac=False, - exclude_configs=['msan'], - large_writes=False, - exclude_iomgrs=['uv']), - 'h2_sockpair': - socketpair_unsecure_fixture_options._replace(ci_mac=False, - exclude_iomgrs=['uv']), - 'h2_sockpair+trace': - socketpair_unsecure_fixture_options._replace(ci_mac=False, - tracing=True, - large_writes=False, - exclude_iomgrs=['uv']), - 'h2_ssl': - default_secure_fixture_options, - 'h2_ssl_cred_reload': - default_secure_fixture_options, - 'h2_tls': - default_secure_fixture_options, - 'h2_local_uds': - local_fixture_options, - 'h2_local_ipv4': - local_fixture_options, - 'h2_local_ipv6': - local_fixture_options, - 'h2_ssl_proxy': - default_secure_fixture_options._replace(includes_proxy=True, - ci_mac=False, - exclude_iomgrs=['uv']), - 'h2_uds': - uds_fixture_options, - 'inproc': - inproc_fixture_options -} -TestOptions = collections.namedtuple( - 'TestOptions', - 'needs_fullstack needs_dns needs_names proxyable secure traceable cpu_cost exclude_iomgrs large_writes flaky allows_compression needs_compression exclude_inproc needs_http2 needs_proxy_auth needs_write_buffering needs_client_channel' -) -default_test_options = TestOptions(False, False, False, True, False, True, 1.0, - [], False, False, True, False, False, False, - False, False, False) -connectivity_test_options = default_test_options._replace(needs_fullstack=True) +_ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../../..')) +os.chdir(_ROOT) -LOWCPU = 0.1 -# maps test names to options -END2END_TESTS = { - 'authority_not_supported': - default_test_options, - 'bad_hostname': - default_test_options._replace(needs_names=True), - 'bad_ping': - connectivity_test_options._replace(proxyable=False), - 'binary_metadata': - default_test_options._replace(cpu_cost=LOWCPU), - 'resource_quota_server': - default_test_options._replace(large_writes=True, - proxyable=False, - allows_compression=False), - 'call_creds': - default_test_options._replace(secure=True), - 'cancel_after_accept': - default_test_options._replace(cpu_cost=LOWCPU), - 'cancel_after_client_done': - default_test_options._replace(cpu_cost=LOWCPU), - 'cancel_after_invoke': - default_test_options._replace(cpu_cost=LOWCPU), - 'cancel_after_round_trip': - default_test_options._replace(cpu_cost=LOWCPU), - 'cancel_before_invoke': - default_test_options._replace(cpu_cost=LOWCPU), - 'cancel_in_a_vacuum': - default_test_options._replace(cpu_cost=LOWCPU), - 'cancel_with_status': - default_test_options._replace(cpu_cost=LOWCPU), - 'compressed_payload': - default_test_options._replace(proxyable=False, needs_compression=True), - 'connectivity': - connectivity_test_options._replace(needs_names=True, - proxyable=False, - cpu_cost=LOWCPU, - exclude_iomgrs=['uv']), - 'channelz': - default_test_options, - 'default_host': - default_test_options._replace(needs_fullstack=True, - needs_dns=True, - needs_names=True), - 'call_host_override': - default_test_options._replace(needs_fullstack=True, - needs_dns=True, - needs_names=True), - 'disappearing_server': - connectivity_test_options._replace(flaky=True, needs_names=True), - 'empty_batch': - default_test_options._replace(cpu_cost=LOWCPU), - 'filter_causes_close': - default_test_options._replace(cpu_cost=LOWCPU), - 'filter_call_init_fails': - default_test_options, - 'filter_context': - default_test_options, - 'filter_latency': - default_test_options._replace(cpu_cost=LOWCPU), - 'filter_status_code': - default_test_options._replace(cpu_cost=LOWCPU), - 'graceful_server_shutdown': - default_test_options._replace(cpu_cost=LOWCPU, exclude_inproc=True), - 'hpack_size': - default_test_options._replace(proxyable=False, - traceable=False, - cpu_cost=LOWCPU), - 'high_initial_seqno': - default_test_options._replace(cpu_cost=LOWCPU), - 'idempotent_request': - default_test_options, - 'invoke_large_request': - default_test_options, - 'keepalive_timeout': - default_test_options._replace(proxyable=False, - cpu_cost=LOWCPU, - needs_http2=True), - 'large_metadata': - default_test_options, - 'max_concurrent_streams': - default_test_options._replace(proxyable=False, - cpu_cost=LOWCPU, - exclude_inproc=True), - 'max_connection_age': - default_test_options._replace(cpu_cost=LOWCPU, exclude_inproc=True), - 'max_connection_idle': - connectivity_test_options._replace(proxyable=False, - exclude_iomgrs=['uv'], - cpu_cost=LOWCPU), - 'max_message_length': - default_test_options._replace(cpu_cost=LOWCPU), - 'negative_deadline': - default_test_options, - 'no_error_on_hotpath': - default_test_options._replace(proxyable=False), - 'no_logging': - default_test_options._replace(traceable=False), - 'no_op': - default_test_options, - 'payload': - default_test_options, - # This cmake target is disabled for now because it depends on OpenCensus, - # which is Bazel-only. - # 'load_reporting_hook': default_test_options, - 'ping_pong_streaming': - default_test_options._replace(cpu_cost=LOWCPU), - 'ping': - connectivity_test_options._replace(proxyable=False, cpu_cost=LOWCPU), - 'proxy_auth': - default_test_options._replace(needs_proxy_auth=True), - 'registered_call': - default_test_options, - 'request_with_flags': - default_test_options._replace(proxyable=False, cpu_cost=LOWCPU), - 'request_with_payload': - default_test_options._replace(cpu_cost=LOWCPU), - # TODO(roth): Remove proxyable=False for all retry tests once we - # have a way for the proxy to propagate the fact that trailing - # metadata is available when initial metadata is returned. - # See https://github.com/grpc/grpc/issues/14467 for context. - 'retry': - default_test_options._replace(cpu_cost=LOWCPU, - needs_client_channel=True, - proxyable=False), - 'retry_cancellation': - default_test_options._replace(cpu_cost=LOWCPU, - needs_client_channel=True, - proxyable=False), - 'retry_disabled': - default_test_options._replace(cpu_cost=LOWCPU, - needs_client_channel=True, - proxyable=False), - 'retry_exceeds_buffer_size_in_initial_batch': - default_test_options._replace(cpu_cost=LOWCPU, - needs_client_channel=True, - proxyable=False), - 'retry_exceeds_buffer_size_in_subsequent_batch': - default_test_options._replace(cpu_cost=LOWCPU, - needs_client_channel=True, - proxyable=False), - 'retry_non_retriable_status': - default_test_options._replace(cpu_cost=LOWCPU, - needs_client_channel=True, - proxyable=False), - 'retry_non_retriable_status_before_recv_trailing_metadata_started': - default_test_options._replace(cpu_cost=LOWCPU, - needs_client_channel=True, - proxyable=False), - 'retry_recv_initial_metadata': - default_test_options._replace(cpu_cost=LOWCPU, - needs_client_channel=True, - proxyable=False), - 'retry_recv_message': - default_test_options._replace(cpu_cost=LOWCPU, - needs_client_channel=True, - proxyable=False), - 'retry_server_pushback_delay': - default_test_options._replace(cpu_cost=LOWCPU, - needs_client_channel=True, - proxyable=False), - 'retry_server_pushback_disabled': - default_test_options._replace(cpu_cost=LOWCPU, - needs_client_channel=True, - proxyable=False), - 'retry_streaming': - default_test_options._replace(cpu_cost=LOWCPU, - needs_client_channel=True, - proxyable=False), - 'retry_streaming_after_commit': - default_test_options._replace(cpu_cost=LOWCPU, - needs_client_channel=True, - proxyable=False), - 'retry_streaming_succeeds_before_replay_finished': - default_test_options._replace(cpu_cost=LOWCPU, - needs_client_channel=True, - proxyable=False), - 'retry_throttled': - default_test_options._replace(cpu_cost=LOWCPU, - needs_client_channel=True, - proxyable=False), - 'retry_too_many_attempts': - default_test_options._replace(cpu_cost=LOWCPU, - needs_client_channel=True, - proxyable=False), - 'server_finishes_request': - default_test_options._replace(cpu_cost=LOWCPU), - 'shutdown_finishes_calls': - default_test_options._replace(cpu_cost=LOWCPU), - 'shutdown_finishes_tags': - default_test_options._replace(cpu_cost=LOWCPU), - 'simple_cacheable_request': - default_test_options._replace(cpu_cost=LOWCPU), - 'stream_compression_compressed_payload': - default_test_options._replace(proxyable=False, exclude_inproc=True), - 'stream_compression_payload': - default_test_options._replace(exclude_inproc=True), - 'stream_compression_ping_pong_streaming': - default_test_options._replace(exclude_inproc=True), - 'simple_delayed_request': - connectivity_test_options, - 'simple_metadata': - default_test_options, - 'simple_request': - default_test_options, - 'streaming_error_response': - default_test_options._replace(cpu_cost=LOWCPU), - 'trailing_metadata': - default_test_options, - 'workaround_cronet_compression': - default_test_options, - 'write_buffering': - default_test_options._replace(cpu_cost=LOWCPU, - needs_write_buffering=True), - 'write_buffering_at_end': - default_test_options._replace(cpu_cost=LOWCPU, - needs_write_buffering=True), -} +def load(*args): + """Replacement of bazel's load() function""" + pass -def compatible(f, t): - if END2END_TESTS[t].needs_fullstack: - if not END2END_FIXTURES[f].fullstack: - return False - if END2END_TESTS[t].needs_dns: - if not END2END_FIXTURES[f].dns_resolver: - return False - if END2END_TESTS[t].needs_names: - if not END2END_FIXTURES[f].name_resolution: - return False - if not END2END_TESTS[t].proxyable: - if END2END_FIXTURES[f].includes_proxy: - return False - if not END2END_TESTS[t].traceable: - if END2END_FIXTURES[f].tracing: - return False - if END2END_TESTS[t].large_writes: - if not END2END_FIXTURES[f].large_writes: - return False - if not END2END_TESTS[t].allows_compression: - if END2END_FIXTURES[f].enables_compression: - return False - if END2END_TESTS[t].needs_compression: - if not END2END_FIXTURES[f].supports_compression: - return False - if END2END_TESTS[t].exclude_inproc: - if END2END_FIXTURES[f].is_inproc: - return False - if END2END_TESTS[t].needs_http2: - if not END2END_FIXTURES[f].is_http2: - return False - if END2END_TESTS[t].needs_proxy_auth: - if not END2END_FIXTURES[f].supports_proxy_auth: - return False - if END2END_TESTS[t].needs_write_buffering: - if not END2END_FIXTURES[f].supports_write_buffering: - return False - if END2END_TESTS[t].needs_client_channel: - if not END2END_FIXTURES[f].client_channel: - return False - return True +def struct(**kwargs): + return kwargs # all the args as a dict -def without(l, e): - l = l[:] - l.remove(e) - return l +# generate_tests.bzl is now the source of truth for end2end tests. +# The .bzl file is basically a python file and we can "execute" it +# to get access to the variables it defines. +execfile('test/core/end2end/generate_tests.bzl') -# Originally, this method was used to generate end2end test cases for build.yaml, -# but since the test cases are now extracted from bazel BUILD file, -# this is not used for generating run_tests.py test cases anymore. -# Nevertheless, subset of the output is still used by end2end_tests.cc.template -# and end2end_nosec_tests.cc.template -# TODO(jtattermusch): cleanup this file, so that it only generates the data we need. -# Right now there's some duplication between generate_tests.bzl and this file. def main(): json = { # needed by end2end_tests.cc.template and end2end_nosec_tests.cc.template 'core_end2end_tests': - dict((t, END2END_TESTS[t].secure) for t in END2END_TESTS.keys()) + dict((t, END2END_TESTS[t]['secure']) for t in END2END_TESTS.keys()) } print(yaml.dump(json)) diff --git a/tools/buildgen/generate_build_additions.sh b/tools/buildgen/generate_build_additions.sh index 01d839284fc..e873789e9a9 100755 --- a/tools/buildgen/generate_build_additions.sh +++ b/tools/buildgen/generate_build_additions.sh @@ -23,7 +23,7 @@ gen_build_yaml_dirs=" \ src/upb \ src/zlib \ src/c-ares \ - test/core/end2end \ + test/core/end2end \ test/cpp/naming \ tools/run_tests/lb_interop_tests" From 9344a259b5d1216a1ba2184dd494a35299969c30 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 6 Apr 2020 13:06:35 -0700 Subject: [PATCH 13/73] Fix MacOS Python 3.5 gevent build --- .../run_tests/helper_scripts/build_python.sh | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index 670b5fc69d8..48b4d13a262 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -165,24 +165,33 @@ pip_install_dir() { cd "$PWD" } +# On library/version/platforms combos that do not have a binary +# published, we may end up building a dependency from source. In that +# case, several of our build environment variables may disrupt the +# third-party build process. This function pipes through only the +# minimal environment necessary. +pip_install() { + /usr/bin/env -i PATH="$PATH" $VENV_PYTHON -m pip install $@ +} + case "$VENV" in *py36_gevent*) # TODO(https://github.com/grpc/grpc/issues/15411) unpin this - $VENV_PYTHON -m pip install gevent==1.3.b1 + pip_install gevent==1.3.b1 ;; *gevent*) - $VENV_PYTHON -m pip install -U gevent + pip_install -U gevent ;; esac -$VENV_PYTHON -m pip install --upgrade pip==19.3.1 -$VENV_PYTHON -m pip install --upgrade setuptools -$VENV_PYTHON -m pip install --upgrade cython -$VENV_PYTHON -m pip install --upgrade six enum34 protobuf +pip_install --upgrade pip==19.3.1 +pip_install --upgrade setuptools +pip_install --upgrade cython +pip_install --upgrade six enum34 protobuf if [ "$("$VENV_PYTHON" -c "import sys; print(sys.version_info[0])")" == "2" ] then - $VENV_PYTHON -m pip install futures + pip_install futures fi pip_install_dir "$ROOT" @@ -214,7 +223,7 @@ pip_install_dir "$ROOT/src/python/grpcio_status" pip_install_dir "$ROOT/src/python/grpcio_testing" # Build/install tests -$VENV_PYTHON -m pip install coverage==4.4 oauth2client==4.1.0 \ +pip_install coverage==4.4 oauth2client==4.1.0 \ google-auth==1.0.0 requests==2.14.2 \ googleapis-common-protos==1.5.5 $VENV_PYTHON "$ROOT/src/python/grpcio_tests/setup.py" preprocess From 4c79018a96a0e59de1a039f6f7af7c410509bdd8 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 6 Apr 2020 13:08:28 -0700 Subject: [PATCH 14/73] Revert changes to abseil-cpp --- third_party/abseil-cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/abseil-cpp b/third_party/abseil-cpp index b832dce8489..df3ea785d8c 160000 --- a/third_party/abseil-cpp +++ b/third_party/abseil-cpp @@ -1 +1 @@ -Subproject commit b832dce8489ef7b6231384909fd9b68d5a5ff2b7 +Subproject commit df3ea785d8c30a9503321a3d35ee7d35808f190d From fb2b2122acdfcb854136fec9bf3916edb56ff102 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 6 Apr 2020 14:21:00 -0700 Subject: [PATCH 15/73] Add quotes --- tools/run_tests/helper_scripts/build_python.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index 48b4d13a262..a633049510b 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -165,13 +165,13 @@ pip_install_dir() { cd "$PWD" } -# On library/version/platforms combos that do not have a binary +# On library/version/platforms combo that do not have a binary # published, we may end up building a dependency from source. In that # case, several of our build environment variables may disrupt the # third-party build process. This function pipes through only the # minimal environment necessary. pip_install() { - /usr/bin/env -i PATH="$PATH" $VENV_PYTHON -m pip install $@ + /usr/bin/env -i PATH="$PATH" "$VENV_PYTHON" -m pip install $@ } case "$VENV" in From 41866c125023109eac0c700fa4576cdec99f1c62 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Thu, 2 Apr 2020 17:19:36 -0700 Subject: [PATCH 16/73] Add try_connect API for UnaryStreamCall and StreamStreamCall --- .../grpc/experimental/aio/_base_call.py | 30 +++++ .../grpcio/grpc/experimental/aio/_call.py | 10 ++ src/python/grpcio_tests/tests_aio/tests.json | 1 + .../grpcio_tests/tests_aio/unit/call_test.py | 55 ++++++--- .../tests_aio/unit/try_connect_test.py | 114 ++++++++++++++++++ 5 files changed, 196 insertions(+), 14 deletions(-) create mode 100644 src/python/grpcio_tests/tests_aio/unit/try_connect_test.py diff --git a/src/python/grpcio/grpc/experimental/aio/_base_call.py b/src/python/grpcio/grpc/experimental/aio/_base_call.py index d116982aa79..e5c93e0f607 100644 --- a/src/python/grpcio/grpc/experimental/aio/_base_call.py +++ b/src/python/grpcio/grpc/experimental/aio/_base_call.py @@ -158,6 +158,21 @@ class UnaryStreamCall(Generic[RequestType, ResponseType], stream. """ + @abstractmethod + async def try_connect(self) -> None: + """Tries to connect to peer and raise aio.AioRpcError if failed. + + This method is available for RPCs with streaming responses. This method + enables the application to ensure if the RPC has been successfully + connected. Otherwise, an AioRpcError will be raised to explain the + reason of the connection failure. + + For RPCs with unary response, the connectivity issue will be raised + once the application awaits the call. + + This method is recommended for building retry mechanisms. + """ + class StreamUnaryCall(Generic[RequestType, ResponseType], Call, @@ -229,3 +244,18 @@ class StreamStreamCall(Generic[RequestType, ResponseType], After done_writing is called, any additional invocation to the write function will fail. This function is idempotent. """ + + @abstractmethod + async def try_connect(self) -> None: + """Tries to connect to peer and raise aio.AioRpcError if failed. + + This method is available for RPCs with streaming responses. This method + enables the application to ensure if the RPC has been successfully + connected. Otherwise, an AioRpcError will be raised to explain the + reason of the connection failure. + + For RPCs with unary response, the connectivity issue will be raised + once the application awaits the call. + + This method is recommended for building retry mechanisms. + """ diff --git a/src/python/grpcio/grpc/experimental/aio/_call.py b/src/python/grpcio/grpc/experimental/aio/_call.py index be4887d239c..07c0d302141 100644 --- a/src/python/grpcio/grpc/experimental/aio/_call.py +++ b/src/python/grpcio/grpc/experimental/aio/_call.py @@ -536,6 +536,11 @@ class UnaryStreamCall(_StreamResponseMixin, Call, _base_call.UnaryStreamCall): self.cancel() raise + async def try_connect(self) -> None: + await self._send_unary_request_task + if self.done(): + await self._raise_for_status() + class StreamUnaryCall(_StreamRequestMixin, _UnaryResponseMixin, Call, _base_call.StreamUnaryCall): @@ -610,3 +615,8 @@ class StreamStreamCall(_StreamRequestMixin, _StreamResponseMixin, Call, if not self.cancelled(): self.cancel() # No need to raise RpcError here, because no one will `await` this task. + + async def try_connect(self) -> None: + await self._metadata_sent.wait() + if self.done(): + await self._raise_for_status() diff --git a/src/python/grpcio_tests/tests_aio/tests.json b/src/python/grpcio_tests/tests_aio/tests.json index 84dbf02b937..b76b5b893b4 100644 --- a/src/python/grpcio_tests/tests_aio/tests.json +++ b/src/python/grpcio_tests/tests_aio/tests.json @@ -28,5 +28,6 @@ "unit.server_interceptor_test.TestServerInterceptor", "unit.server_test.TestServer", "unit.timeout_test.TestTimeout", + "unit.try_connect_test.TestTryConnect", "unit.wait_for_ready_test.TestWaitForReady" ] diff --git a/src/python/grpcio_tests/tests_aio/unit/call_test.py b/src/python/grpcio_tests/tests_aio/unit/call_test.py index 5b52f0e1724..b0c126640a9 100644 --- a/src/python/grpcio_tests/tests_aio/unit/call_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/call_test.py @@ -16,22 +16,22 @@ import asyncio import logging import unittest +import datetime import grpc from grpc.experimental import aio from src.proto.grpc.testing import messages_pb2, test_pb2_grpc -from tests.unit.framework.common import test_constants from tests_aio.unit._test_base import AioTestBase -from tests.unit import resources - from tests_aio.unit._test_server import start_test_server +_SHORT_TIMEOUT_S = datetime.timedelta(seconds=1).total_seconds() + _NUM_STREAM_RESPONSES = 5 _RESPONSE_PAYLOAD_SIZE = 42 _REQUEST_PAYLOAD_SIZE = 7 _LOCAL_CANCEL_DETAILS_EXPECTATION = 'Locally cancelled by application!' -_RESPONSE_INTERVAL_US = test_constants.SHORT_TIMEOUT * 1000 * 1000 +_RESPONSE_INTERVAL_US = int(_SHORT_TIMEOUT_S * 1000 * 1000) _UNREACHABLE_TARGET = '0.1:1111' _INFINITE_INTERVAL_US = 2**31 - 1 @@ -434,24 +434,24 @@ class TestUnaryStreamCall(_MulticallableTestMixin, AioTestBase): interval_us=_RESPONSE_INTERVAL_US, )) - call = self._stub.StreamingOutputCall( - request, timeout=test_constants.SHORT_TIMEOUT * 2) + call = self._stub.StreamingOutputCall(request, + timeout=_SHORT_TIMEOUT_S * 2) response = await call.read() self.assertEqual(_RESPONSE_PAYLOAD_SIZE, len(response.payload.body)) # Should be around the same as the timeout remained_time = call.time_remaining() - self.assertGreater(remained_time, test_constants.SHORT_TIMEOUT * 3 / 2) - self.assertLess(remained_time, test_constants.SHORT_TIMEOUT * 5 / 2) + self.assertGreater(remained_time, _SHORT_TIMEOUT_S * 3 / 2) + self.assertLess(remained_time, _SHORT_TIMEOUT_S * 5 / 2) response = await call.read() self.assertEqual(_RESPONSE_PAYLOAD_SIZE, len(response.payload.body)) # Should be around the timeout minus a unit of wait time remained_time = call.time_remaining() - self.assertGreater(remained_time, test_constants.SHORT_TIMEOUT / 2) - self.assertLess(remained_time, test_constants.SHORT_TIMEOUT * 3 / 2) + self.assertGreater(remained_time, _SHORT_TIMEOUT_S / 2) + self.assertLess(remained_time, _SHORT_TIMEOUT_S * 3 / 2) self.assertEqual(grpc.StatusCode.OK, await call.code()) @@ -538,14 +538,14 @@ class TestStreamUnaryCall(_MulticallableTestMixin, AioTestBase): with self.assertRaises(asyncio.CancelledError): for _ in range(_NUM_STREAM_RESPONSES): yield request - await asyncio.sleep(test_constants.SHORT_TIMEOUT) + await asyncio.sleep(_SHORT_TIMEOUT_S) request_iterator_received_the_exception.set() call = self._stub.StreamingInputCall(request_iterator()) # Cancel the RPC after at least one response async def cancel_later(): - await asyncio.sleep(test_constants.SHORT_TIMEOUT * 2) + await asyncio.sleep(_SHORT_TIMEOUT_S * 2) call.cancel() cancel_later_task = self.loop.create_task(cancel_later()) @@ -576,6 +576,33 @@ class TestStreamUnaryCall(_MulticallableTestMixin, AioTestBase): self.assertEqual(await call.code(), grpc.StatusCode.OK) + async def test_call_rpc_error(self): + async with aio.insecure_channel(_UNREACHABLE_TARGET) as channel: + stub = test_pb2_grpc.TestServiceStub(channel) + + # The error should be raised automatically without any traffic. + call = stub.StreamingInputCall() + with self.assertRaises(aio.AioRpcError) as exception_context: + await call + + self.assertEqual(grpc.StatusCode.UNAVAILABLE, + exception_context.exception.code()) + + self.assertTrue(call.done()) + self.assertEqual(grpc.StatusCode.UNAVAILABLE, await call.code()) + + async def test_timeout(self): + call = self._stub.StreamingInputCall(timeout=_SHORT_TIMEOUT_S) + + # The error should be raised automatically without any traffic. + with self.assertRaises(aio.AioRpcError) as exception_context: + await call + + rpc_error = exception_context.exception + self.assertEqual(grpc.StatusCode.DEADLINE_EXCEEDED, rpc_error.code()) + self.assertTrue(call.done()) + self.assertEqual(grpc.StatusCode.DEADLINE_EXCEEDED, await call.code()) + # Prepares the request that stream in a ping-pong manner. _STREAM_OUTPUT_REQUEST_ONE_RESPONSE = messages_pb2.StreamingOutputCallRequest() @@ -733,14 +760,14 @@ class TestStreamStreamCall(_MulticallableTestMixin, AioTestBase): with self.assertRaises(asyncio.CancelledError): for _ in range(_NUM_STREAM_RESPONSES): yield request - await asyncio.sleep(test_constants.SHORT_TIMEOUT) + await asyncio.sleep(_SHORT_TIMEOUT_S) request_iterator_received_the_exception.set() call = self._stub.FullDuplexCall(request_iterator()) # Cancel the RPC after at least one response async def cancel_later(): - await asyncio.sleep(test_constants.SHORT_TIMEOUT * 2) + await asyncio.sleep(_SHORT_TIMEOUT_S * 2) call.cancel() cancel_later_task = self.loop.create_task(cancel_later()) diff --git a/src/python/grpcio_tests/tests_aio/unit/try_connect_test.py b/src/python/grpcio_tests/tests_aio/unit/try_connect_test.py new file mode 100644 index 00000000000..61d740e93b7 --- /dev/null +++ b/src/python/grpcio_tests/tests_aio/unit/try_connect_test.py @@ -0,0 +1,114 @@ +# Copyright 2020 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests behavior of the try connect API on client side.""" + +import asyncio +import logging +import unittest +import datetime +from typing import Callable, Tuple + +import grpc +from grpc.experimental import aio + +from tests_aio.unit._test_base import AioTestBase +from tests_aio.unit._test_server import start_test_server +from tests_aio.unit import _common +from src.proto.grpc.testing import messages_pb2, test_pb2_grpc + +_REQUEST = b'\x01\x02\x03' +_UNREACHABLE_TARGET = '0.1:1111' +_TEST_METHOD = '/test/Test' + +_NUM_STREAM_RESPONSES = 5 +_REQUEST_PAYLOAD_SIZE = 7 +_RESPONSE_PAYLOAD_SIZE = 42 + + +class TestTryConnect(AioTestBase): + """Tests if try connect raises connectivity issue.""" + + async def setUp(self): + address, self._server = await start_test_server() + self._channel = aio.insecure_channel(address) + self._dummy_channel = aio.insecure_channel(_UNREACHABLE_TARGET) + self._stub = test_pb2_grpc.TestServiceStub(self._channel) + + async def tearDown(self): + await self._dummy_channel.close() + await self._channel.close() + await self._server.stop(None) + + async def test_unary_stream_ok(self): + request = messages_pb2.StreamingOutputCallRequest() + for _ in range(_NUM_STREAM_RESPONSES): + request.response_parameters.append( + messages_pb2.ResponseParameters(size=_RESPONSE_PAYLOAD_SIZE)) + + call = self._stub.StreamingOutputCall(request) + + # No exception raised and no message swallowed. + await call.try_connect() + + response_cnt = 0 + async for response in call: + response_cnt += 1 + self.assertIs(type(response), + messages_pb2.StreamingOutputCallResponse) + self.assertEqual(_RESPONSE_PAYLOAD_SIZE, len(response.payload.body)) + + self.assertEqual(_NUM_STREAM_RESPONSES, response_cnt) + self.assertEqual(await call.code(), grpc.StatusCode.OK) + + async def test_stream_stream_ok(self): + call = self._stub.FullDuplexCall() + + # No exception raised and no message swallowed. + await call.try_connect() + + request = messages_pb2.StreamingOutputCallRequest() + request.response_parameters.append( + messages_pb2.ResponseParameters(size=_RESPONSE_PAYLOAD_SIZE)) + + for _ in range(_NUM_STREAM_RESPONSES): + await call.write(request) + response = await call.read() + self.assertIsInstance(response, + messages_pb2.StreamingOutputCallResponse) + self.assertEqual(_RESPONSE_PAYLOAD_SIZE, len(response.payload.body)) + + await call.done_writing() + + self.assertEqual(grpc.StatusCode.OK, await call.code()) + + async def test_unary_stream_error(self): + call = self._dummy_channel.unary_stream(_TEST_METHOD)(_REQUEST) + + with self.assertRaises(aio.AioRpcError) as exception_context: + await call.try_connect() + rpc_error = exception_context.exception + self.assertEqual(grpc.StatusCode.UNAVAILABLE, rpc_error.code()) + + async def test_stream_stream_error(self): + call = self._dummy_channel.stream_stream(_TEST_METHOD)() + + with self.assertRaises(aio.AioRpcError) as exception_context: + await call.try_connect() + rpc_error = exception_context.exception + self.assertEqual(grpc.StatusCode.UNAVAILABLE, rpc_error.code()) + + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG) + unittest.main(verbosity=2) From c07520a3507ae491b6a4942b04be35d1724057b1 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Thu, 2 Apr 2020 17:20:58 -0700 Subject: [PATCH 17/73] Mark it as experimental --- src/python/grpcio/grpc/experimental/aio/_base_call.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/python/grpcio/grpc/experimental/aio/_base_call.py b/src/python/grpcio/grpc/experimental/aio/_base_call.py index e5c93e0f607..703e4ca621d 100644 --- a/src/python/grpcio/grpc/experimental/aio/_base_call.py +++ b/src/python/grpcio/grpc/experimental/aio/_base_call.py @@ -162,6 +162,8 @@ class UnaryStreamCall(Generic[RequestType, ResponseType], async def try_connect(self) -> None: """Tries to connect to peer and raise aio.AioRpcError if failed. + This is an EXPERIMENTAL method. + This method is available for RPCs with streaming responses. This method enables the application to ensure if the RPC has been successfully connected. Otherwise, an AioRpcError will be raised to explain the @@ -249,6 +251,8 @@ class StreamStreamCall(Generic[RequestType, ResponseType], async def try_connect(self) -> None: """Tries to connect to peer and raise aio.AioRpcError if failed. + This is an EXPERIMENTAL method. + This method is available for RPCs with streaming responses. This method enables the application to ensure if the RPC has been successfully connected. Otherwise, an AioRpcError will be raised to explain the From 58beda2f73801c920d0b274348bf5fe106fdab3f Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Fri, 3 Apr 2020 12:08:17 -0700 Subject: [PATCH 18/73] Add try_connect API to StreamUnaryCall --- .../grpc/experimental/aio/_base_call.py | 41 +++++++++++++------ .../grpcio/grpc/experimental/aio/_call.py | 10 ++--- .../tests_aio/unit/try_connect_test.py | 28 +++++++++++++ 3 files changed, 62 insertions(+), 17 deletions(-) diff --git a/src/python/grpcio/grpc/experimental/aio/_base_call.py b/src/python/grpcio/grpc/experimental/aio/_base_call.py index 703e4ca621d..e71076fe3d5 100644 --- a/src/python/grpcio/grpc/experimental/aio/_base_call.py +++ b/src/python/grpcio/grpc/experimental/aio/_base_call.py @@ -164,13 +164,13 @@ class UnaryStreamCall(Generic[RequestType, ResponseType], This is an EXPERIMENTAL method. - This method is available for RPCs with streaming responses. This method - enables the application to ensure if the RPC has been successfully - connected. Otherwise, an AioRpcError will be raised to explain the - reason of the connection failure. + This method is available for streaming RPCs. This method enables the + application to ensure if the RPC has been successfully connected. + Otherwise, an AioRpcError will be raised to explain the reason of the + connection failure. - For RPCs with unary response, the connectivity issue will be raised - once the application awaits the call. + For unary-unary RPCs, the connectivity issue will be raised once the + application awaits the call. This method is recommended for building retry mechanisms. """ @@ -204,6 +204,23 @@ class StreamUnaryCall(Generic[RequestType, ResponseType], The response message of the stream. """ + @abstractmethod + async def try_connect(self) -> None: + """Tries to connect to peer and raise aio.AioRpcError if failed. + + This is an EXPERIMENTAL method. + + This method is available for streaming RPCs. This method enables the + application to ensure if the RPC has been successfully connected. + Otherwise, an AioRpcError will be raised to explain the reason of the + connection failure. + + For unary-unary RPCs, the connectivity issue will be raised once the + application awaits the call. + + This method is recommended for building retry mechanisms. + """ + class StreamStreamCall(Generic[RequestType, ResponseType], Call, @@ -253,13 +270,13 @@ class StreamStreamCall(Generic[RequestType, ResponseType], This is an EXPERIMENTAL method. - This method is available for RPCs with streaming responses. This method - enables the application to ensure if the RPC has been successfully - connected. Otherwise, an AioRpcError will be raised to explain the - reason of the connection failure. + This method is available for streaming RPCs. This method enables the + application to ensure if the RPC has been successfully connected. + Otherwise, an AioRpcError will be raised to explain the reason of the + connection failure. - For RPCs with unary response, the connectivity issue will be raised - once the application awaits the call. + For unary-unary RPCs, the connectivity issue will be raised once the + application awaits the call. This method is recommended for building retry mechanisms. """ diff --git a/src/python/grpcio/grpc/experimental/aio/_call.py b/src/python/grpcio/grpc/experimental/aio/_call.py index 07c0d302141..dc9dbfe481b 100644 --- a/src/python/grpcio/grpc/experimental/aio/_call.py +++ b/src/python/grpcio/grpc/experimental/aio/_call.py @@ -458,6 +458,11 @@ class _StreamRequestMixin(Call): self._raise_for_different_style(_APIStyle.READER_WRITER) await self._done_writing() + async def try_connect(self) -> None: + await self._metadata_sent.wait() + if self.done(): + await self._raise_for_status() + class UnaryUnaryCall(_UnaryResponseMixin, Call, _base_call.UnaryUnaryCall): """Object for managing unary-unary RPC calls. @@ -615,8 +620,3 @@ class StreamStreamCall(_StreamRequestMixin, _StreamResponseMixin, Call, if not self.cancelled(): self.cancel() # No need to raise RpcError here, because no one will `await` this task. - - async def try_connect(self) -> None: - await self._metadata_sent.wait() - if self.done(): - await self._raise_for_status() diff --git a/src/python/grpcio_tests/tests_aio/unit/try_connect_test.py b/src/python/grpcio_tests/tests_aio/unit/try_connect_test.py index 61d740e93b7..af4cfa303ff 100644 --- a/src/python/grpcio_tests/tests_aio/unit/try_connect_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/try_connect_test.py @@ -71,6 +71,26 @@ class TestTryConnect(AioTestBase): self.assertEqual(_NUM_STREAM_RESPONSES, response_cnt) self.assertEqual(await call.code(), grpc.StatusCode.OK) + async def test_stream_unary_ok(self): + call = self._stub.StreamingInputCall() + + # No exception raised and no message swallowed. + await call.try_connect() + + payload = messages_pb2.Payload(body=b'\0' * _REQUEST_PAYLOAD_SIZE) + request = messages_pb2.StreamingInputCallRequest(payload=payload) + + for _ in range(_NUM_STREAM_RESPONSES): + await call.write(request) + await call.done_writing() + + response = await call + self.assertIsInstance(response, messages_pb2.StreamingInputCallResponse) + self.assertEqual(_NUM_STREAM_RESPONSES * _REQUEST_PAYLOAD_SIZE, + response.aggregated_payload_size) + + self.assertEqual(await call.code(), grpc.StatusCode.OK) + async def test_stream_stream_ok(self): call = self._stub.FullDuplexCall() @@ -100,6 +120,14 @@ class TestTryConnect(AioTestBase): rpc_error = exception_context.exception self.assertEqual(grpc.StatusCode.UNAVAILABLE, rpc_error.code()) + async def test_stream_unary_error(self): + call = self._dummy_channel.stream_unary(_TEST_METHOD)(_REQUEST) + + with self.assertRaises(aio.AioRpcError) as exception_context: + await call.try_connect() + rpc_error = exception_context.exception + self.assertEqual(grpc.StatusCode.UNAVAILABLE, rpc_error.code()) + async def test_stream_stream_error(self): call = self._dummy_channel.stream_stream(_TEST_METHOD)() From 2bbf0a79f66e975c8be8dafa7fed32c199dbc495 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Fri, 3 Apr 2020 14:08:56 -0700 Subject: [PATCH 19/73] Make pytype happy --- src/python/grpcio_tests/tests_aio/unit/try_connect_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/grpcio_tests/tests_aio/unit/try_connect_test.py b/src/python/grpcio_tests/tests_aio/unit/try_connect_test.py index af4cfa303ff..7fc292ea3a1 100644 --- a/src/python/grpcio_tests/tests_aio/unit/try_connect_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/try_connect_test.py @@ -121,7 +121,7 @@ class TestTryConnect(AioTestBase): self.assertEqual(grpc.StatusCode.UNAVAILABLE, rpc_error.code()) async def test_stream_unary_error(self): - call = self._dummy_channel.stream_unary(_TEST_METHOD)(_REQUEST) + call = self._dummy_channel.stream_unary(_TEST_METHOD)() with self.assertRaises(aio.AioRpcError) as exception_context: await call.try_connect() From 2b6037f113f8bbce209225251efb13d886947f92 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Mon, 6 Apr 2020 16:06:11 -0700 Subject: [PATCH 20/73] Rename to wait_for_conneciton && Add to unary-unary RPC --- .../grpc/experimental/aio/_base_call.py | 64 ++++--------------- .../grpcio/grpc/experimental/aio/_call.py | 25 ++++++-- .../grpc/experimental/aio/_interceptor.py | 7 ++ src/python/grpcio_tests/tests_aio/tests.json | 2 +- .../grpcio_tests/tests_aio/unit/call_test.py | 6 +- ...ct_test.py => wait_for_connection_test.py} | 39 +++++++---- 6 files changed, 70 insertions(+), 73 deletions(-) rename src/python/grpcio_tests/tests_aio/unit/{try_connect_test.py => wait_for_connection_test.py} (79%) diff --git a/src/python/grpcio/grpc/experimental/aio/_base_call.py b/src/python/grpcio/grpc/experimental/aio/_base_call.py index e71076fe3d5..8ea8e90c8b1 100644 --- a/src/python/grpcio/grpc/experimental/aio/_base_call.py +++ b/src/python/grpcio/grpc/experimental/aio/_base_call.py @@ -117,6 +117,19 @@ class Call(RpcContext, metaclass=ABCMeta): The details string of the RPC. """ + @abstractmethod + async def wait_for_connection(self) -> None: + """Waits until connected to peer and raises aio.AioRpcError if failed. + + This is an EXPERIMENTAL method. + + This method makes ensure if the RPC has been successfully connected. + Otherwise, an AioRpcError will be raised to explain the reason of the + connection failure. + + This method is recommended for building retry mechanisms. + """ + class UnaryUnaryCall(Generic[RequestType, ResponseType], Call, @@ -158,23 +171,6 @@ class UnaryStreamCall(Generic[RequestType, ResponseType], stream. """ - @abstractmethod - async def try_connect(self) -> None: - """Tries to connect to peer and raise aio.AioRpcError if failed. - - This is an EXPERIMENTAL method. - - This method is available for streaming RPCs. This method enables the - application to ensure if the RPC has been successfully connected. - Otherwise, an AioRpcError will be raised to explain the reason of the - connection failure. - - For unary-unary RPCs, the connectivity issue will be raised once the - application awaits the call. - - This method is recommended for building retry mechanisms. - """ - class StreamUnaryCall(Generic[RequestType, ResponseType], Call, @@ -204,23 +200,6 @@ class StreamUnaryCall(Generic[RequestType, ResponseType], The response message of the stream. """ - @abstractmethod - async def try_connect(self) -> None: - """Tries to connect to peer and raise aio.AioRpcError if failed. - - This is an EXPERIMENTAL method. - - This method is available for streaming RPCs. This method enables the - application to ensure if the RPC has been successfully connected. - Otherwise, an AioRpcError will be raised to explain the reason of the - connection failure. - - For unary-unary RPCs, the connectivity issue will be raised once the - application awaits the call. - - This method is recommended for building retry mechanisms. - """ - class StreamStreamCall(Generic[RequestType, ResponseType], Call, @@ -263,20 +242,3 @@ class StreamStreamCall(Generic[RequestType, ResponseType], After done_writing is called, any additional invocation to the write function will fail. This function is idempotent. """ - - @abstractmethod - async def try_connect(self) -> None: - """Tries to connect to peer and raise aio.AioRpcError if failed. - - This is an EXPERIMENTAL method. - - This method is available for streaming RPCs. This method enables the - application to ensure if the RPC has been successfully connected. - Otherwise, an AioRpcError will be raised to explain the reason of the - connection failure. - - For unary-unary RPCs, the connectivity issue will be raised once the - application awaits the call. - - This method is recommended for building retry mechanisms. - """ diff --git a/src/python/grpcio/grpc/experimental/aio/_call.py b/src/python/grpcio/grpc/experimental/aio/_call.py index dc9dbfe481b..ab8056e7339 100644 --- a/src/python/grpcio/grpc/experimental/aio/_call.py +++ b/src/python/grpcio/grpc/experimental/aio/_call.py @@ -18,7 +18,7 @@ import enum import inspect import logging from functools import partial -from typing import AsyncIterable, Awaitable, Optional, Tuple +from typing import AsyncIterable, Optional, Tuple import grpc from grpc import _common @@ -250,9 +250,8 @@ class _APIStyle(enum.IntEnum): class _UnaryResponseMixin(Call): _call_response: asyncio.Task - def _init_unary_response_mixin(self, - response_coro: Awaitable[ResponseType]): - self._call_response = self._loop.create_task(response_coro) + def _init_unary_response_mixin(self, response_task: asyncio.Task): + self._call_response = response_task def cancel(self) -> bool: if super().cancel(): @@ -458,7 +457,7 @@ class _StreamRequestMixin(Call): self._raise_for_different_style(_APIStyle.READER_WRITER) await self._done_writing() - async def try_connect(self) -> None: + async def wait_for_connection(self) -> None: await self._metadata_sent.wait() if self.done(): await self._raise_for_status() @@ -470,6 +469,7 @@ class UnaryUnaryCall(_UnaryResponseMixin, Call, _base_call.UnaryUnaryCall): Returned when an instance of `UnaryUnaryMultiCallable` object is called. """ _request: RequestType + _invocation_task: asyncio.Task # pylint: disable=too-many-arguments def __init__(self, request: RequestType, deadline: Optional[float], @@ -483,7 +483,8 @@ class UnaryUnaryCall(_UnaryResponseMixin, Call, _base_call.UnaryUnaryCall): channel.call(method, deadline, credentials, wait_for_ready), metadata, request_serializer, response_deserializer, loop) self._request = request - self._init_unary_response_mixin(self._invoke()) + self._invocation_task = loop.create_task(self._invoke()) + self._init_unary_response_mixin(self._invocation_task) async def _invoke(self) -> ResponseType: serialized_request = _common.serialize(self._request, @@ -505,6 +506,11 @@ class UnaryUnaryCall(_UnaryResponseMixin, Call, _base_call.UnaryUnaryCall): else: return cygrpc.EOF + async def wait_for_connection(self) -> None: + await self._invocation_task + if self.done(): + await self._raise_for_status() + class UnaryStreamCall(_StreamResponseMixin, Call, _base_call.UnaryStreamCall): """Object for managing unary-stream RPC calls. @@ -541,7 +547,7 @@ class UnaryStreamCall(_StreamResponseMixin, Call, _base_call.UnaryStreamCall): self.cancel() raise - async def try_connect(self) -> None: + async def wait_for_connection(self) -> None: await self._send_unary_request_task if self.done(): await self._raise_for_status() @@ -566,8 +572,13 @@ class StreamUnaryCall(_StreamRequestMixin, _UnaryResponseMixin, Call, channel.call(method, deadline, credentials, wait_for_ready), metadata, request_serializer, response_deserializer, loop) +<<<<<<< HEAD self._init_stream_request_mixin(request_iterator) self._init_unary_response_mixin(self._conduct_rpc()) +======= + self._init_stream_request_mixin(request_async_iterator) + self._init_unary_response_mixin(loop.create_task(self._conduct_rpc())) +>>>>>>> Rename to wait_for_conneciton && Add to unary-unary RPC async def _conduct_rpc(self) -> ResponseType: try: diff --git a/src/python/grpcio/grpc/experimental/aio/_interceptor.py b/src/python/grpcio/grpc/experimental/aio/_interceptor.py index 9e99a1b125d..d4aca3ae0fc 100644 --- a/src/python/grpcio/grpc/experimental/aio/_interceptor.py +++ b/src/python/grpcio/grpc/experimental/aio/_interceptor.py @@ -330,6 +330,10 @@ class InterceptedUnaryUnaryCall(_base_call.UnaryUnaryCall): response = yield from call.__await__() return response + async def wait_for_connection(self) -> None: + call = await self._interceptors_task + return await call.wait_for_connection() + class UnaryUnaryCallResponse(_base_call.UnaryUnaryCall): """Final UnaryUnaryCall class finished with a response.""" @@ -374,3 +378,6 @@ class UnaryUnaryCallResponse(_base_call.UnaryUnaryCall): # for telling the interpreter that __await__ is a generator. yield None return self._response + + async def wait_for_connection(self) -> None: + pass diff --git a/src/python/grpcio_tests/tests_aio/tests.json b/src/python/grpcio_tests/tests_aio/tests.json index b76b5b893b4..71f8733f5f9 100644 --- a/src/python/grpcio_tests/tests_aio/tests.json +++ b/src/python/grpcio_tests/tests_aio/tests.json @@ -28,6 +28,6 @@ "unit.server_interceptor_test.TestServerInterceptor", "unit.server_test.TestServer", "unit.timeout_test.TestTimeout", - "unit.try_connect_test.TestTryConnect", + "unit.wait_for_connection_test.TestWaitForConnection", "unit.wait_for_ready_test.TestWaitForReady" ] diff --git a/src/python/grpcio_tests/tests_aio/unit/call_test.py b/src/python/grpcio_tests/tests_aio/unit/call_test.py index b0c126640a9..2548e777783 100644 --- a/src/python/grpcio_tests/tests_aio/unit/call_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/call_test.py @@ -24,6 +24,7 @@ from grpc.experimental import aio from src.proto.grpc.testing import messages_pb2, test_pb2_grpc from tests_aio.unit._test_base import AioTestBase from tests_aio.unit._test_server import start_test_server +from tests_aio.unit._constants import UNREACHABLE_TARGET _SHORT_TIMEOUT_S = datetime.timedelta(seconds=1).total_seconds() @@ -32,7 +33,6 @@ _RESPONSE_PAYLOAD_SIZE = 42 _REQUEST_PAYLOAD_SIZE = 7 _LOCAL_CANCEL_DETAILS_EXPECTATION = 'Locally cancelled by application!' _RESPONSE_INTERVAL_US = int(_SHORT_TIMEOUT_S * 1000 * 1000) -_UNREACHABLE_TARGET = '0.1:1111' _INFINITE_INTERVAL_US = 2**31 - 1 @@ -78,7 +78,7 @@ class TestUnaryUnaryCall(_MulticallableTestMixin, AioTestBase): self.assertIs(response, response_retry) async def test_call_rpc_error(self): - async with aio.insecure_channel(_UNREACHABLE_TARGET) as channel: + async with aio.insecure_channel(UNREACHABLE_TARGET) as channel: stub = test_pb2_grpc.TestServiceStub(channel) call = stub.UnaryCall(messages_pb2.SimpleRequest()) @@ -577,7 +577,7 @@ class TestStreamUnaryCall(_MulticallableTestMixin, AioTestBase): self.assertEqual(await call.code(), grpc.StatusCode.OK) async def test_call_rpc_error(self): - async with aio.insecure_channel(_UNREACHABLE_TARGET) as channel: + async with aio.insecure_channel(UNREACHABLE_TARGET) as channel: stub = test_pb2_grpc.TestServiceStub(channel) # The error should be raised automatically without any traffic. diff --git a/src/python/grpcio_tests/tests_aio/unit/try_connect_test.py b/src/python/grpcio_tests/tests_aio/unit/wait_for_connection_test.py similarity index 79% rename from src/python/grpcio_tests/tests_aio/unit/try_connect_test.py rename to src/python/grpcio_tests/tests_aio/unit/wait_for_connection_test.py index 7fc292ea3a1..cb6f7985290 100644 --- a/src/python/grpcio_tests/tests_aio/unit/try_connect_test.py +++ b/src/python/grpcio_tests/tests_aio/unit/wait_for_connection_test.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Tests behavior of the try connect API on client side.""" +"""Tests behavior of the wait for connection API on client side.""" import asyncio import logging @@ -26,9 +26,9 @@ from tests_aio.unit._test_base import AioTestBase from tests_aio.unit._test_server import start_test_server from tests_aio.unit import _common from src.proto.grpc.testing import messages_pb2, test_pb2_grpc +from tests_aio.unit._constants import UNREACHABLE_TARGET _REQUEST = b'\x01\x02\x03' -_UNREACHABLE_TARGET = '0.1:1111' _TEST_METHOD = '/test/Test' _NUM_STREAM_RESPONSES = 5 @@ -36,13 +36,13 @@ _REQUEST_PAYLOAD_SIZE = 7 _RESPONSE_PAYLOAD_SIZE = 42 -class TestTryConnect(AioTestBase): - """Tests if try connect raises connectivity issue.""" +class TestWaitForConnection(AioTestBase): + """Tests if wait_for_connection raises connectivity issue.""" async def setUp(self): address, self._server = await start_test_server() self._channel = aio.insecure_channel(address) - self._dummy_channel = aio.insecure_channel(_UNREACHABLE_TARGET) + self._dummy_channel = aio.insecure_channel(UNREACHABLE_TARGET) self._stub = test_pb2_grpc.TestServiceStub(self._channel) async def tearDown(self): @@ -50,6 +50,15 @@ class TestTryConnect(AioTestBase): await self._channel.close() await self._server.stop(None) + async def test_unary_unary_ok(self): + call = self._stub.UnaryCall(messages_pb2.SimpleRequest()) + + # No exception raised and no message swallowed. + await call.wait_for_connection() + + response = await call + self.assertIsInstance(response, messages_pb2.SimpleResponse) + async def test_unary_stream_ok(self): request = messages_pb2.StreamingOutputCallRequest() for _ in range(_NUM_STREAM_RESPONSES): @@ -59,7 +68,7 @@ class TestTryConnect(AioTestBase): call = self._stub.StreamingOutputCall(request) # No exception raised and no message swallowed. - await call.try_connect() + await call.wait_for_connection() response_cnt = 0 async for response in call: @@ -75,7 +84,7 @@ class TestTryConnect(AioTestBase): call = self._stub.StreamingInputCall() # No exception raised and no message swallowed. - await call.try_connect() + await call.wait_for_connection() payload = messages_pb2.Payload(body=b'\0' * _REQUEST_PAYLOAD_SIZE) request = messages_pb2.StreamingInputCallRequest(payload=payload) @@ -95,7 +104,7 @@ class TestTryConnect(AioTestBase): call = self._stub.FullDuplexCall() # No exception raised and no message swallowed. - await call.try_connect() + await call.wait_for_connection() request = messages_pb2.StreamingOutputCallRequest() request.response_parameters.append( @@ -112,11 +121,19 @@ class TestTryConnect(AioTestBase): self.assertEqual(grpc.StatusCode.OK, await call.code()) + async def test_unary_unary_error(self): + call = self._dummy_channel.unary_unary(_TEST_METHOD)(_REQUEST) + + with self.assertRaises(aio.AioRpcError) as exception_context: + await call.wait_for_connection() + rpc_error = exception_context.exception + self.assertEqual(grpc.StatusCode.UNAVAILABLE, rpc_error.code()) + async def test_unary_stream_error(self): call = self._dummy_channel.unary_stream(_TEST_METHOD)(_REQUEST) with self.assertRaises(aio.AioRpcError) as exception_context: - await call.try_connect() + await call.wait_for_connection() rpc_error = exception_context.exception self.assertEqual(grpc.StatusCode.UNAVAILABLE, rpc_error.code()) @@ -124,7 +141,7 @@ class TestTryConnect(AioTestBase): call = self._dummy_channel.stream_unary(_TEST_METHOD)() with self.assertRaises(aio.AioRpcError) as exception_context: - await call.try_connect() + await call.wait_for_connection() rpc_error = exception_context.exception self.assertEqual(grpc.StatusCode.UNAVAILABLE, rpc_error.code()) @@ -132,7 +149,7 @@ class TestTryConnect(AioTestBase): call = self._dummy_channel.stream_stream(_TEST_METHOD)() with self.assertRaises(aio.AioRpcError) as exception_context: - await call.try_connect() + await call.wait_for_connection() rpc_error = exception_context.exception self.assertEqual(grpc.StatusCode.UNAVAILABLE, rpc_error.code()) From 4eb596446d6070780bb6b59d6f64ec88a7413f37 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Mon, 6 Apr 2020 16:11:13 -0700 Subject: [PATCH 21/73] Resolve conflict with master --- src/python/grpcio/grpc/experimental/aio/_call.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/python/grpcio/grpc/experimental/aio/_call.py b/src/python/grpcio/grpc/experimental/aio/_call.py index ab8056e7339..3d1d19fd3fa 100644 --- a/src/python/grpcio/grpc/experimental/aio/_call.py +++ b/src/python/grpcio/grpc/experimental/aio/_call.py @@ -572,13 +572,8 @@ class StreamUnaryCall(_StreamRequestMixin, _UnaryResponseMixin, Call, channel.call(method, deadline, credentials, wait_for_ready), metadata, request_serializer, response_deserializer, loop) -<<<<<<< HEAD self._init_stream_request_mixin(request_iterator) - self._init_unary_response_mixin(self._conduct_rpc()) -======= - self._init_stream_request_mixin(request_async_iterator) self._init_unary_response_mixin(loop.create_task(self._conduct_rpc())) ->>>>>>> Rename to wait_for_conneciton && Add to unary-unary RPC async def _conduct_rpc(self) -> ResponseType: try: From 0aa77e410f90d5bf5b6111e79553a4ebff45594e Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 6 Apr 2020 16:40:06 -0700 Subject: [PATCH 22/73] Restrict set of interpreters on MacOS --- tools/run_tests/run_tests.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 8ed4c88f683..9008004cdad 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -866,6 +866,15 @@ class PythonLanguage(object): else: if args.iomgr_platform == 'asyncio': return (python36_config,) + else if os.uname()[0] == 'Darwin': + # NOTE(rbellevi): Testing takes significantly longer on + # MacOS, so we restrict the number of interpreter versions + # tested. + return ( + python27_config, + python36_config, + python37_config, + ) else: return ( python27_config, From 0a56dcabb910b331cad0ab424c9c5ed8c550f39d Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Mon, 6 Apr 2020 21:14:21 -0700 Subject: [PATCH 23/73] Fix cronet transport crash --- .../cronet/transport/cronet_transport.cc | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.cc b/src/core/ext/transport/cronet/transport/cronet_transport.cc index acdc3fc1ff1..f2c4639b3d1 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.cc +++ b/src/core/ext/transport/cronet/transport/cronet_transport.cc @@ -1072,9 +1072,15 @@ static enum e_op_result execute_stream_op(struct op_and_state* oas) { op_can_be_run(stream_op, s, &oas->state, OP_SEND_MESSAGE)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_MESSAGE", oas); stream_state->pending_send_message = false; - if (stream_state->state_callback_received[OP_FAILED]) { + if (stream_state->state_op_done[OP_CANCEL_ERROR]) { + result = NO_ACTION_POSSIBLE; + CRONET_LOG(GPR_DEBUG, "Stream is cancelled."); + } else if (stream_state->state_callback_received[OP_FAILED]) { + result = NO_ACTION_POSSIBLE; + CRONET_LOG(GPR_DEBUG, "Stream failed."); + } else if (stream_state->state_callback_received[OP_SUCCEEDED]) { result = NO_ACTION_POSSIBLE; - CRONET_LOG(GPR_DEBUG, "Stream is either cancelled or failed."); + CRONET_LOG(GPR_DEBUG, "Stream is already finished."); } else { grpc_slice_buffer write_slice_buffer; grpc_slice slice; @@ -1131,9 +1137,15 @@ static enum e_op_result execute_stream_op(struct op_and_state* oas) { op_can_be_run(stream_op, s, &oas->state, OP_SEND_TRAILING_METADATA)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_TRAILING_METADATA", oas); - if (stream_state->state_callback_received[OP_FAILED]) { + if (stream_state->state_op_done[OP_CANCEL_ERROR]) { + result = NO_ACTION_POSSIBLE; + CRONET_LOG(GPR_DEBUG, "Stream is cancelled."); + } else if (stream_state->state_callback_received[OP_FAILED]) { + result = NO_ACTION_POSSIBLE; + CRONET_LOG(GPR_DEBUG, "Stream failed."); + } else if (stream_state->state_callback_received[OP_SUCCEEDED]) { result = NO_ACTION_POSSIBLE; - CRONET_LOG(GPR_DEBUG, "Stream is either cancelled or failed."); + CRONET_LOG(GPR_DEBUG, "Stream is already done."); } else { CRONET_LOG(GPR_DEBUG, "bidirectional_stream_write (%p, 0)", s->cbs); stream_state->state_callback_received[OP_SEND_MESSAGE] = false; From d94f41c40ed63360f37948e1509dfd127301c88f Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Mon, 6 Apr 2020 22:19:33 -0700 Subject: [PATCH 24/73] Fix clang error --- .../cronet/transport/cronet_transport.cc | 24 +++++++------------ 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.cc b/src/core/ext/transport/cronet/transport/cronet_transport.cc index f2c4639b3d1..b3a0c401dbc 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.cc +++ b/src/core/ext/transport/cronet/transport/cronet_transport.cc @@ -1072,15 +1072,11 @@ static enum e_op_result execute_stream_op(struct op_and_state* oas) { op_can_be_run(stream_op, s, &oas->state, OP_SEND_MESSAGE)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_MESSAGE", oas); stream_state->pending_send_message = false; - if (stream_state->state_op_done[OP_CANCEL_ERROR]) { - result = NO_ACTION_POSSIBLE; - CRONET_LOG(GPR_DEBUG, "Stream is cancelled."); - } else if (stream_state->state_callback_received[OP_FAILED]) { - result = NO_ACTION_POSSIBLE; - CRONET_LOG(GPR_DEBUG, "Stream failed."); - } else if (stream_state->state_callback_received[OP_SUCCEEDED]) { + if (stream_state->state_op_done[OP_CANCEL_ERROR] || + stream_state->state_callback_received[OP_FAILED] || + stream_state->state_callback_received[OP_SUCCEEDED]) { result = NO_ACTION_POSSIBLE; - CRONET_LOG(GPR_DEBUG, "Stream is already finished."); + CRONET_LOG(GPR_DEBUG, "Stream is either cancelled, failed or finished"); } else { grpc_slice_buffer write_slice_buffer; grpc_slice slice; @@ -1137,15 +1133,11 @@ static enum e_op_result execute_stream_op(struct op_and_state* oas) { op_can_be_run(stream_op, s, &oas->state, OP_SEND_TRAILING_METADATA)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_TRAILING_METADATA", oas); - if (stream_state->state_op_done[OP_CANCEL_ERROR]) { - result = NO_ACTION_POSSIBLE; - CRONET_LOG(GPR_DEBUG, "Stream is cancelled."); - } else if (stream_state->state_callback_received[OP_FAILED]) { - result = NO_ACTION_POSSIBLE; - CRONET_LOG(GPR_DEBUG, "Stream failed."); - } else if (stream_state->state_callback_received[OP_SUCCEEDED]) { + if (stream_state->state_op_done[OP_CANCEL_ERROR] || + stream_state->state_callback_received[OP_FAILED] || + stream_state->state_callback_received[OP_SUCCEEDED]) { result = NO_ACTION_POSSIBLE; - CRONET_LOG(GPR_DEBUG, "Stream is already done."); + CRONET_LOG(GPR_DEBUG, "Stream is either cancelled, failed or finished"); } else { CRONET_LOG(GPR_DEBUG, "bidirectional_stream_write (%p, 0)", s->cbs); stream_state->state_callback_received[OP_SEND_MESSAGE] = false; From 14f55eb202ec7b9a19abec09639ea4b29c61cad3 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 7 Apr 2020 07:38:21 -0700 Subject: [PATCH 25/73] Temporarily disable client_channel_stress_test for msan and tsan. --- test/cpp/client/BUILD | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/cpp/client/BUILD b/test/cpp/client/BUILD index 011d937bcb8..8f594edfde9 100644 --- a/test/cpp/client/BUILD +++ b/test/cpp/client/BUILD @@ -38,9 +38,15 @@ grpc_cc_test( srcs = ["client_channel_stress_test.cc"], # TODO(jtattermusch): test fails frequently on Win RBE, but passes locally # reenable the tests once it works reliably on Win RBE. + # TODO(roth): Test disabled on msan and tsan due to variable + # duration problem triggered by https://github.com/grpc/grpc/pull/22481. + # Re-enable once the problem is diagnosed and fixed. Tracked + # internally in b/153136407. tags = [ "no_test_android", # fails on android due to "Too many open files". "no_windows", + "nomsan", + "notsan", ], deps = [ "//:gpr", From b518ee3cc83f75220a63f5c1863d170b1cce9ae1 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 7 Apr 2020 07:57:04 -0700 Subject: [PATCH 26/73] Remove size=large, since it didn't help. --- test/cpp/client/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/test/cpp/client/BUILD b/test/cpp/client/BUILD index 8f594edfde9..73b0aa37457 100644 --- a/test/cpp/client/BUILD +++ b/test/cpp/client/BUILD @@ -34,7 +34,6 @@ grpc_cc_test( grpc_cc_test( name = "client_channel_stress_test", - size = "large", srcs = ["client_channel_stress_test.cc"], # TODO(jtattermusch): test fails frequently on Win RBE, but passes locally # reenable the tests once it works reliably on Win RBE. From 8079de2466a3077ff8facfd11cb6401d4b054dbb Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Tue, 7 Apr 2020 10:07:15 -0700 Subject: [PATCH 27/73] Accidentally a word --- tools/run_tests/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 9008004cdad..b1c56762bd1 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -866,7 +866,7 @@ class PythonLanguage(object): else: if args.iomgr_platform == 'asyncio': return (python36_config,) - else if os.uname()[0] == 'Darwin': + elif os.uname()[0] == 'Darwin': # NOTE(rbellevi): Testing takes significantly longer on # MacOS, so we restrict the number of interpreter versions # tested. From dbf3bb31cdc3360c73f7c34c3d1c6a1261236db9 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Tue, 7 Apr 2020 11:09:27 -0700 Subject: [PATCH 28/73] Shellcheck --- tools/run_tests/helper_scripts/build_python.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index a633049510b..cfcb151bcce 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -171,7 +171,7 @@ pip_install_dir() { # third-party build process. This function pipes through only the # minimal environment necessary. pip_install() { - /usr/bin/env -i PATH="$PATH" "$VENV_PYTHON" -m pip install $@ + /usr/bin/env -i PATH="$PATH" "$VENV_PYTHON" -m pip install "$@" } case "$VENV" in From e50c9a8e2e2a09be8ade94f5e5ed873da11d1379 Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Tue, 7 Apr 2020 13:54:21 -0700 Subject: [PATCH 29/73] Fix typo in the comment --- src/python/grpcio/grpc/experimental/aio/_base_call.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/python/grpcio/grpc/experimental/aio/_base_call.py b/src/python/grpcio/grpc/experimental/aio/_base_call.py index 8ea8e90c8b1..214e208c005 100644 --- a/src/python/grpcio/grpc/experimental/aio/_base_call.py +++ b/src/python/grpcio/grpc/experimental/aio/_base_call.py @@ -123,9 +123,9 @@ class Call(RpcContext, metaclass=ABCMeta): This is an EXPERIMENTAL method. - This method makes ensure if the RPC has been successfully connected. - Otherwise, an AioRpcError will be raised to explain the reason of the - connection failure. + This method ensures the RPC has been successfully connected. Otherwise, + an AioRpcError will be raised to explain the reason of the connection + failure. This method is recommended for building retry mechanisms. """ From 3de85634f3525086d13898afddfd6d88390aa1f0 Mon Sep 17 00:00:00 2001 From: Hannah Shi Date: Tue, 7 Apr 2020 20:54:48 +0000 Subject: [PATCH 30/73] add an instruction of setting a flag before c-core make, after #22416 merged --- src/php/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/php/README.md b/src/php/README.md index f064ff0d698..e589408b0c3 100644 --- a/src/php/README.md +++ b/src/php/README.md @@ -58,7 +58,7 @@ $ git clone -b RELEASE_TAG_HERE https://github.com/grpc/grpc ```sh $ cd grpc $ git submodule update --init -$ make +$ EXTRA_DEFINES=GRPC_POSIX_FORK_ALLOW_PTHREAD_ATFORK make $ [sudo] make install ``` From f8f6034d1a0d3ab1fe2471096546bf699878d49c Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 7 Apr 2020 10:03:05 -0700 Subject: [PATCH 31/73] Refactor xds LB policy --- BUILD | 105 +- BUILD.gn | 8 +- CMakeLists.txt | 14 +- Makefile | 12 +- build_autogenerated.yaml | 16 +- config.m4 | 8 +- config.w32 | 8 +- doc/environment_variables.md | 5 +- gRPC-C++.podspec | 2 + gRPC-Core.podspec | 8 +- grpc.gemspec | 7 +- grpc.gyp | 14 +- include/grpc/impl/codegen/grpc_types.h | 17 +- package.xml | 7 +- .../lb_policy/address_filtering.cc | 83 + .../lb_policy/address_filtering.h | 99 + .../lb_policy/priority/priority.cc | 875 ++++++++ .../weighted_target/weighted_target.cc | 722 +++++++ .../client_channel/lb_policy/xds/cds.cc | 65 +- .../client_channel/lb_policy/xds/eds.cc | 1172 +++++++++++ .../client_channel/lb_policy/xds/lrs.cc | 524 +++++ .../client_channel/lb_policy/xds/xds.cc | 1754 ----------------- .../client_channel/lb_policy/xds/xds.h | 3 +- .../client_channel/lb_policy_registry.cc | 2 + .../client_channel/xds/xds_client_stats.h | 18 +- .../plugin_registry/grpc_plugin_registry.cc | 20 +- .../grpc_unsecure_plugin_registry.cc | 20 +- src/python/grpcio/grpc_core_dependencies.py | 6 +- .../client_channel/service_config_test.cc | 8 +- test/cpp/end2end/xds_end2end_test.cc | 8 +- tools/doxygen/Doxyfile.c++.internal | 7 +- tools/doxygen/Doxyfile.core.internal | 7 +- .../grpc_xds_bazel_python_test_in_docker.sh | 2 +- .../linux/grpc_xds_bazel_test_in_docker.sh | 2 +- 34 files changed, 3789 insertions(+), 1839 deletions(-) create mode 100644 src/core/ext/filters/client_channel/lb_policy/address_filtering.cc create mode 100644 src/core/ext/filters/client_channel/lb_policy/address_filtering.h create mode 100644 src/core/ext/filters/client_channel/lb_policy/priority/priority.cc create mode 100644 src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc create mode 100644 src/core/ext/filters/client_channel/lb_policy/xds/eds.cc create mode 100644 src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc delete mode 100644 src/core/ext/filters/client_channel/lb_policy/xds/xds.cc diff --git a/BUILD b/BUILD index 992d1d403cc..81de8203a03 100644 --- a/BUILD +++ b/BUILD @@ -319,8 +319,9 @@ grpc_cc_library( deps = [ "grpc_common", "grpc_lb_policy_cds", + "grpc_lb_policy_eds", "grpc_lb_policy_grpclb", - "grpc_lb_policy_xds", + "grpc_lb_policy_lrs", "grpc_resolver_xds", ], ) @@ -337,8 +338,9 @@ grpc_cc_library( deps = [ "grpc_common", "grpc_lb_policy_cds_secure", + "grpc_lb_policy_eds_secure", "grpc_lb_policy_grpclb_secure", - "grpc_lb_policy_xds_secure", + "grpc_lb_policy_lrs_secure", "grpc_resolver_xds_secure", "grpc_secure", "grpc_transport_chttp2_client_secure", @@ -1023,7 +1025,9 @@ grpc_cc_library( "grpc_deadline_filter", "grpc_client_authority_filter", "grpc_lb_policy_pick_first", + "grpc_lb_policy_priority", "grpc_lb_policy_round_robin", + "grpc_lb_policy_weighted_target", "grpc_client_idle_filter", "grpc_max_age_filter", "grpc_message_size_filter", @@ -1357,41 +1361,75 @@ grpc_cc_library( ) grpc_cc_library( - name = "grpc_lb_policy_xds", + name = "grpc_lb_policy_cds", srcs = [ - "src/core/ext/filters/client_channel/lb_policy/xds/xds.cc", + "src/core/ext/filters/client_channel/lb_policy/xds/cds.cc", + ], + language = "c++", + deps = [ + "grpc_base", + "grpc_client_channel", + "grpc_xds_client", + ], +) + +grpc_cc_library( + name = "grpc_lb_policy_cds_secure", + srcs = [ + "src/core/ext/filters/client_channel/lb_policy/xds/cds.cc", + ], + language = "c++", + deps = [ + "grpc_base", + "grpc_client_channel", + "grpc_xds_client_secure", + ], +) + +grpc_cc_library( + name = "grpc_lb_policy_eds", + srcs = [ + "src/core/ext/filters/client_channel/lb_policy/xds/eds.cc", ], hdrs = [ "src/core/ext/filters/client_channel/lb_policy/xds/xds.h", ], + external_deps = [ + "absl/strings", + ], language = "c++", deps = [ "grpc_base", "grpc_client_channel", + "grpc_lb_address_filtering", "grpc_xds_client", ], ) grpc_cc_library( - name = "grpc_lb_policy_xds_secure", + name = "grpc_lb_policy_eds_secure", srcs = [ - "src/core/ext/filters/client_channel/lb_policy/xds/xds.cc", + "src/core/ext/filters/client_channel/lb_policy/xds/eds.cc", ], hdrs = [ "src/core/ext/filters/client_channel/lb_policy/xds/xds.h", ], + external_deps = [ + "absl/strings", + ], language = "c++", deps = [ "grpc_base", "grpc_client_channel", + "grpc_lb_address_filtering", "grpc_xds_client_secure", ], ) grpc_cc_library( - name = "grpc_lb_policy_cds", + name = "grpc_lb_policy_lrs", srcs = [ - "src/core/ext/filters/client_channel/lb_policy/xds/cds.cc", + "src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc", ], language = "c++", deps = [ @@ -1402,9 +1440,9 @@ grpc_cc_library( ) grpc_cc_library( - name = "grpc_lb_policy_cds_secure", + name = "grpc_lb_policy_lrs_secure", srcs = [ - "src/core/ext/filters/client_channel/lb_policy/xds/cds.cc", + "src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc", ], language = "c++", deps = [ @@ -1414,6 +1452,24 @@ grpc_cc_library( ], ) +grpc_cc_library( + name = "grpc_lb_address_filtering", + srcs = [ + "src/core/ext/filters/client_channel/lb_policy/address_filtering.cc", + ], + hdrs = [ + "src/core/ext/filters/client_channel/lb_policy/address_filtering.h", + ], + external_deps = [ + "absl/strings", + ], + language = "c++", + deps = [ + "grpc_base", + "grpc_client_channel", + ], +) + grpc_cc_library( name = "grpc_lb_subchannel_list", hdrs = [ @@ -1452,6 +1508,35 @@ grpc_cc_library( ], ) +grpc_cc_library( + name = "grpc_lb_policy_priority", + srcs = [ + "src/core/ext/filters/client_channel/lb_policy/priority/priority.cc", + ], + external_deps = [ + "absl/strings", + ], + language = "c++", + deps = [ + "grpc_base", + "grpc_client_channel", + "grpc_lb_address_filtering", + ], +) + +grpc_cc_library( + name = "grpc_lb_policy_weighted_target", + srcs = [ + "src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc", + ], + language = "c++", + deps = [ + "grpc_base", + "grpc_client_channel", + "grpc_lb_address_filtering", + ], +) + grpc_cc_library( name = "lb_server_load_reporting_filter", srcs = [ diff --git a/BUILD.gn b/BUILD.gn index 306f2bfa6d8..129ad2b2639 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -223,6 +223,8 @@ config("grpc_config") { "src/core/ext/filters/client_channel/http_proxy.h", "src/core/ext/filters/client_channel/lb_policy.cc", "src/core/ext/filters/client_channel/lb_policy.h", + "src/core/ext/filters/client_channel/lb_policy/address_filtering.cc", + "src/core/ext/filters/client_channel/lb_policy/address_filtering.h", "src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc", "src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h", "src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc", @@ -238,10 +240,13 @@ config("grpc_config") { "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc", "src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h", "src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc", + "src/core/ext/filters/client_channel/lb_policy/priority/priority.cc", "src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc", "src/core/ext/filters/client_channel/lb_policy/subchannel_list.h", + "src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc", "src/core/ext/filters/client_channel/lb_policy/xds/cds.cc", - "src/core/ext/filters/client_channel/lb_policy/xds/xds.cc", + "src/core/ext/filters/client_channel/lb_policy/xds/eds.cc", + "src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc", "src/core/ext/filters/client_channel/lb_policy/xds/xds.h", "src/core/ext/filters/client_channel/lb_policy_factory.h", "src/core/ext/filters/client_channel/lb_policy_registry.cc", @@ -962,6 +967,7 @@ config("grpc_config") { ":address_sorting", ":upb", ":absl/types:optional", + ":absl/strings:strings", ":absl/container:inlined_vector", "//third_party/cares", ":address_sorting", diff --git a/CMakeLists.txt b/CMakeLists.txt index 4496b174ba6..27bf340b65a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1315,6 +1315,7 @@ add_library(grpc src/core/ext/filters/client_channel/http_connect_handshaker.cc src/core/ext/filters/client_channel/http_proxy.cc src/core/ext/filters/client_channel/lb_policy.cc + src/core/ext/filters/client_channel/lb_policy/address_filtering.cc src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -1323,9 +1324,12 @@ add_library(grpc src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc + src/core/ext/filters/client_channel/lb_policy/priority/priority.cc src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc + src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc src/core/ext/filters/client_channel/lb_policy/xds/cds.cc - src/core/ext/filters/client_channel/lb_policy/xds/xds.cc + src/core/ext/filters/client_channel/lb_policy/xds/eds.cc + src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc src/core/ext/filters/client_channel/lb_policy_registry.cc src/core/ext/filters/client_channel/local_subchannel_pool.cc src/core/ext/filters/client_channel/parse_address.cc @@ -1743,6 +1747,7 @@ target_link_libraries(grpc address_sorting upb absl::optional + absl::strings absl::inlined_vector ) if(_gRPC_PLATFORM_IOS OR _gRPC_PLATFORM_MAC) @@ -1969,6 +1974,7 @@ add_library(grpc_unsecure src/core/ext/filters/client_channel/http_connect_handshaker.cc src/core/ext/filters/client_channel/http_proxy.cc src/core/ext/filters/client_channel/lb_policy.cc + src/core/ext/filters/client_channel/lb_policy/address_filtering.cc src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -1977,9 +1983,12 @@ add_library(grpc_unsecure src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc + src/core/ext/filters/client_channel/lb_policy/priority/priority.cc src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc + src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc src/core/ext/filters/client_channel/lb_policy/xds/cds.cc - src/core/ext/filters/client_channel/lb_policy/xds/xds.cc + src/core/ext/filters/client_channel/lb_policy/xds/eds.cc + src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc src/core/ext/filters/client_channel/lb_policy_registry.cc src/core/ext/filters/client_channel/local_subchannel_pool.cc src/core/ext/filters/client_channel/parse_address.cc @@ -2321,6 +2330,7 @@ target_link_libraries(grpc_unsecure address_sorting upb absl::optional + absl::strings absl::inlined_vector ) if(_gRPC_PLATFORM_IOS OR _gRPC_PLATFORM_MAC) diff --git a/Makefile b/Makefile index 465c2e3a284..d1a58c87a8b 100644 --- a/Makefile +++ b/Makefile @@ -3640,6 +3640,7 @@ LIBGRPC_SRC = \ src/core/ext/filters/client_channel/http_connect_handshaker.cc \ src/core/ext/filters/client_channel/http_proxy.cc \ src/core/ext/filters/client_channel/lb_policy.cc \ + src/core/ext/filters/client_channel/lb_policy/address_filtering.cc \ src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc \ @@ -3648,9 +3649,12 @@ LIBGRPC_SRC = \ src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc \ src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc \ + src/core/ext/filters/client_channel/lb_policy/priority/priority.cc \ src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc \ + src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc \ src/core/ext/filters/client_channel/lb_policy/xds/cds.cc \ - src/core/ext/filters/client_channel/lb_policy/xds/xds.cc \ + src/core/ext/filters/client_channel/lb_policy/xds/eds.cc \ + src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc \ src/core/ext/filters/client_channel/lb_policy_registry.cc \ src/core/ext/filters/client_channel/local_subchannel_pool.cc \ src/core/ext/filters/client_channel/parse_address.cc \ @@ -4269,6 +4273,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/ext/filters/client_channel/http_connect_handshaker.cc \ src/core/ext/filters/client_channel/http_proxy.cc \ src/core/ext/filters/client_channel/lb_policy.cc \ + src/core/ext/filters/client_channel/lb_policy/address_filtering.cc \ src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc \ @@ -4277,9 +4282,12 @@ LIBGRPC_UNSECURE_SRC = \ src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc \ src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc \ + src/core/ext/filters/client_channel/lb_policy/priority/priority.cc \ src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc \ + src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc \ src/core/ext/filters/client_channel/lb_policy/xds/cds.cc \ - src/core/ext/filters/client_channel/lb_policy/xds/xds.cc \ + src/core/ext/filters/client_channel/lb_policy/xds/eds.cc \ + src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc \ src/core/ext/filters/client_channel/lb_policy_registry.cc \ src/core/ext/filters/client_channel/local_subchannel_pool.cc \ src/core/ext/filters/client_channel/parse_address.cc \ diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 9a188b41121..69e0b124799 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -382,6 +382,7 @@ libs: - src/core/ext/filters/client_channel/http_connect_handshaker.h - src/core/ext/filters/client_channel/http_proxy.h - src/core/ext/filters/client_channel/lb_policy.h + - src/core/ext/filters/client_channel/lb_policy/address_filtering.h - src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h - src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h @@ -740,6 +741,7 @@ libs: - src/core/ext/filters/client_channel/http_connect_handshaker.cc - src/core/ext/filters/client_channel/http_proxy.cc - src/core/ext/filters/client_channel/lb_policy.cc + - src/core/ext/filters/client_channel/lb_policy/address_filtering.cc - src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc - src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -748,9 +750,12 @@ libs: - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc - src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc - src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc + - src/core/ext/filters/client_channel/lb_policy/priority/priority.cc - src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc + - src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc - src/core/ext/filters/client_channel/lb_policy/xds/cds.cc - - src/core/ext/filters/client_channel/lb_policy/xds/xds.cc + - src/core/ext/filters/client_channel/lb_policy/xds/eds.cc + - src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc - src/core/ext/filters/client_channel/lb_policy_registry.cc - src/core/ext/filters/client_channel/local_subchannel_pool.cc - src/core/ext/filters/client_channel/parse_address.cc @@ -1132,6 +1137,7 @@ libs: - address_sorting - upb - absl/types:optional + - absl/strings:strings - absl/container:inlined_vector baselib: true dll: true @@ -1278,6 +1284,7 @@ libs: - src/core/ext/filters/client_channel/http_connect_handshaker.h - src/core/ext/filters/client_channel/http_proxy.h - src/core/ext/filters/client_channel/lb_policy.h + - src/core/ext/filters/client_channel/lb_policy/address_filtering.h - src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h - src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h @@ -1571,6 +1578,7 @@ libs: - src/core/ext/filters/client_channel/http_connect_handshaker.cc - src/core/ext/filters/client_channel/http_proxy.cc - src/core/ext/filters/client_channel/lb_policy.cc + - src/core/ext/filters/client_channel/lb_policy/address_filtering.cc - src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc - src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -1579,9 +1587,12 @@ libs: - src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc - src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc - src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc + - src/core/ext/filters/client_channel/lb_policy/priority/priority.cc - src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc + - src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc - src/core/ext/filters/client_channel/lb_policy/xds/cds.cc - - src/core/ext/filters/client_channel/lb_policy/xds/xds.cc + - src/core/ext/filters/client_channel/lb_policy/xds/eds.cc + - src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc - src/core/ext/filters/client_channel/lb_policy_registry.cc - src/core/ext/filters/client_channel/local_subchannel_pool.cc - src/core/ext/filters/client_channel/parse_address.cc @@ -1888,6 +1899,7 @@ libs: - address_sorting - upb - absl/types:optional + - absl/strings:strings - absl/container:inlined_vector baselib: true dll: true diff --git a/config.m4 b/config.m4 index 66922ad5527..6b450315f4f 100644 --- a/config.m4 +++ b/config.m4 @@ -50,6 +50,7 @@ if test "$PHP_GRPC" != "no"; then src/core/ext/filters/client_channel/http_connect_handshaker.cc \ src/core/ext/filters/client_channel/http_proxy.cc \ src/core/ext/filters/client_channel/lb_policy.cc \ + src/core/ext/filters/client_channel/lb_policy/address_filtering.cc \ src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc \ @@ -58,9 +59,12 @@ if test "$PHP_GRPC" != "no"; then src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc \ src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc \ + src/core/ext/filters/client_channel/lb_policy/priority/priority.cc \ src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc \ + src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc \ src/core/ext/filters/client_channel/lb_policy/xds/cds.cc \ - src/core/ext/filters/client_channel/lb_policy/xds/xds.cc \ + src/core/ext/filters/client_channel/lb_policy/xds/eds.cc \ + src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc \ src/core/ext/filters/client_channel/lb_policy_registry.cc \ src/core/ext/filters/client_channel/local_subchannel_pool.cc \ src/core/ext/filters/client_channel/parse_address.cc \ @@ -821,7 +825,9 @@ if test "$PHP_GRPC" != "no"; then PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy/grpclb) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy/pick_first) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy/priority) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy/round_robin) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy/weighted_target) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy/xds) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/resolver/dns) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/resolver/dns/c_ares) diff --git a/config.w32 b/config.w32 index 541cc74b602..59cdbdb1ce4 100644 --- a/config.w32 +++ b/config.w32 @@ -19,6 +19,7 @@ if (PHP_GRPC != "no") { "src\\core\\ext\\filters\\client_channel\\http_connect_handshaker.cc " + "src\\core\\ext\\filters\\client_channel\\http_proxy.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy.cc " + + "src\\core\\ext\\filters\\client_channel\\lb_policy\\address_filtering.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\child_policy_handler.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\grpclb\\client_load_reporting_filter.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\grpclb\\grpclb.cc " + @@ -27,9 +28,12 @@ if (PHP_GRPC != "no") { "src\\core\\ext\\filters\\client_channel\\lb_policy\\grpclb\\grpclb_client_stats.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\grpclb\\load_balancer_api.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\pick_first\\pick_first.cc " + + "src\\core\\ext\\filters\\client_channel\\lb_policy\\priority\\priority.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\round_robin\\round_robin.cc " + + "src\\core\\ext\\filters\\client_channel\\lb_policy\\weighted_target\\weighted_target.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy\\xds\\cds.cc " + - "src\\core\\ext\\filters\\client_channel\\lb_policy\\xds\\xds.cc " + + "src\\core\\ext\\filters\\client_channel\\lb_policy\\xds\\eds.cc " + + "src\\core\\ext\\filters\\client_channel\\lb_policy\\xds\\lrs.cc " + "src\\core\\ext\\filters\\client_channel\\lb_policy_registry.cc " + "src\\core\\ext\\filters\\client_channel\\local_subchannel_pool.cc " + "src\\core\\ext\\filters\\client_channel\\parse_address.cc " + @@ -821,7 +825,9 @@ if (PHP_GRPC != "no") { FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\lb_policy"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\lb_policy\\grpclb"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\lb_policy\\pick_first"); + FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\lb_policy\\priority"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\lb_policy\\round_robin"); + FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\lb_policy\\weighted_target"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\lb_policy\\xds"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\resolver"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\resolver\\dns"); diff --git a/doc/environment_variables.md b/doc/environment_variables.md index e79c390c9a3..ab45f937bff 100644 --- a/doc/environment_variables.md +++ b/doc/environment_variables.md @@ -57,6 +57,7 @@ some configuration as environment variables that can be set. - compression - traces compression operations - connectivity_state - traces connectivity state changes to channels - cronet - traces state in the cronet transport engine + - eds_lb - traces eds LB policy - executor - traces grpc's internal thread pool ('the executor') - glb - traces the grpclb load balancer - handshaker - traces handshaking state @@ -66,12 +67,14 @@ some configuration as environment variables that can be set. - http1 - traces HTTP/1.x operations performed by gRPC - inproc - traces the in-process transport - flowctl - traces http2 flow control + - lrs_lb - traces lrs LB policy - op_failure - traces error information when failure is pushed onto a completion queue - pick_first - traces the pick first load balancing policy - plugin_credentials - traces plugin credentials - pollable_refcount - traces reference counting of 'pollable' objects (only in DEBUG) + - priority_lb - traces priority LB policy - resource_quota - trace resource quota objects internals - round_robin - traces the round_robin load balancing policy - queue_pluck @@ -84,8 +87,8 @@ some configuration as environment variables that can be set. - transport_security - traces metadata about secure channel establishment - tcp - traces bytes in and out of a channel - tsi - traces tsi transport security + - weighted_target_lb - traces weighted_target LB policy - xds_client - traces xds client - - xds_lb - traces xds LB policy - xds_resolver - traces xds resolver The following tracers will only run in binaries built in DEBUG mode. This is diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index c8b176b23ea..77980999b59 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -233,6 +233,7 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/http_connect_handshaker.h', 'src/core/ext/filters/client_channel/http_proxy.h', 'src/core/ext/filters/client_channel/lb_policy.h', + 'src/core/ext/filters/client_channel/lb_policy/address_filtering.h', 'src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h', @@ -683,6 +684,7 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/http_connect_handshaker.h', 'src/core/ext/filters/client_channel/http_proxy.h', 'src/core/ext/filters/client_channel/lb_policy.h', + 'src/core/ext/filters/client_channel/lb_policy/address_filtering.h', 'src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index d140b005d7f..749e77b3fa7 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -206,6 +206,8 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/http_proxy.h', 'src/core/ext/filters/client_channel/lb_policy.cc', 'src/core/ext/filters/client_channel/lb_policy.h', + 'src/core/ext/filters/client_channel/lb_policy/address_filtering.cc', + 'src/core/ext/filters/client_channel/lb_policy/address_filtering.h', 'src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc', 'src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc', @@ -221,10 +223,13 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h', 'src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc', + 'src/core/ext/filters/client_channel/lb_policy/priority/priority.cc', 'src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc', 'src/core/ext/filters/client_channel/lb_policy/subchannel_list.h', + 'src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc', 'src/core/ext/filters/client_channel/lb_policy/xds/cds.cc', - 'src/core/ext/filters/client_channel/lb_policy/xds/xds.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/eds.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc', 'src/core/ext/filters/client_channel/lb_policy/xds/xds.h', 'src/core/ext/filters/client_channel/lb_policy_factory.h', 'src/core/ext/filters/client_channel/lb_policy_registry.cc', @@ -1032,6 +1037,7 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/http_connect_handshaker.h', 'src/core/ext/filters/client_channel/http_proxy.h', 'src/core/ext/filters/client_channel/lb_policy.h', + 'src/core/ext/filters/client_channel/lb_policy/address_filtering.h', 'src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.h', 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.h', diff --git a/grpc.gemspec b/grpc.gemspec index c9a1a8835fd..29e5d3d436b 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -128,6 +128,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/filters/client_channel/http_proxy.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy.cc ) s.files += %w( src/core/ext/filters/client_channel/lb_policy.h ) + s.files += %w( src/core/ext/filters/client_channel/lb_policy/address_filtering.cc ) + s.files += %w( src/core/ext/filters/client_channel/lb_policy/address_filtering.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc ) @@ -143,10 +145,13 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc ) + s.files += %w( src/core/ext/filters/client_channel/lb_policy/priority/priority.cc ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/subchannel_list.h ) + s.files += %w( src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/cds.cc ) - s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/xds.cc ) + s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/eds.cc ) + s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc ) s.files += %w( src/core/ext/filters/client_channel/lb_policy/xds/xds.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy_factory.h ) s.files += %w( src/core/ext/filters/client_channel/lb_policy_registry.cc ) diff --git a/grpc.gyp b/grpc.gyp index b065ee6f2af..a62c034d65a 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -426,6 +426,7 @@ 'address_sorting', 'upb', 'absl/types:optional', + 'absl/strings:strings', 'absl/container:inlined_vector', ], 'sources': [ @@ -442,6 +443,7 @@ 'src/core/ext/filters/client_channel/http_connect_handshaker.cc', 'src/core/ext/filters/client_channel/http_proxy.cc', 'src/core/ext/filters/client_channel/lb_policy.cc', + 'src/core/ext/filters/client_channel/lb_policy/address_filtering.cc', 'src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc', @@ -450,9 +452,12 @@ 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc', 'src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc', + 'src/core/ext/filters/client_channel/lb_policy/priority/priority.cc', 'src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc', + 'src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc', 'src/core/ext/filters/client_channel/lb_policy/xds/cds.cc', - 'src/core/ext/filters/client_channel/lb_policy/xds/xds.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/eds.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc', 'src/core/ext/filters/client_channel/lb_policy_registry.cc', 'src/core/ext/filters/client_channel/local_subchannel_pool.cc', 'src/core/ext/filters/client_channel/parse_address.cc', @@ -916,6 +921,7 @@ 'address_sorting', 'upb', 'absl/types:optional', + 'absl/strings:strings', 'absl/container:inlined_vector', ], 'sources': [ @@ -932,6 +938,7 @@ 'src/core/ext/filters/client_channel/http_connect_handshaker.cc', 'src/core/ext/filters/client_channel/http_proxy.cc', 'src/core/ext/filters/client_channel/lb_policy.cc', + 'src/core/ext/filters/client_channel/lb_policy/address_filtering.cc', 'src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc', @@ -940,9 +947,12 @@ 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc', 'src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc', + 'src/core/ext/filters/client_channel/lb_policy/priority/priority.cc', 'src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc', + 'src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc', 'src/core/ext/filters/client_channel/lb_policy/xds/cds.cc', - 'src/core/ext/filters/client_channel/lb_policy/xds/xds.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/eds.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc', 'src/core/ext/filters/client_channel/lb_policy_registry.cc', 'src/core/ext/filters/client_channel/local_subchannel_pool.cc', 'src/core/ext/filters/client_channel/parse_address.cc', diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index ab4c39f9310..ff45450f3a3 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -348,18 +348,11 @@ typedef struct { balancer before using fallback backend addresses from the resolver. If 0, enter fallback mode immediately. Default value is 10000. */ #define GRPC_ARG_XDS_FALLBACK_TIMEOUT_MS "grpc.xds_fallback_timeout_ms" -/* Time in milliseconds to wait before a locality is deleted after it's removed - from the received EDS update. If 0, delete the locality immediately. Default - value is 15 minutes. */ -#define GRPC_ARG_LOCALITY_RETENTION_INTERVAL_MS \ - "grpc.xds_locality_retention_interval_ms" -/* Timeout in milliseconds to wait for the localities of a specific priority to - complete their initial connection attempt before xDS fails over to the next - priority. Specifically, the connection attempt of a priority is considered - completed when any locality of that priority is ready or all the localities - of that priority fail to connect. If 0, failover happens immediately. Default - value is 10 seconds. */ -#define GRPC_ARG_XDS_FAILOVER_TIMEOUT_MS "grpc.xds_failover_timeout_ms" +/* Timeout in milliseconds to wait for the child of a specific priority to + complete its initial connection attempt before the priority LB policy fails + over to the next priority. Default value is 10 seconds. */ +#define GRPC_ARG_PRIORITY_FAILOVER_TIMEOUT_MS \ + "grpc.priority_failover_timeout_ms" /* Timeout in milliseconds to wait for a resource to be returned from * the xds server before assuming that it does not exist. * The default is 15 seconds. */ diff --git a/package.xml b/package.xml index 32930b6f65c..06d1669ec48 100644 --- a/package.xml +++ b/package.xml @@ -108,6 +108,8 @@ + + @@ -123,10 +125,13 @@ + + - + + diff --git a/src/core/ext/filters/client_channel/lb_policy/address_filtering.cc b/src/core/ext/filters/client_channel/lb_policy/address_filtering.cc new file mode 100644 index 00000000000..67843df78e5 --- /dev/null +++ b/src/core/ext/filters/client_channel/lb_policy/address_filtering.cc @@ -0,0 +1,83 @@ +// +// Copyright 2020 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include + +#include "src/core/ext/filters/client_channel/lb_policy/address_filtering.h" + +#include "src/core/lib/channel/channel_args.h" + +#define GRPC_ARG_HIERARCHICAL_PATH "grpc.internal.address.hierarchical_path" + +namespace grpc_core { + +namespace { + +void* HierarchicalPathCopy(void* p) { + std::vector* path = static_cast*>(p); + return static_cast(new std::vector(*path)); +} + +void HierarchicalPathDestroy(void* p) { + std::vector* path = static_cast*>(p); + delete path; +} + +int HierarchicalPathCompare(void* p1, void* p2) { + std::vector* path1 = static_cast*>(p1); + std::vector* path2 = static_cast*>(p2); + for (size_t i = 0; i < path1->size(); ++i) { + if (path2->size() == i) return 1; + int r = (*path1)[i].compare((*path2)[i]); + if (r != 0) return r; + } + if (path2->size() > path1->size()) return -1; + return 0; +} + +const grpc_arg_pointer_vtable hierarchical_path_arg_vtable = { + HierarchicalPathCopy, HierarchicalPathDestroy, HierarchicalPathCompare}; + +} // namespace + +grpc_arg MakeHierarchicalPathArg(const std::vector& path) { + return grpc_channel_arg_pointer_create( + const_cast(GRPC_ARG_HIERARCHICAL_PATH), + const_cast*>(&path), + &hierarchical_path_arg_vtable); +} + +HierarchicalAddressMap MakeHierarchicalAddressMap( + const ServerAddressList& addresses) { + HierarchicalAddressMap result; + for (const ServerAddress& address : addresses) { + auto* path = grpc_channel_args_find_pointer>( + address.args(), GRPC_ARG_HIERARCHICAL_PATH); + if (path == nullptr || path->empty()) continue; + auto it = path->begin(); + ServerAddressList& target_list = result[*it]; + ++it; + std::vector remaining_path(it, path->end()); + const char* name_to_remove = GRPC_ARG_HIERARCHICAL_PATH; + grpc_arg new_arg = MakeHierarchicalPathArg(remaining_path); + grpc_channel_args* new_args = grpc_channel_args_copy_and_add_and_remove( + address.args(), &name_to_remove, 1, &new_arg, 1); + target_list.emplace_back(address.address(), new_args); + } + return result; +} + +} // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/lb_policy/address_filtering.h b/src/core/ext/filters/client_channel/lb_policy/address_filtering.h new file mode 100644 index 00000000000..03a1c228e7a --- /dev/null +++ b/src/core/ext/filters/client_channel/lb_policy/address_filtering.h @@ -0,0 +1,99 @@ +// +// Copyright 2020 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_ADDRESS_FILTERING_H +#define GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_ADDRESS_FILTERING_H + +#include + +#include +#include +#include + +#include "absl/strings/string_view.h" + +#include "src/core/ext/filters/client_channel/server_address.h" + +// The resolver returns a flat list of addresses. When a hierarchy of +// LB policies is in use, each leaf of the hierarchy will need a +// different subset of those addresses. This library provides a +// mechanism for determining which address is passed to which leaf +// policy. +// +// Each address will have an associated path that indicates which child +// it should be sent to at each level of the hierarchy to wind up at the +// right leaf policy. Each LB policy will look at the first element of +// the path of each address to determine which child to send the address +// to. It will then remove that first element when passing the address +// down to its child. +// +// For example, consider the following LB policy hierarchy: +// +// - priority +// - child0 (weighted_target) +// - localityA (round_robin) +// - localityB (round_robin) +// - child1 (weighted_target) +// - localityC (round_robin) +// - localityD (round_robin) +// +// Now consider the following addresses: +// - 10.0.0.1:80 path=["child0", "localityA"] +// - 10.0.0.2:80 path=["child0", "localityB"] +// - 10.0.0.3:80 path=["child1", "localityC"] +// - 10.0.0.4:80 path=["child1", "localityD"] +// +// The priority policy will split this up into two lists, one for each +// of its children: +// - child0: +// - 10.0.0.1:80 path=["localityA"] +// - 10.0.0.2:80 path=["localityB"] +// - child1: +// - 10.0.0.3:80 path=["localityC"] +// - 10.0.0.4:80 path=["localityD"] +// +// The weighted_target policy for child0 will split its list up into two +// lists, one for each of its children: +// - localityA: +// - 10.0.0.1:80 path=[] +// - localityB: +// - 10.0.0.2:80 path=[] +// +// Similarly, the weighted_target policy for child1 will split its list +// up into two lists, one for each of its children: +// - localityC: +// - 10.0.0.3:80 path=[] +// - localityD: +// - 10.0.0.4:80 path=[] + +namespace grpc_core { + +// Constructs a channel arg containing the hierarchical path +// to be associated with an address. +grpc_arg MakeHierarchicalPathArg(const std::vector& path); + +// A map from the next path element to the addresses that fall under +// that path element. +using HierarchicalAddressMap = std::map; + +// Splits up the addresses into a separate list for each child. +HierarchicalAddressMap MakeHierarchicalAddressMap( + const ServerAddressList& addresses); + +} // namespace grpc_core + +#endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_ADDRESS_FILTERING_H \ + */ diff --git a/src/core/ext/filters/client_channel/lb_policy/priority/priority.cc b/src/core/ext/filters/client_channel/lb_policy/priority/priority.cc new file mode 100644 index 00000000000..ff69112387c --- /dev/null +++ b/src/core/ext/filters/client_channel/lb_policy/priority/priority.cc @@ -0,0 +1,875 @@ +// +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include + +#include +#include + +#include "absl/strings/str_cat.h" + +#include + +#include "src/core/ext/filters/client_channel/lb_policy.h" +#include "src/core/ext/filters/client_channel/lb_policy/address_filtering.h" +#include "src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h" +#include "src/core/ext/filters/client_channel/lb_policy_factory.h" +#include "src/core/ext/filters/client_channel/lb_policy_registry.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/gprpp/orphanable.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/iomgr/combiner.h" +#include "src/core/lib/iomgr/timer.h" + +namespace grpc_core { + +TraceFlag grpc_lb_priority_trace(false, "priority_lb"); + +namespace { + +constexpr char kPriority[] = "priority_experimental"; + +// How long we keep a child around for after it is no longer being used +// (either because it has been removed from the config or because we +// have switched to a higher-priority child). +constexpr int kChildRetentionIntervalMs = 15 * 60 * 1000; + +// Default for how long we wait for a newly created child to get connected +// before starting to attempt the next priority. Overridable via channel arg. +constexpr int kDefaultChildFailoverTimeoutMs = 10000; + +// Config for priority LB policy. +class PriorityLbConfig : public LoadBalancingPolicy::Config { + public: + PriorityLbConfig( + std::map> + children, + std::vector priorities) + : children_(std::move(children)), priorities_(std::move(priorities)) {} + + const char* name() const override { return kPriority; } + + const std::map>& + children() const { + return children_; + } + const std::vector& priorities() const { return priorities_; } + + private: + const std::map> + children_; + const std::vector priorities_; +}; + +// priority LB policy. +class PriorityLb : public LoadBalancingPolicy { + public: + explicit PriorityLb(Args args); + + const char* name() const override { return kPriority; } + + void UpdateLocked(UpdateArgs args) override; + void ExitIdleLocked() override; + void ResetBackoffLocked() override; + + private: + // Each ChildPriority holds a ref to the PriorityLb. + class ChildPriority : public InternallyRefCounted { + public: + ChildPriority(RefCountedPtr priority_policy, std::string name); + + ~ChildPriority() { + priority_policy_.reset(DEBUG_LOCATION, "ChildPriority"); + } + + const std::string& name() const { return name_; } + + void UpdateLocked(RefCountedPtr config); + void ExitIdleLocked(); + void ResetBackoffLocked(); + void DeactivateLocked(); + void MaybeReactivateLocked(); + void MaybeCancelFailoverTimerLocked(); + + void Orphan() override; + + std::unique_ptr GetPicker() { + return absl::make_unique(picker_wrapper_); + } + + grpc_connectivity_state connectivity_state() const { + return connectivity_state_; + } + bool failover_timer_callback_pending() const { + return failover_timer_callback_pending_; + } + + private: + // A simple wrapper for ref-counting a picker from the child policy. + class RefCountedPicker : public RefCounted { + public: + explicit RefCountedPicker(std::unique_ptr picker) + : picker_(std::move(picker)) {} + PickResult Pick(PickArgs args) { return picker_->Pick(args); } + + private: + std::unique_ptr picker_; + }; + + // A non-ref-counted wrapper for RefCountedPicker. + class RefCountedPickerWrapper : public SubchannelPicker { + public: + explicit RefCountedPickerWrapper(RefCountedPtr picker) + : picker_(std::move(picker)) {} + PickResult Pick(PickArgs args) override { return picker_->Pick(args); } + + private: + RefCountedPtr picker_; + }; + + class Helper : public ChannelControlHelper { + public: + explicit Helper(RefCountedPtr priority) + : priority_(std::move(priority)) {} + + ~Helper() { priority_.reset(DEBUG_LOCATION, "Helper"); } + + RefCountedPtr CreateSubchannel( + const grpc_channel_args& args) override; + void UpdateState(grpc_connectivity_state state, + std::unique_ptr picker) override; + void RequestReresolution() override; + void AddTraceEvent(TraceSeverity severity, StringView message) override; + + private: + RefCountedPtr priority_; + }; + + // Methods for dealing with the child policy. + OrphanablePtr CreateChildPolicyLocked( + const grpc_channel_args* args); + + void OnConnectivityStateUpdateLocked( + grpc_connectivity_state state, + std::unique_ptr picker); + + void StartFailoverTimerLocked(); + + static void OnFailoverTimer(void* arg, grpc_error* error); + static void OnFailoverTimerLocked(void* arg, grpc_error* error); + static void OnDeactivationTimer(void* arg, grpc_error* error); + static void OnDeactivationTimerLocked(void* arg, grpc_error* error); + + RefCountedPtr priority_policy_; + const std::string name_; + + OrphanablePtr child_policy_; + + grpc_connectivity_state connectivity_state_ = GRPC_CHANNEL_CONNECTING; + RefCountedPtr picker_wrapper_; + + // States for delayed removal. + grpc_timer deactivation_timer_; + grpc_closure on_deactivation_timer_; + bool deactivation_timer_callback_pending_ = false; + + // States of failover. + grpc_timer failover_timer_; + grpc_closure on_failover_timer_; + grpc_closure on_failover_timer_locked_; + bool failover_timer_callback_pending_ = false; + }; + + ~PriorityLb(); + + void ShutdownLocked() override; + + // Returns UINT32_MAX if child is not in current priority list. + uint32_t GetChildPriorityLocked(const std::string& child_name) const; + + void HandleChildConnectivityStateChangeLocked(ChildPriority* child); + void DeleteChild(ChildPriority* child); + + void TryNextPriorityLocked(bool report_connecting); + void SelectPriorityLocked(uint32_t priority); + + const int child_failover_timeout_ms_; + + // Current channel args and config from the resolver. + const grpc_channel_args* args_ = nullptr; + RefCountedPtr config_; + HierarchicalAddressMap addresses_; + + // Internal state. + bool shutting_down_ = false; + + std::map> children_; + // The priority that is being used. + uint32_t current_priority_ = UINT32_MAX; + // Points to the current child from before the most recent update. + // We will continue to use this child until we decide which of the new + // children to use. + ChildPriority* current_child_from_before_update_ = nullptr; +}; + +// +// PriorityLb +// + +PriorityLb::PriorityLb(Args args) + : LoadBalancingPolicy(std::move(args)), + child_failover_timeout_ms_(grpc_channel_args_find_integer( + args.args, GRPC_ARG_PRIORITY_FAILOVER_TIMEOUT_MS, + {kDefaultChildFailoverTimeoutMs, 0, INT_MAX})) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, "[priority_lb %p] created", this); + } +} + +PriorityLb::~PriorityLb() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, "[priority_lb %p] destroying priority LB policy", this); + } + grpc_channel_args_destroy(args_); +} + +void PriorityLb::ShutdownLocked() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, "[priority_lb %p] shutting down", this); + } + shutting_down_ = true; + children_.clear(); +} + +void PriorityLb::ExitIdleLocked() { + if (current_priority_ != UINT32_MAX) { + const std::string& child_name = config_->priorities()[current_priority_]; + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, + "[priority_lb %p] exiting IDLE for current priority %d child %s", + this, current_priority_, child_name.c_str()); + } + children_[child_name]->ExitIdleLocked(); + } +} + +void PriorityLb::ResetBackoffLocked() { + for (const auto& p : children_) p.second->ResetBackoffLocked(); +} + +void PriorityLb::UpdateLocked(UpdateArgs args) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, "[priority_lb %p] received update", this); + } + // Save current child. + if (current_priority_ != UINT32_MAX) { + const std::string& child_name = config_->priorities()[current_priority_]; + current_child_from_before_update_ = children_[child_name].get(); + // Unset current_priority_, since it was an index into the old + // config's priority list and may no longer be valid. It will be + // reset later by TryNextPriorityLocked(), but we unset it here in + // case updating any of our children triggers a state update. + current_priority_ = UINT32_MAX; + } + // Update config. + config_ = std::move(args.config); + // Update args. + grpc_channel_args_destroy(args_); + args_ = args.args; + args.args = nullptr; + // Update addresses. + addresses_ = MakeHierarchicalAddressMap(args.addresses); + // Check all existing children against the new config. + for (const auto& p : children_) { + const std::string& child_name = p.first; + auto& child = p.second; + auto config_it = config_->children().find(child_name); + if (config_it == config_->children().end()) { + // Existing child not found in new config. Deactivate it. + child->DeactivateLocked(); + } else { + // Existing child found in new config. Update it. + child->UpdateLocked(config_it->second); + } + } + // Try to get connected. + TryNextPriorityLocked(/*report_connecting=*/children_.empty()); +} + +uint32_t PriorityLb::GetChildPriorityLocked( + const std::string& child_name) const { + for (uint32_t priority = 0; priority < config_->priorities().size(); + ++priority) { + if (config_->priorities()[priority] == child_name) return priority; + } + return UINT32_MAX; +} + +void PriorityLb::HandleChildConnectivityStateChangeLocked( + ChildPriority* child) { + // Special case for the child that was the current child before the + // most recent update. + if (child == current_child_from_before_update_) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, + "[priority_lb %p] state update for current child from before " + "config update", + this); + } + if (child->connectivity_state() == GRPC_CHANNEL_READY || + child->connectivity_state() == GRPC_CHANNEL_IDLE) { + // If it's still READY or IDLE, we stick with this child, so pass + // the new picker up to our parent. + channel_control_helper()->UpdateState(child->connectivity_state(), + child->GetPicker()); + } else { + // If it's no longer READY or IDLE, we should stop using it. + // We already started trying other priorities as a result of the + // update, but calling TryNextPriorityLocked() ensures that we will + // properly select between CONNECTING and TRANSIENT_FAILURE as the + // new state to report to our parent. + current_child_from_before_update_ = nullptr; + TryNextPriorityLocked(/*report_connecting=*/true); + } + return; + } + // Otherwise, find the child's priority. + uint32_t child_priority = GetChildPriorityLocked(child->name()); + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, "[priority_lb %p] state update for priority %d, child %s", + this, child_priority, child->name().c_str()); + } + // Ignore priorities not in the current config. + if (child_priority == UINT32_MAX) return; + // Ignore lower-than-current priorities. + if (child_priority > current_priority_) return; + // If a child reports TRANSIENT_FAILURE, start trying the next priority. + // Note that even if this is for a higher-than-current priority, we + // may still need to create some children between this priority and + // the current one (e.g., if we got an update that inserted new + // priorities ahead of the current one). + if (child->connectivity_state() == GRPC_CHANNEL_TRANSIENT_FAILURE) { + TryNextPriorityLocked( + /*report_connecting=*/child_priority == current_priority_); + return; + } + // The update is for a higher-than-current priority (or for any + // priority if we don't have any current priority). + if (child_priority < current_priority_) { + // If the child reports READY or IDLE, switch to that priority. + // Otherwise, ignore the update. + if (child->connectivity_state() == GRPC_CHANNEL_READY || + child->connectivity_state() == GRPC_CHANNEL_IDLE) { + SelectPriorityLocked(child_priority); + } + return; + } + // The current priority has returned a new picker, so pass it up to + // our parent. + channel_control_helper()->UpdateState(child->connectivity_state(), + child->GetPicker()); +} + +void PriorityLb::DeleteChild(ChildPriority* child) { + // If this was the current child from before the most recent update, + // stop using it. We already started trying other priorities as a + // result of the update, but calling TryNextPriorityLocked() ensures that + // we will properly select between CONNECTING and TRANSIENT_FAILURE as the + // new state to report to our parent. + if (current_child_from_before_update_ == child) { + current_child_from_before_update_ = nullptr; + TryNextPriorityLocked(/*report_connecting=*/true); + } + children_.erase(child->name()); +} + +void PriorityLb::TryNextPriorityLocked(bool report_connecting) { + for (uint32_t priority = 0; priority < config_->priorities().size(); + ++priority) { + // If the child for the priority does not exist yet, create it. + const std::string& child_name = config_->priorities()[priority]; + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, "[priority_lb %p] trying priority %d, child %s", this, + priority, child_name.c_str()); + } + auto& child = children_[child_name]; + if (child == nullptr) { + if (report_connecting) { + channel_control_helper()->UpdateState( + GRPC_CHANNEL_CONNECTING, + absl::make_unique(Ref(DEBUG_LOCATION, "QueuePicker"))); + } + child = MakeOrphanable( + Ref(DEBUG_LOCATION, "ChildPriority"), child_name); + child->UpdateLocked(config_->children().find(child_name)->second); + return; + } + // The child already exists. + child->MaybeReactivateLocked(); + // If the child is in state READY or IDLE, switch to it. + if (child->connectivity_state() == GRPC_CHANNEL_READY || + child->connectivity_state() == GRPC_CHANNEL_IDLE) { + SelectPriorityLocked(priority); + return; + } + // Child is not READY or IDLE. + // If its failover timer is still pending, give it time to fire. + if (child->failover_timer_callback_pending()) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, + "[priority_lb %p] priority %d, child %s: child still " + "attempting to connect, will wait", + this, priority, child_name.c_str()); + } + if (report_connecting) { + channel_control_helper()->UpdateState( + GRPC_CHANNEL_CONNECTING, + absl::make_unique(Ref(DEBUG_LOCATION, "QueuePicker"))); + } + return; + } + // Child has been failing for a while. Move on to the next priority. + } + // If there are no more priorities to try, report TRANSIENT_FAILURE. + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, + "[priority_lb %p] no priority reachable, putting channel in " + "TRANSIENT_FAILURE", + this); + } + current_priority_ = UINT32_MAX; + current_child_from_before_update_ = nullptr; + grpc_error* error = grpc_error_set_int( + GRPC_ERROR_CREATE_FROM_STATIC_STRING("no ready priority"), + GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE); + channel_control_helper()->UpdateState( + GRPC_CHANNEL_TRANSIENT_FAILURE, + absl::make_unique(error)); +} + +void PriorityLb::SelectPriorityLocked(uint32_t priority) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, "[priority_lb %p] selected priority %d, child %s", this, + priority, config_->priorities()[priority].c_str()); + } + current_priority_ = priority; + current_child_from_before_update_ = nullptr; + // Deactivate lower priorities. + for (uint32_t p = priority + 1; p < config_->priorities().size(); ++p) { + const std::string& child_name = config_->priorities()[p]; + auto it = children_.find(child_name); + if (it != children_.end()) it->second->DeactivateLocked(); + } + // Update picker. + auto& child = children_[config_->priorities()[priority]]; + channel_control_helper()->UpdateState(child->connectivity_state(), + child->GetPicker()); +} + +// +// PriorityLb::ChildPriority +// + +PriorityLb::ChildPriority::ChildPriority( + RefCountedPtr priority_policy, std::string name) + : priority_policy_(std::move(priority_policy)), name_(std::move(name)) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, "[priority_lb %p] creating child %s (%p)", + priority_policy_.get(), name_.c_str(), this); + } + GRPC_CLOSURE_INIT(&on_failover_timer_, OnFailoverTimer, this, + grpc_schedule_on_exec_ctx); + GRPC_CLOSURE_INIT(&on_failover_timer_locked_, OnFailoverTimerLocked, this, + nullptr); + // Start the failover timer. + StartFailoverTimerLocked(); +} + +void PriorityLb::ChildPriority::Orphan() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, "[priority_lb %p] child %s (%p): orphaned", + priority_policy_.get(), name_.c_str(), this); + } + MaybeCancelFailoverTimerLocked(); + if (deactivation_timer_callback_pending_) { + grpc_timer_cancel(&deactivation_timer_); + } + // Remove the child policy's interested_parties pollset_set from the + // xDS policy. + grpc_pollset_set_del_pollset_set(child_policy_->interested_parties(), + priority_policy_->interested_parties()); + child_policy_.reset(); + // Drop our ref to the child's picker, in case it's holding a ref to + // the child. + picker_wrapper_.reset(); + if (deactivation_timer_callback_pending_) { + grpc_timer_cancel(&deactivation_timer_); + } + Unref(DEBUG_LOCATION, "ChildPriority+Orphan"); +} + +void PriorityLb::ChildPriority::UpdateLocked( + RefCountedPtr config) { + if (priority_policy_->shutting_down_) return; + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, "[priority_lb %p] child %s (%p): start update", + priority_policy_.get(), name_.c_str(), this); + } + // Create policy if needed. + if (child_policy_ == nullptr) { + child_policy_ = CreateChildPolicyLocked(priority_policy_->args_); + } + // Construct update args. + UpdateArgs update_args; + update_args.config = std::move(config); + update_args.addresses = priority_policy_->addresses_[name_]; + update_args.args = grpc_channel_args_copy(priority_policy_->args_); + // Update the policy. + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, + "[priority_lb %p] child %s (%p): updating child policy handler %p", + priority_policy_.get(), name_.c_str(), this, child_policy_.get()); + } + child_policy_->UpdateLocked(std::move(update_args)); +} + +OrphanablePtr +PriorityLb::ChildPriority::CreateChildPolicyLocked( + const grpc_channel_args* args) { + LoadBalancingPolicy::Args lb_policy_args; + lb_policy_args.combiner = priority_policy_->combiner(); + lb_policy_args.args = args; + lb_policy_args.channel_control_helper = + absl::make_unique(this->Ref(DEBUG_LOCATION, "Helper")); + OrphanablePtr lb_policy = + MakeOrphanable(std::move(lb_policy_args), + &grpc_lb_priority_trace); + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, + "[priority_lb %p] child %s (%p): created new child policy " + "handler %p", + priority_policy_.get(), name_.c_str(), this, lb_policy.get()); + } + // Add the parent's interested_parties pollset_set to that of the newly + // created child policy. This will make the child policy progress upon + // activity on the parent LB, which in turn is tied to the application's call. + grpc_pollset_set_add_pollset_set(lb_policy->interested_parties(), + priority_policy_->interested_parties()); + return lb_policy; +} + +void PriorityLb::ChildPriority::ExitIdleLocked() { + if (connectivity_state_ == GRPC_CHANNEL_IDLE && + !failover_timer_callback_pending_) { + StartFailoverTimerLocked(); + } + child_policy_->ExitIdleLocked(); +} + +void PriorityLb::ChildPriority::ResetBackoffLocked() { + child_policy_->ResetBackoffLocked(); +} + +void PriorityLb::ChildPriority::OnConnectivityStateUpdateLocked( + grpc_connectivity_state state, std::unique_ptr picker) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, + "[priority_lb %p] child %s (%p): state update: %s, picker %p", + priority_policy_.get(), name_.c_str(), this, + ConnectivityStateName(state), picker.get()); + } + // Store the state and picker. + connectivity_state_ = state; + picker_wrapper_ = MakeRefCounted(std::move(picker)); + // If READY or TRANSIENT_FAILURE, cancel failover timer. + if (state == GRPC_CHANNEL_READY || state == GRPC_CHANNEL_TRANSIENT_FAILURE) { + MaybeCancelFailoverTimerLocked(); + } + // Notify the parent policy. + priority_policy_->HandleChildConnectivityStateChangeLocked(this); +} + +void PriorityLb::ChildPriority::StartFailoverTimerLocked() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, + "[priority_lb %p] child %s (%p): starting failover timer for %d ms", + priority_policy_.get(), name_.c_str(), this, + priority_policy_->child_failover_timeout_ms_); + } + Ref(DEBUG_LOCATION, "ChildPriority+OnFailoverTimerLocked").release(); + grpc_timer_init( + &failover_timer_, + ExecCtx::Get()->Now() + priority_policy_->child_failover_timeout_ms_, + &on_failover_timer_); + failover_timer_callback_pending_ = true; +} + +void PriorityLb::ChildPriority::MaybeCancelFailoverTimerLocked() { + if (failover_timer_callback_pending_) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, + "[priority_lb %p] child %s (%p): cancelling failover timer", + priority_policy_.get(), name_.c_str(), this); + } + grpc_timer_cancel(&failover_timer_); + failover_timer_callback_pending_ = false; + } +} + +void PriorityLb::ChildPriority::OnFailoverTimer(void* arg, grpc_error* error) { + ChildPriority* self = static_cast(arg); + self->priority_policy_->combiner()->Run(&self->on_failover_timer_locked_, + GRPC_ERROR_REF(error)); +} + +void PriorityLb::ChildPriority::OnFailoverTimerLocked(void* arg, + grpc_error* error) { + ChildPriority* self = static_cast(arg); + if (error == GRPC_ERROR_NONE && self->failover_timer_callback_pending_ && + !self->priority_policy_->shutting_down_) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, + "[priority_lb %p] child %s (%p): failover timer fired, " + "reporting TRANSIENT_FAILURE", + self->priority_policy_.get(), self->name_.c_str(), self); + } + self->failover_timer_callback_pending_ = false; + self->OnConnectivityStateUpdateLocked(GRPC_CHANNEL_TRANSIENT_FAILURE, + nullptr); + } + self->Unref(DEBUG_LOCATION, "ChildPriority+OnFailoverTimerLocked"); +} + +void PriorityLb::ChildPriority::DeactivateLocked() { + // If already deactivated, don't do it again. + if (deactivation_timer_callback_pending_) return; + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, + "[priority_lb %p] child %s (%p): deactivating -- will remove in %d " + "ms.", + priority_policy_.get(), name_.c_str(), this, + kChildRetentionIntervalMs); + } + MaybeCancelFailoverTimerLocked(); + // Start a timer to delete the child. + Ref(DEBUG_LOCATION, "ChildPriority+timer").release(); + GRPC_CLOSURE_INIT(&on_deactivation_timer_, OnDeactivationTimer, this, + grpc_schedule_on_exec_ctx); + grpc_timer_init(&deactivation_timer_, + ExecCtx::Get()->Now() + kChildRetentionIntervalMs, + &on_deactivation_timer_); + deactivation_timer_callback_pending_ = true; +} + +void PriorityLb::ChildPriority::MaybeReactivateLocked() { + if (deactivation_timer_callback_pending_) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, "[priority_lb %p] child %s (%p): reactivating", + priority_policy_.get(), name_.c_str(), this); + } + deactivation_timer_callback_pending_ = false; + grpc_timer_cancel(&deactivation_timer_); + } +} + +void PriorityLb::ChildPriority::OnDeactivationTimer(void* arg, + grpc_error* error) { + ChildPriority* self = static_cast(arg); + self->priority_policy_->combiner()->Run( + GRPC_CLOSURE_INIT(&self->on_deactivation_timer_, + OnDeactivationTimerLocked, self, nullptr), + GRPC_ERROR_REF(error)); +} + +void PriorityLb::ChildPriority::OnDeactivationTimerLocked(void* arg, + grpc_error* error) { + ChildPriority* self = static_cast(arg); + if (error == GRPC_ERROR_NONE && self->deactivation_timer_callback_pending_ && + !self->priority_policy_->shutting_down_) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_priority_trace)) { + gpr_log(GPR_INFO, + "[priority_lb %p] child %s (%p): deactivation timer fired, " + "deleting child", + self->priority_policy_.get(), self->name_.c_str(), self); + } + self->deactivation_timer_callback_pending_ = false; + self->priority_policy_->DeleteChild(self); + } + self->Unref(DEBUG_LOCATION, "ChildPriority+timer"); +} + +// +// PriorityLb::ChildPriority::Helper +// + +void PriorityLb::ChildPriority::Helper::RequestReresolution() { + if (priority_->priority_policy_->shutting_down_) return; + priority_->priority_policy_->channel_control_helper()->RequestReresolution(); +} + +RefCountedPtr +PriorityLb::ChildPriority::Helper::CreateSubchannel( + const grpc_channel_args& args) { + if (priority_->priority_policy_->shutting_down_) return nullptr; + return priority_->priority_policy_->channel_control_helper() + ->CreateSubchannel(args); +} + +void PriorityLb::ChildPriority::Helper::UpdateState( + grpc_connectivity_state state, std::unique_ptr picker) { + if (priority_->priority_policy_->shutting_down_) return; + // Notify the priority. + priority_->OnConnectivityStateUpdateLocked(state, std::move(picker)); +} + +void PriorityLb::ChildPriority::Helper::AddTraceEvent(TraceSeverity severity, + StringView message) { + if (priority_->priority_policy_->shutting_down_) return; + priority_->priority_policy_->channel_control_helper()->AddTraceEvent(severity, + message); +} + +// +// factory +// + +class PriorityLbFactory : public LoadBalancingPolicyFactory { + public: + OrphanablePtr CreateLoadBalancingPolicy( + LoadBalancingPolicy::Args args) const override { + return MakeOrphanable(std::move(args)); + } + + const char* name() const override { return kPriority; } + + RefCountedPtr ParseLoadBalancingConfig( + const Json& json, grpc_error** error) const override { + GPR_DEBUG_ASSERT(error != nullptr && *error == GRPC_ERROR_NONE); + if (json.type() == Json::Type::JSON_NULL) { + // priority was mentioned as a policy in the deprecated + // loadBalancingPolicy field or in the client API. + *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:loadBalancingPolicy error:priority policy requires " + "configuration. Please use loadBalancingConfig field of service " + "config instead."); + return nullptr; + } + std::vector error_list; + // Children. + std::map> children; + auto it = json.object_value().find("children"); + if (it == json.object_value().end()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:children error:required field missing")); + } else if (it->second.type() != Json::Type::OBJECT) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:children error:type should be object")); + } else { + const Json::Object& object = it->second.object_value(); + for (const auto& p : object) { + const std::string& child_name = p.first; + const Json& element = p.second; + if (element.type() != Json::Type::OBJECT) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("field:children key:", child_name, + " error:should be type object") + .c_str())); + } else { + auto it2 = element.object_value().find("config"); + if (it2 == element.object_value().end()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("field:children key:", child_name, + " error:missing 'config' field") + .c_str())); + } else { + grpc_error* parse_error = GRPC_ERROR_NONE; + auto config = LoadBalancingPolicyRegistry::ParseLoadBalancingConfig( + it2->second, &parse_error); + if (config == nullptr) { + GPR_DEBUG_ASSERT(parse_error != GRPC_ERROR_NONE); + error_list.push_back( + GRPC_ERROR_CREATE_REFERENCING_FROM_COPIED_STRING( + absl::StrCat("field:children key:", child_name).c_str(), + &parse_error, 1)); + GRPC_ERROR_UNREF(parse_error); + } + children[child_name] = std::move(config); + } + } + } + } + // Priorities. + std::vector priorities; + it = json.object_value().find("priorities"); + if (it == json.object_value().end()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:priorities error:required field missing")); + } else if (it->second.type() != Json::Type::ARRAY) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:priorities error:type should be array")); + } else { + const Json::Array& array = it->second.array_value(); + for (size_t i = 0; i < array.size(); ++i) { + const Json& element = array[i]; + if (element.type() != Json::Type::STRING) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("field:priorities element:", i, + " error:should be type string") + .c_str())); + } else if (children.find(element.string_value()) == children.end()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("field:priorities element:", i, + " error:unknown child '", element.string_value(), + "'") + .c_str())); + } else { + priorities.emplace_back(element.string_value()); + } + } + if (priorities.size() != children.size()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("field:priorities error:priorities size (", + priorities.size(), ") != children size (", + children.size(), ")") + .c_str())); + } + } + if (error_list.empty()) { + return MakeRefCounted(std::move(children), + std::move(priorities)); + } else { + *error = GRPC_ERROR_CREATE_FROM_VECTOR( + "priority_experimental LB policy config", &error_list); + return nullptr; + } + } +}; + +} // namespace + +} // namespace grpc_core + +// +// Plugin registration +// + +void grpc_lb_policy_priority_init() { + grpc_core::LoadBalancingPolicyRegistry::Builder:: + RegisterLoadBalancingPolicyFactory( + absl::make_unique()); +} + +void grpc_lb_policy_priority_shutdown() {} diff --git a/src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc b/src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc new file mode 100644 index 00000000000..a319c40cd31 --- /dev/null +++ b/src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc @@ -0,0 +1,722 @@ +// +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include + +#include +#include +#include + +#include "absl/strings/str_cat.h" + +#include + +#include "src/core/ext/filters/client_channel/lb_policy.h" +#include "src/core/ext/filters/client_channel/lb_policy/address_filtering.h" +#include "src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h" +#include "src/core/ext/filters/client_channel/lb_policy_factory.h" +#include "src/core/ext/filters/client_channel/lb_policy_registry.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/gpr/string.h" +#include "src/core/lib/gprpp/orphanable.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/iomgr/combiner.h" +#include "src/core/lib/iomgr/timer.h" + +namespace grpc_core { + +TraceFlag grpc_lb_weighted_target_trace(false, "weighted_target_lb"); + +namespace { + +constexpr char kWeightedTarget[] = "weighted_target_experimental"; + +// How long we keep a child around for after it has been removed from +// the config. +constexpr int kChildRetentionIntervalMs = 15 * 60 * 1000; + +// Config for weighted_target LB policy. +class WeightedTargetLbConfig : public LoadBalancingPolicy::Config { + public: + struct ChildConfig { + uint32_t weight; + RefCountedPtr config; + }; + + using TargetMap = std::map; + + explicit WeightedTargetLbConfig(TargetMap target_map) + : target_map_(std::move(target_map)) {} + + const char* name() const override { return kWeightedTarget; } + + const TargetMap& target_map() const { return target_map_; } + + private: + TargetMap target_map_; +}; + +// weighted_target LB policy. +class WeightedTargetLb : public LoadBalancingPolicy { + public: + explicit WeightedTargetLb(Args args); + + const char* name() const override { return kWeightedTarget; } + + void UpdateLocked(UpdateArgs args) override; + void ResetBackoffLocked() override; + + private: + // A simple wrapper for ref-counting a picker from the child policy. + class ChildPickerWrapper : public RefCounted { + public: + explicit ChildPickerWrapper(std::unique_ptr picker) + : picker_(std::move(picker)) {} + PickResult Pick(PickArgs args) { return picker_->Pick(args); } + + private: + std::unique_ptr picker_; + }; + + // Picks a child using stateless WRR and then delegates to that + // child's picker. + class WeightedPicker : public SubchannelPicker { + public: + // Maintains a weighted list of pickers from each child that is in + // ready state. The first element in the pair represents the end of a + // range proportional to the child's weight. The start of the range + // is the previous value in the vector and is 0 for the first element. + using PickerList = + InlinedVector>, + 1>; + + explicit WeightedPicker(PickerList pickers) + : pickers_(std::move(pickers)) {} + + PickResult Pick(PickArgs args) override; + + private: + PickerList pickers_; + }; + + // Each WeightedChild holds a ref to its parent WeightedTargetLb. + class WeightedChild : public InternallyRefCounted { + public: + WeightedChild(RefCountedPtr weighted_target_policy, + const std::string& name); + ~WeightedChild(); + + void Orphan() override; + + void UpdateLocked(const WeightedTargetLbConfig::ChildConfig& config, + ServerAddressList addresses, + const grpc_channel_args* args); + void ResetBackoffLocked(); + void DeactivateLocked(); + + uint32_t weight() const { return weight_; } + grpc_connectivity_state connectivity_state() const { + return connectivity_state_; + } + RefCountedPtr picker_wrapper() const { + return picker_wrapper_; + } + + private: + class Helper : public ChannelControlHelper { + public: + explicit Helper(RefCountedPtr weighted_child) + : weighted_child_(std::move(weighted_child)) {} + + ~Helper() { weighted_child_.reset(DEBUG_LOCATION, "Helper"); } + + RefCountedPtr CreateSubchannel( + const grpc_channel_args& args) override; + void UpdateState(grpc_connectivity_state state, + std::unique_ptr picker) override; + void RequestReresolution() override; + void AddTraceEvent(TraceSeverity severity, StringView message) override; + + private: + RefCountedPtr weighted_child_; + }; + + // Methods for dealing with the child policy. + OrphanablePtr CreateChildPolicyLocked( + const grpc_channel_args* args); + + void OnConnectivityStateUpdateLocked( + grpc_connectivity_state state, + std::unique_ptr picker); + + static void OnDelayedRemovalTimer(void* arg, grpc_error* error); + static void OnDelayedRemovalTimerLocked(void* arg, grpc_error* error); + + // The owning LB policy. + RefCountedPtr weighted_target_policy_; + + const std::string& name_; + + uint32_t weight_; + + OrphanablePtr child_policy_; + + RefCountedPtr picker_wrapper_; + grpc_connectivity_state connectivity_state_ = GRPC_CHANNEL_CONNECTING; + bool seen_failure_since_ready_ = false; + + // States for delayed removal. + grpc_timer delayed_removal_timer_; + grpc_closure on_delayed_removal_timer_; + bool delayed_removal_timer_callback_pending_ = false; + bool shutdown_ = false; + }; + + ~WeightedTargetLb(); + + void ShutdownLocked() override; + + void UpdateStateLocked(); + + // Current config from the resolver. + RefCountedPtr config_; + + // Internal state. + bool shutting_down_ = false; + + // Children. + std::map> targets_; +}; + +// +// WeightedTargetLb::WeightedPicker +// + +WeightedTargetLb::PickResult WeightedTargetLb::WeightedPicker::Pick( + PickArgs args) { + // Generate a random number in [0, total weight). + const uint32_t key = rand() % pickers_[pickers_.size() - 1].first; + // Find the index in pickers_ corresponding to key. + size_t mid = 0; + size_t start_index = 0; + size_t end_index = pickers_.size() - 1; + size_t index = 0; + while (end_index > start_index) { + mid = (start_index + end_index) / 2; + if (pickers_[mid].first > key) { + end_index = mid; + } else if (pickers_[mid].first < key) { + start_index = mid + 1; + } else { + index = mid + 1; + break; + } + } + if (index == 0) index = start_index; + GPR_ASSERT(pickers_[index].first > key); + // Delegate to the child picker. + return pickers_[index].second->Pick(args); +} + +// +// WeightedTargetLb +// + +WeightedTargetLb::WeightedTargetLb(Args args) + : LoadBalancingPolicy(std::move(args)) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, "[weighted_target_lb %p] created", this); + } +} + +WeightedTargetLb::~WeightedTargetLb() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, + "[weighted_target_lb %p] destroying weighted_target LB policy", + this); + } +} + +void WeightedTargetLb::ShutdownLocked() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, "[weighted_target_lb %p] shutting down", this); + } + shutting_down_ = true; + targets_.clear(); +} + +void WeightedTargetLb::ResetBackoffLocked() { + for (auto& p : targets_) p.second->ResetBackoffLocked(); +} + +void WeightedTargetLb::UpdateLocked(UpdateArgs args) { + if (shutting_down_) return; + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, "[weighted_target_lb %p] Received update", this); + } + // Update config. + config_ = std::move(args.config); + // Deactivate the targets not in the new config. + for (const auto& p : targets_) { + const std::string& name = p.first; + WeightedChild* child = p.second.get(); + if (config_->target_map().find(name) == config_->target_map().end()) { + child->DeactivateLocked(); + } + } + // Add or update the targets in the new config. + HierarchicalAddressMap address_map = + MakeHierarchicalAddressMap(args.addresses); + for (const auto& p : config_->target_map()) { + const std::string& name = p.first; + const WeightedTargetLbConfig::ChildConfig& config = p.second; + auto it = targets_.find(name); + if (it == targets_.end()) { + it = targets_.emplace(std::make_pair(name, nullptr)).first; + it->second = MakeOrphanable( + Ref(DEBUG_LOCATION, "WeightedChild"), it->first); + } + it->second->UpdateLocked(config, std::move(address_map[name]), args.args); + } +} + +void WeightedTargetLb::UpdateStateLocked() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, + "[weighted_target_lb %p] scanning children to determine " + "connectivity state", + this); + } + // Construct a new picker which maintains a map of all child pickers + // that are ready. Each child is represented by a portion of the range + // proportional to its weight, such that the total range is the sum of the + // weights of all children. + WeightedPicker::PickerList picker_list; + uint32_t end = 0; + // Also count the number of children in each state, to determine the + // overall state. + size_t num_connecting = 0; + size_t num_idle = 0; + size_t num_transient_failures = 0; + for (const auto& p : targets_) { + const std::string& child_name = p.first; + const WeightedChild* child = p.second.get(); + // Skip the targets that are not in the latest update. + if (config_->target_map().find(child_name) == config_->target_map().end()) { + continue; + } + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, + "[weighted_target_lb %p] child=%s state=%s weight=%d picker=%p", + this, child_name.c_str(), + ConnectivityStateName(child->connectivity_state()), + child->weight(), child->picker_wrapper().get()); + } + switch (child->connectivity_state()) { + case GRPC_CHANNEL_READY: { + end += child->weight(); + picker_list.push_back(std::make_pair(end, child->picker_wrapper())); + break; + } + case GRPC_CHANNEL_CONNECTING: { + ++num_connecting; + break; + } + case GRPC_CHANNEL_IDLE: { + ++num_idle; + break; + } + case GRPC_CHANNEL_TRANSIENT_FAILURE: { + ++num_transient_failures; + break; + } + default: + GPR_UNREACHABLE_CODE(return ); + } + } + // Determine aggregated connectivity state. + grpc_connectivity_state connectivity_state; + if (!picker_list.empty()) { + connectivity_state = GRPC_CHANNEL_READY; + } else if (num_connecting > 0) { + connectivity_state = GRPC_CHANNEL_CONNECTING; + } else if (num_idle > 0) { + connectivity_state = GRPC_CHANNEL_IDLE; + } else { + connectivity_state = GRPC_CHANNEL_TRANSIENT_FAILURE; + } + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, "[weighted_target_lb %p] connectivity changed to %s", + this, ConnectivityStateName(connectivity_state)); + } + std::unique_ptr picker; + switch (connectivity_state) { + case GRPC_CHANNEL_READY: + picker = absl::make_unique(std::move(picker_list)); + break; + case GRPC_CHANNEL_CONNECTING: + case GRPC_CHANNEL_IDLE: + picker = + absl::make_unique(Ref(DEBUG_LOCATION, "QueuePicker")); + break; + default: + picker = absl::make_unique( + GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "weighted_target: all children report state TRANSIENT_FAILURE")); + } + channel_control_helper()->UpdateState(connectivity_state, std::move(picker)); +} + +// +// WeightedTargetLb::WeightedChild +// + +WeightedTargetLb::WeightedChild::WeightedChild( + RefCountedPtr weighted_target_policy, + const std::string& name) + : weighted_target_policy_(std::move(weighted_target_policy)), name_(name) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, "[weighted_target_lb %p] created WeightedChild %p for %s", + weighted_target_policy_.get(), this, name_.c_str()); + } +} + +WeightedTargetLb::WeightedChild::~WeightedChild() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, + "[weighted_target_lb %p] WeightedChild %p %s: destroying child", + weighted_target_policy_.get(), this, name_.c_str()); + } + weighted_target_policy_.reset(DEBUG_LOCATION, "WeightedChild"); +} + +void WeightedTargetLb::WeightedChild::Orphan() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, + "[weighted_target_lb %p] WeightedChild %p %s: shutting down child", + weighted_target_policy_.get(), this, name_.c_str()); + } + // Remove the child policy's interested_parties pollset_set from the + // xDS policy. + grpc_pollset_set_del_pollset_set( + child_policy_->interested_parties(), + weighted_target_policy_->interested_parties()); + child_policy_.reset(); + // Drop our ref to the child's picker, in case it's holding a ref to + // the child. + picker_wrapper_.reset(); + if (delayed_removal_timer_callback_pending_) { + delayed_removal_timer_callback_pending_ = false; + grpc_timer_cancel(&delayed_removal_timer_); + } + shutdown_ = true; + Unref(); +} + +OrphanablePtr +WeightedTargetLb::WeightedChild::CreateChildPolicyLocked( + const grpc_channel_args* args) { + LoadBalancingPolicy::Args lb_policy_args; + lb_policy_args.combiner = weighted_target_policy_->combiner(); + lb_policy_args.args = args; + lb_policy_args.channel_control_helper = + absl::make_unique(this->Ref(DEBUG_LOCATION, "Helper")); + OrphanablePtr lb_policy = + MakeOrphanable(std::move(lb_policy_args), + &grpc_lb_weighted_target_trace); + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, + "[weighted_target_lb %p] WeightedChild %p %s: Created new child " + "policy handler %p", + weighted_target_policy_.get(), this, name_.c_str(), + lb_policy.get()); + } + // Add the xDS's interested_parties pollset_set to that of the newly created + // child policy. This will make the child policy progress upon activity on + // xDS LB, which in turn is tied to the application's call. + grpc_pollset_set_add_pollset_set( + lb_policy->interested_parties(), + weighted_target_policy_->interested_parties()); + return lb_policy; +} + +void WeightedTargetLb::WeightedChild::UpdateLocked( + const WeightedTargetLbConfig::ChildConfig& config, + ServerAddressList addresses, const grpc_channel_args* args) { + if (weighted_target_policy_->shutting_down_) return; + // Update child weight. + weight_ = config.weight; + // Reactivate if needed. + if (delayed_removal_timer_callback_pending_) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, + "[weighted_target_lb %p] WeightedChild %p %s: reactivating", + weighted_target_policy_.get(), this, name_.c_str()); + } + delayed_removal_timer_callback_pending_ = false; + grpc_timer_cancel(&delayed_removal_timer_); + } + // Create child policy if needed. + if (child_policy_ == nullptr) { + child_policy_ = CreateChildPolicyLocked(args); + } + // Construct update args. + UpdateArgs update_args; + update_args.config = config.config; + update_args.addresses = std::move(addresses); + update_args.args = grpc_channel_args_copy(args); + // Update the policy. + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, + "[weighted_target_lb %p] WeightedChild %p %s: Updating child " + "policy handler %p", + weighted_target_policy_.get(), this, name_.c_str(), + child_policy_.get()); + } + child_policy_->UpdateLocked(std::move(update_args)); +} + +void WeightedTargetLb::WeightedChild::ResetBackoffLocked() { + child_policy_->ResetBackoffLocked(); +} + +void WeightedTargetLb::WeightedChild::OnConnectivityStateUpdateLocked( + grpc_connectivity_state state, std::unique_ptr picker) { + // Cache the picker in the WeightedChild. + picker_wrapper_ = MakeRefCounted(std::move(picker)); + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, + "[weighted_target_lb %p] WeightedChild %p %s: connectivity " + "state update: state=%s picker_wrapper=%p", + weighted_target_policy_.get(), this, name_.c_str(), + ConnectivityStateName(state), picker_wrapper_.get()); + } + // If the child reports IDLE, immediately tell it to exit idle. + if (state == GRPC_CHANNEL_IDLE) child_policy_->ExitIdleLocked(); + // Decide what state to report for aggregation purposes. + // If we haven't seen a failure since the last time we were in state + // READY, then we report the state change as-is. However, once we do see + // a failure, we report TRANSIENT_FAILURE and ignore any subsequent state + // changes until we go back into state READY. + if (!seen_failure_since_ready_) { + if (state == GRPC_CHANNEL_TRANSIENT_FAILURE) { + seen_failure_since_ready_ = true; + } + } else { + if (state != GRPC_CHANNEL_READY) return; + seen_failure_since_ready_ = false; + } + connectivity_state_ = state; + // Notify the LB policy. + weighted_target_policy_->UpdateStateLocked(); +} + +void WeightedTargetLb::WeightedChild::DeactivateLocked() { + // If already deactivated, don't do that again. + if (weight_ == 0) return; + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_weighted_target_trace)) { + gpr_log(GPR_INFO, + "[weighted_target_lb %p] WeightedChild %p %s: deactivating", + weighted_target_policy_.get(), this, name_.c_str()); + } + // Set the child weight to 0 so that future picker won't contain this child. + weight_ = 0; + // Start a timer to delete the child. + Ref(DEBUG_LOCATION, "WeightedChild+timer").release(); + GRPC_CLOSURE_INIT(&on_delayed_removal_timer_, OnDelayedRemovalTimer, this, + grpc_schedule_on_exec_ctx); + delayed_removal_timer_callback_pending_ = true; + grpc_timer_init(&delayed_removal_timer_, + ExecCtx::Get()->Now() + kChildRetentionIntervalMs, + &on_delayed_removal_timer_); +} + +void WeightedTargetLb::WeightedChild::OnDelayedRemovalTimer(void* arg, + grpc_error* error) { + WeightedChild* self = static_cast(arg); + self->weighted_target_policy_->combiner()->Run( + GRPC_CLOSURE_INIT(&self->on_delayed_removal_timer_, + OnDelayedRemovalTimerLocked, self, nullptr), + GRPC_ERROR_REF(error)); +} + +void WeightedTargetLb::WeightedChild::OnDelayedRemovalTimerLocked( + void* arg, grpc_error* error) { + WeightedChild* self = static_cast(arg); + if (error == GRPC_ERROR_NONE && + self->delayed_removal_timer_callback_pending_ && !self->shutdown_ && + self->weight_ == 0) { + self->delayed_removal_timer_callback_pending_ = false; + self->weighted_target_policy_->targets_.erase(self->name_); + } + self->Unref(DEBUG_LOCATION, "WeightedChild+timer"); +} + +// +// WeightedTargetLb::WeightedChild::Helper +// + +RefCountedPtr +WeightedTargetLb::WeightedChild::Helper::CreateSubchannel( + const grpc_channel_args& args) { + if (weighted_child_->weighted_target_policy_->shutting_down_) return nullptr; + return weighted_child_->weighted_target_policy_->channel_control_helper() + ->CreateSubchannel(args); +} + +void WeightedTargetLb::WeightedChild::Helper::UpdateState( + grpc_connectivity_state state, std::unique_ptr picker) { + if (weighted_child_->weighted_target_policy_->shutting_down_) return; + weighted_child_->OnConnectivityStateUpdateLocked(state, std::move(picker)); +} + +void WeightedTargetLb::WeightedChild::Helper::RequestReresolution() { + if (weighted_child_->weighted_target_policy_->shutting_down_) return; + weighted_child_->weighted_target_policy_->channel_control_helper() + ->RequestReresolution(); +} + +void WeightedTargetLb::WeightedChild::Helper::AddTraceEvent( + TraceSeverity severity, StringView message) { + if (weighted_child_->weighted_target_policy_->shutting_down_) return; + weighted_child_->weighted_target_policy_->channel_control_helper() + ->AddTraceEvent(severity, message); +} + +// +// factory +// + +class WeightedTargetLbFactory : public LoadBalancingPolicyFactory { + public: + OrphanablePtr CreateLoadBalancingPolicy( + LoadBalancingPolicy::Args args) const override { + return MakeOrphanable(std::move(args)); + } + + const char* name() const override { return kWeightedTarget; } + + RefCountedPtr ParseLoadBalancingConfig( + const Json& json, grpc_error** error) const override { + GPR_DEBUG_ASSERT(error != nullptr && *error == GRPC_ERROR_NONE); + if (json.type() == Json::Type::JSON_NULL) { + // weighted_target was mentioned as a policy in the deprecated + // loadBalancingPolicy field or in the client API. + *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:loadBalancingPolicy error:weighted_target policy requires " + "configuration. Please use loadBalancingConfig field of service " + "config instead."); + return nullptr; + } + std::vector error_list; + // Weight map. + WeightedTargetLbConfig::TargetMap target_map; + auto it = json.object_value().find("targets"); + if (it == json.object_value().end()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:targets error:required field not present")); + } else if (it->second.type() != Json::Type::OBJECT) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:targets error:type should be object")); + } else { + for (const auto& p : it->second.object_value()) { + WeightedTargetLbConfig::ChildConfig child_config; + std::vector child_errors = + ParseChildConfig(p.second, &child_config); + if (!child_errors.empty()) { + // Can't use GRPC_ERROR_CREATE_FROM_VECTOR() here, because the error + // string is not static in this case. + grpc_error* error = GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("field:targets key:", p.first).c_str()); + for (grpc_error* child_error : child_errors) { + error = grpc_error_add_child(error, child_error); + } + error_list.push_back(error); + } else { + target_map[p.first] = std::move(child_config); + } + } + } + if (!error_list.empty()) { + *error = GRPC_ERROR_CREATE_FROM_VECTOR( + "weighted_target_experimental LB policy config", &error_list); + return nullptr; + } + return MakeRefCounted(std::move(target_map)); + } + + private: + static std::vector ParseChildConfig( + const Json& json, WeightedTargetLbConfig::ChildConfig* child_config) { + std::vector error_list; + if (json.type() != Json::Type::OBJECT) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "value should be of type object")); + return error_list; + } + // Weight. + auto it = json.object_value().find("weight"); + if (it == json.object_value().end()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "required field \"weight\" not specified")); + } else if (it->second.type() != Json::Type::NUMBER) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:weight error:must be of type number")); + } else { + child_config->weight = + gpr_parse_nonnegative_int(it->second.string_value().c_str()); + if (child_config->weight == -1) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:weight error:unparseable value")); + } else if (child_config->weight == 0) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:weight error:value must be greater than zero")); + } + } + // Child policy. + it = json.object_value().find("childPolicy"); + if (it != json.object_value().end()) { + grpc_error* parse_error = GRPC_ERROR_NONE; + child_config->config = + LoadBalancingPolicyRegistry::ParseLoadBalancingConfig(it->second, + &parse_error); + if (child_config->config == nullptr) { + GPR_DEBUG_ASSERT(parse_error != GRPC_ERROR_NONE); + std::vector child_errors; + child_errors.push_back(parse_error); + error_list.push_back( + GRPC_ERROR_CREATE_FROM_VECTOR("field:childPolicy", &child_errors)); + } + } + return error_list; + } +}; + +} // namespace + +} // namespace grpc_core + +// +// Plugin registration +// + +void grpc_lb_policy_weighted_target_init() { + grpc_core::LoadBalancingPolicyRegistry::Builder:: + RegisterLoadBalancingPolicyFactory( + absl::make_unique()); +} + +void grpc_lb_policy_weighted_target_shutdown() {} diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc b/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc index 51e250a3cad..29f571de414 100644 --- a/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc +++ b/src/core/ext/filters/client_channel/lb_policy/xds/cds.cc @@ -37,9 +37,9 @@ namespace { constexpr char kCds[] = "cds_experimental"; // Config for this LB policy. -class CdsConfig : public LoadBalancingPolicy::Config { +class CdsLbConfig : public LoadBalancingPolicy::Config { public: - explicit CdsConfig(std::string cluster) : cluster_(std::move(cluster)) {} + explicit CdsLbConfig(std::string cluster) : cluster_(std::move(cluster)) {} const std::string& cluster() const { return cluster_; } const char* name() const override { return kCds; } @@ -50,7 +50,7 @@ class CdsConfig : public LoadBalancingPolicy::Config { // CDS LB policy. class CdsLb : public LoadBalancingPolicy { public: - explicit CdsLb(Args args); + CdsLb(RefCountedPtr xds_client, Args args); const char* name() const override { return kCds; } @@ -89,7 +89,7 @@ class CdsLb : public LoadBalancingPolicy { void ShutdownLocked() override; - RefCountedPtr config_; + RefCountedPtr config_; // Current channel args from the resolver. const grpc_channel_args* args_ = nullptr; @@ -124,21 +124,37 @@ void CdsLb::ClusterWatcher::OnClusterChanged(XdsApi::CdsUpdate cluster_data) { } // Construct config for child policy. Json::Object child_config = { - {"edsServiceName", - (cluster_data.eds_service_name.empty() ? parent_->config_->cluster() - : cluster_data.eds_service_name)}, + {"clusterName", parent_->config_->cluster()}, + {"localityPickingPolicy", + Json::Array{ + Json::Object{ + {"weighted_target_experimental", + Json::Object{ + {"targets", Json::Object()}, + }}, + }, + }}, + {"endpointPickingPolicy", + Json::Array{ + Json::Object{ + {"round_robin", Json::Object()}, + }, + }}, }; + if (!cluster_data.eds_service_name.empty()) { + child_config["edsServiceName"] = cluster_data.eds_service_name; + } if (cluster_data.lrs_load_reporting_server_name.has_value()) { child_config["lrsLoadReportingServerName"] = cluster_data.lrs_load_reporting_server_name.value(); } Json json = Json::Array{ Json::Object{ - {"xds_experimental", std::move(child_config)}, + {"eds_experimental", std::move(child_config)}, }, }; if (GRPC_TRACE_FLAG_ENABLED(grpc_cds_lb_trace)) { - std::string json_str = json.Dump(); + std::string json_str = json.Dump(/*indent=*/1); gpr_log(GPR_INFO, "[cdslb %p] generated config for child policy: %s", parent_.get(), json_str.c_str()); } @@ -156,19 +172,19 @@ void CdsLb::ClusterWatcher::OnClusterChanged(XdsApi::CdsUpdate cluster_data) { args.args = parent_->args_; args.channel_control_helper = absl::make_unique(parent_->Ref()); parent_->child_policy_ = - LoadBalancingPolicyRegistry::CreateLoadBalancingPolicy( - "xds_experimental", std::move(args)); + LoadBalancingPolicyRegistry::CreateLoadBalancingPolicy(config->name(), + std::move(args)); if (parent_->child_policy_ == nullptr) { OnError(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "failed to create xds_experimental child policy")); + "failed to create child policy")); return; } grpc_pollset_set_add_pollset_set( parent_->child_policy_->interested_parties(), parent_->interested_parties()); if (GRPC_TRACE_FLAG_ENABLED(grpc_cds_lb_trace)) { - gpr_log(GPR_INFO, "[cdslb %p] created child policy xds_experimental (%p)", - parent_.get(), parent_->child_policy_.get()); + gpr_log(GPR_INFO, "[cdslb %p] created child policy %s (%p)", + parent_.get(), config->name(), parent_->child_policy_.get()); } } // Update child policy. @@ -232,9 +248,8 @@ void CdsLb::Helper::AddTraceEvent(TraceSeverity severity, StringView message) { // CdsLb // -CdsLb::CdsLb(Args args) - : LoadBalancingPolicy(std::move(args)), - xds_client_(XdsClient::GetFromChannelArgs(*args.args)) { +CdsLb::CdsLb(RefCountedPtr xds_client, Args args) + : LoadBalancingPolicy(std::move(args)), xds_client_(std::move(xds_client)) { if (GRPC_TRACE_FLAG_ENABLED(grpc_cds_lb_trace)) { gpr_log(GPR_INFO, "[cdslb %p] created -- using xds client %p from channel", this, xds_client_.get()); @@ -313,11 +328,19 @@ void CdsLb::UpdateLocked(UpdateArgs args) { // factory // -class CdsFactory : public LoadBalancingPolicyFactory { +class CdsLbFactory : public LoadBalancingPolicyFactory { public: OrphanablePtr CreateLoadBalancingPolicy( LoadBalancingPolicy::Args args) const override { - return MakeOrphanable(std::move(args)); + RefCountedPtr xds_client = + XdsClient::GetFromChannelArgs(*args.args); + if (xds_client == nullptr) { + gpr_log(GPR_ERROR, + "XdsClient not present in channel args -- cannot instantiate " + "cds LB policy"); + return nullptr; + } + return MakeOrphanable(std::move(xds_client), std::move(args)); } const char* name() const override { return kCds; } @@ -349,7 +372,7 @@ class CdsFactory : public LoadBalancingPolicyFactory { *error = GRPC_ERROR_CREATE_FROM_VECTOR("Cds Parser", &error_list); return nullptr; } - return MakeRefCounted(std::move(cluster)); + return MakeRefCounted(std::move(cluster)); } }; @@ -364,7 +387,7 @@ class CdsFactory : public LoadBalancingPolicyFactory { void grpc_lb_policy_cds_init() { grpc_core::LoadBalancingPolicyRegistry::Builder:: RegisterLoadBalancingPolicyFactory( - absl::make_unique()); + absl::make_unique()); } void grpc_lb_policy_cds_shutdown() {} diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/eds.cc b/src/core/ext/filters/client_channel/lb_policy/xds/eds.cc new file mode 100644 index 00000000000..2f1acdb9f80 --- /dev/null +++ b/src/core/ext/filters/client_channel/lb_policy/xds/eds.cc @@ -0,0 +1,1172 @@ +// +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include + +#include +#include + +#include "absl/strings/str_cat.h" +#include "absl/types/optional.h" + +#include + +#include "src/core/ext/filters/client_channel/client_channel.h" +#include "src/core/ext/filters/client_channel/lb_policy.h" +#include "src/core/ext/filters/client_channel/lb_policy/address_filtering.h" +#include "src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h" +#include "src/core/ext/filters/client_channel/lb_policy/xds/xds.h" +#include "src/core/ext/filters/client_channel/lb_policy_factory.h" +#include "src/core/ext/filters/client_channel/lb_policy_registry.h" +#include "src/core/ext/filters/client_channel/server_address.h" +#include "src/core/ext/filters/client_channel/xds/xds_client.h" +#include "src/core/ext/filters/client_channel/xds/xds_client_stats.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/gprpp/orphanable.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/iomgr/combiner.h" +#include "src/core/lib/iomgr/timer.h" +#include "src/core/lib/uri/uri_parser.h" + +#define GRPC_EDS_DEFAULT_FALLBACK_TIMEOUT 10000 + +namespace grpc_core { + +TraceFlag grpc_lb_eds_trace(false, "eds_lb"); + +namespace { + +constexpr char kXds[] = "xds_experimental"; +constexpr char kEds[] = "eds_experimental"; + +// Config for EDS LB policy. +class EdsLbConfig : public LoadBalancingPolicy::Config { + public: + EdsLbConfig(const char* name, std::string cluster_name, + std::string eds_service_name, + absl::optional lrs_load_reporting_server_name, + Json locality_picking_policy, Json endpoint_picking_policy, + RefCountedPtr fallback_policy) + : name_(name), + cluster_name_(std::move(cluster_name)), + eds_service_name_(std::move(eds_service_name)), + lrs_load_reporting_server_name_( + std::move(lrs_load_reporting_server_name)), + locality_picking_policy_(std::move(locality_picking_policy)), + endpoint_picking_policy_(std::move(endpoint_picking_policy)), + fallback_policy_(std::move(fallback_policy)) {} + + const char* name() const override { return name_; } + + const std::string& cluster_name() const { return cluster_name_; } + const std::string& eds_service_name() const { return eds_service_name_; } + const absl::optional& lrs_load_reporting_server_name() const { + return lrs_load_reporting_server_name_; + }; + const Json& locality_picking_policy() const { + return locality_picking_policy_; + } + const Json& endpoint_picking_policy() const { + return endpoint_picking_policy_; + } + RefCountedPtr fallback_policy() const { + return fallback_policy_; + } + + private: + const char* name_; + std::string cluster_name_; + std::string eds_service_name_; + absl::optional lrs_load_reporting_server_name_; + Json locality_picking_policy_; + Json endpoint_picking_policy_; + RefCountedPtr fallback_policy_; +}; + +// EDS LB policy. +class EdsLb : public LoadBalancingPolicy { + public: + EdsLb(const char* name, Args args); + + const char* name() const override { return name_; } + + void UpdateLocked(UpdateArgs args) override; + void ResetBackoffLocked() override; + + private: + class EndpointWatcher; + + // A simple wrapper for ref-counting a picker from the child policy. + class ChildPickerWrapper : public RefCounted { + public: + explicit ChildPickerWrapper(std::unique_ptr picker) + : picker_(std::move(picker)) {} + PickResult Pick(PickArgs args) { return picker_->Pick(args); } + + private: + std::unique_ptr picker_; + }; + + // A picker that handles drops. + class DropPicker : public SubchannelPicker { + public: + explicit DropPicker(EdsLb* eds_policy); + + PickResult Pick(PickArgs args) override; + + private: + RefCountedPtr drop_config_; + RefCountedPtr drop_stats_; + RefCountedPtr child_picker_; + }; + + class Helper : public ChannelControlHelper { + public: + explicit Helper(RefCountedPtr eds_policy) + : eds_policy_(std::move(eds_policy)) {} + + ~Helper() { eds_policy_.reset(DEBUG_LOCATION, "Helper"); } + + RefCountedPtr CreateSubchannel( + const grpc_channel_args& args) override; + void UpdateState(grpc_connectivity_state state, + std::unique_ptr picker) override; + // This is a no-op, because we get the addresses from the xds + // client, which is a watch-based API. + void RequestReresolution() override {} + void AddTraceEvent(TraceSeverity severity, StringView message) override; + + private: + RefCountedPtr eds_policy_; + }; + + class FallbackHelper : public ChannelControlHelper { + public: + explicit FallbackHelper(RefCountedPtr parent) + : parent_(std::move(parent)) {} + + ~FallbackHelper() { parent_.reset(DEBUG_LOCATION, "FallbackHelper"); } + + RefCountedPtr CreateSubchannel( + const grpc_channel_args& args) override; + void UpdateState(grpc_connectivity_state state, + std::unique_ptr picker) override; + void RequestReresolution() override; + void AddTraceEvent(TraceSeverity severity, StringView message) override; + + private: + RefCountedPtr parent_; + }; + + ~EdsLb(); + + void ShutdownLocked() override; + + void UpdatePriorityList(XdsApi::PriorityListUpdate priority_list_update); + void UpdateChildPolicyLocked(); + OrphanablePtr CreateChildPolicyLocked( + const grpc_channel_args* args); + ServerAddressList CreateChildPolicyAddressesLocked(); + RefCountedPtr CreateChildPolicyConfigLocked(); + grpc_channel_args* CreateChildPolicyArgsLocked( + const grpc_channel_args* args_in); + void MaybeUpdateDropPickerLocked(); + + // Methods for dealing with fallback state. + void MaybeCancelFallbackAtStartupChecks(); + static void OnFallbackTimer(void* arg, grpc_error* error); + static void OnFallbackTimerLocked(void* arg, grpc_error* error); + void UpdateFallbackPolicyLocked(); + OrphanablePtr CreateFallbackPolicyLocked( + const grpc_channel_args* args); + void MaybeExitFallbackMode(); + + // Caller must ensure that config_ is set before calling. + const StringView GetEdsResourceName() const { + if (xds_client_from_channel_ == nullptr) return server_name_; + if (!config_->eds_service_name().empty()) { + return config_->eds_service_name(); + } + return config_->cluster_name(); + } + + // Returns a pair containing the cluster and eds_service_name to use + // for LRS load reporting. + // Caller must ensure that config_ is set before calling. + std::pair GetLrsClusterKey() const { + if (xds_client_from_channel_ == nullptr) return {server_name_, nullptr}; + return {config_->cluster_name(), config_->eds_service_name()}; + } + + XdsClient* xds_client() const { + return xds_client_from_channel_ != nullptr ? xds_client_from_channel_.get() + : xds_client_.get(); + } + + // Policy name (kXds or kEds). + const char* name_; + + // Server name from target URI. + std::string server_name_; + + // Current channel args and config from the resolver. + const grpc_channel_args* args_ = nullptr; + RefCountedPtr config_; + + // Internal state. + bool shutting_down_ = false; + + // The xds client and endpoint watcher. + // If we get the XdsClient from the channel, we store it in + // xds_client_from_channel_; if we create it ourselves, we store it in + // xds_client_. + RefCountedPtr xds_client_from_channel_; + OrphanablePtr xds_client_; + // A pointer to the endpoint watcher, to be used when cancelling the watch. + // Note that this is not owned, so this pointer must never be derefernced. + EndpointWatcher* endpoint_watcher_ = nullptr; + // The latest data from the endpoint watcher. + XdsApi::PriorityListUpdate priority_list_update_; + // State used to retain child policy names for priority policy. + std::vector priority_child_numbers_; + + RefCountedPtr drop_config_; + RefCountedPtr drop_stats_; + + OrphanablePtr child_policy_; + + // The latest state and picker returned from the child policy. + grpc_connectivity_state child_state_; + RefCountedPtr child_picker_; + + // Non-null iff we are in fallback mode. + OrphanablePtr fallback_policy_; + + // Whether the checks for fallback at startup are ALL pending. There are + // several cases where this can be reset: + // 1. The fallback timer fires, we enter fallback mode. + // 2. Before the fallback timer fires, the endpoint watcher reports an + // error, we enter fallback mode. + // 3. Before the fallback timer fires, if any child policy in the locality map + // becomes READY, we cancel the fallback timer. + bool fallback_at_startup_checks_pending_ = false; + // Timeout in milliseconds for before using fallback backend addresses. + // 0 means not using fallback. + const grpc_millis lb_fallback_timeout_ms_; + // The backend addresses from the resolver. + ServerAddressList fallback_backend_addresses_; + // Fallback timer. + grpc_timer lb_fallback_timer_; + grpc_closure lb_on_fallback_; +}; + +// +// EdsLb::DropPicker +// + +EdsLb::DropPicker::DropPicker(EdsLb* eds_policy) + : drop_config_(eds_policy->drop_config_), + drop_stats_(eds_policy->drop_stats_), + child_picker_(eds_policy->child_picker_) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] constructed new drop picker %p", eds_policy, + this); + } +} + +EdsLb::PickResult EdsLb::DropPicker::Pick(PickArgs args) { + // Handle drop. + const std::string* drop_category; + if (drop_config_->ShouldDrop(&drop_category)) { + if (drop_stats_ != nullptr) drop_stats_->AddCallDropped(*drop_category); + PickResult result; + result.type = PickResult::PICK_COMPLETE; + return result; + } + // If we're not dropping all calls, we should always have a child picker. + if (child_picker_ == nullptr) { // Should never happen. + PickResult result; + result.type = PickResult::PICK_FAILED; + result.error = + grpc_error_set_int(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "eds drop picker not given any child picker"), + GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_INTERNAL); + return result; + } + // Not dropping, so delegate to child's picker. + return child_picker_->Pick(args); +} + +// +// EdsLb::Helper +// + +RefCountedPtr EdsLb::Helper::CreateSubchannel( + const grpc_channel_args& args) { + if (eds_policy_->shutting_down_) return nullptr; + return eds_policy_->channel_control_helper()->CreateSubchannel(args); +} + +void EdsLb::Helper::UpdateState(grpc_connectivity_state state, + std::unique_ptr picker) { + if (eds_policy_->shutting_down_) return; + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] child policy updated state=%s picker=%p", + eds_policy_.get(), ConnectivityStateName(state), picker.get()); + } + // Save the state and picker. + eds_policy_->child_state_ = state; + eds_policy_->child_picker_ = + MakeRefCounted(std::move(picker)); + // If the new state is READY, cancel the fallback-at-startup checks. + if (state == GRPC_CHANNEL_READY) { + eds_policy_->MaybeCancelFallbackAtStartupChecks(); + eds_policy_->MaybeExitFallbackMode(); + } + // TODO(roth): If the child reports TRANSIENT_FAILURE and the + // fallback-at-startup checks are pending, we should probably go into + // fallback mode immediately (cancelling the fallback-at-startup timer + // if needed). + // Wrap the picker in a DropPicker and pass it up. + eds_policy_->MaybeUpdateDropPickerLocked(); +} + +void EdsLb::Helper::AddTraceEvent(TraceSeverity severity, StringView message) { + if (eds_policy_->shutting_down_) return; + eds_policy_->channel_control_helper()->AddTraceEvent(severity, message); +} + +// +// EdsLb::FallbackHelper +// + +RefCountedPtr EdsLb::FallbackHelper::CreateSubchannel( + const grpc_channel_args& args) { + if (parent_->shutting_down_) return nullptr; + return parent_->channel_control_helper()->CreateSubchannel(args); +} + +void EdsLb::FallbackHelper::UpdateState( + grpc_connectivity_state state, std::unique_ptr picker) { + if (parent_->shutting_down_) return; + parent_->channel_control_helper()->UpdateState(state, std::move(picker)); +} + +void EdsLb::FallbackHelper::RequestReresolution() { + if (parent_->shutting_down_) return; + parent_->channel_control_helper()->RequestReresolution(); +} + +void EdsLb::FallbackHelper::AddTraceEvent(TraceSeverity severity, + StringView message) { + if (parent_->shutting_down_) return; + parent_->channel_control_helper()->AddTraceEvent(severity, message); +} + +// +// EdsLb::EndpointWatcher +// + +class EdsLb::EndpointWatcher : public XdsClient::EndpointWatcherInterface { + public: + explicit EndpointWatcher(RefCountedPtr eds_policy) + : eds_policy_(std::move(eds_policy)) {} + + ~EndpointWatcher() { eds_policy_.reset(DEBUG_LOCATION, "EndpointWatcher"); } + + void OnEndpointChanged(XdsApi::EdsUpdate update) override { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] Received EDS update from xds client", + eds_policy_.get()); + } + // If the balancer tells us to drop all the calls, we should exit fallback + // mode immediately. + if (update.drop_config->drop_all()) eds_policy_->MaybeExitFallbackMode(); + // Update the drop config. + const bool drop_config_changed = + eds_policy_->drop_config_ == nullptr || + *eds_policy_->drop_config_ != *update.drop_config; + if (drop_config_changed) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] Updating drop config", eds_policy_.get()); + } + eds_policy_->drop_config_ = std::move(update.drop_config); + eds_policy_->MaybeUpdateDropPickerLocked(); + } + // Update priority and locality info. + if (eds_policy_->priority_list_update_ == update.priority_list_update) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, + "[edslb %p] Incoming locality update identical to current, " + "ignoring. (drop_config_changed=%d)", + eds_policy_.get(), drop_config_changed); + } + return; + } + // Update the child policy with the new priority and endpoint data. + eds_policy_->UpdatePriorityList(std::move(update.priority_list_update)); + } + + void OnError(grpc_error* error) override { + // If the fallback-at-startup checks are pending, go into fallback mode + // immediately. This short-circuits the timeout for the + // fallback-at-startup case. + if (eds_policy_->fallback_at_startup_checks_pending_) { + gpr_log(GPR_ERROR, + "[edslb %p] xds watcher reported error; entering fallback " + "mode: %s", + eds_policy_.get(), grpc_error_string(error)); + eds_policy_->fallback_at_startup_checks_pending_ = false; + grpc_timer_cancel(&eds_policy_->lb_fallback_timer_); + eds_policy_->UpdateFallbackPolicyLocked(); + // If the xds call failed, request re-resolution. + // TODO(roth): We check the error string contents here to + // differentiate between the xds call failing and the xds channel + // going into TRANSIENT_FAILURE. This is a pretty ugly hack, + // but it's okay for now, since we're not yet sure whether we will + // continue to support the current fallback functionality. If we + // decide to keep the fallback approach, then we should either + // find a cleaner way to expose the difference between these two + // cases or decide that we're okay re-resolving in both cases. + // Note that even if we do keep the current fallback functionality, + // this re-resolution will only be necessary if we are going to be + // using this LB policy with resolvers other than the xds resolver. + if (strstr(grpc_error_string(error), "xds call failed")) { + eds_policy_->channel_control_helper()->RequestReresolution(); + } + } + GRPC_ERROR_UNREF(error); + } + + private: + RefCountedPtr eds_policy_; +}; + +// +// EdsLb public methods +// + +EdsLb::EdsLb(const char* name, Args args) + : LoadBalancingPolicy(std::move(args)), + name_(name), + xds_client_from_channel_(XdsClient::GetFromChannelArgs(*args.args)), + lb_fallback_timeout_ms_(grpc_channel_args_find_integer( + args.args, GRPC_ARG_XDS_FALLBACK_TIMEOUT_MS, + {GRPC_EDS_DEFAULT_FALLBACK_TIMEOUT, 0, INT_MAX})) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] created -- xds client from channel: %p", this, + xds_client_from_channel_.get()); + } + // Record server name. + const grpc_arg* arg = grpc_channel_args_find(args.args, GRPC_ARG_SERVER_URI); + const char* server_uri = grpc_channel_arg_get_string(arg); + GPR_ASSERT(server_uri != nullptr); + grpc_uri* uri = grpc_uri_parse(server_uri, true); + GPR_ASSERT(uri->path[0] != '\0'); + server_name_ = uri->path[0] == '/' ? uri->path + 1 : uri->path; + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] server name from channel: %s", this, + server_name_.c_str()); + } + grpc_uri_destroy(uri); +} + +EdsLb::~EdsLb() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] destroying xds LB policy", this); + } + grpc_channel_args_destroy(args_); +} + +void EdsLb::ShutdownLocked() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] shutting down", this); + } + shutting_down_ = true; + MaybeCancelFallbackAtStartupChecks(); + // Drop our ref to the child's picker, in case it's holding a ref to + // the child. + child_picker_.reset(); + if (child_policy_ != nullptr) { + grpc_pollset_set_del_pollset_set(child_policy_->interested_parties(), + interested_parties()); + child_policy_.reset(); + } + if (fallback_policy_ != nullptr) { + grpc_pollset_set_del_pollset_set(fallback_policy_->interested_parties(), + interested_parties()); + fallback_policy_.reset(); + } + drop_stats_.reset(); + // Cancel the endpoint watch here instead of in our dtor if we are using the + // xds resolver, because the watcher holds a ref to us and we might not be + // destroying the XdsClient, leading to a situation where this LB policy is + // never destroyed. + if (xds_client_from_channel_ != nullptr) { + if (config_ != nullptr) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] cancelling xds watch for %s", this, + std::string(GetEdsResourceName()).c_str()); + } + xds_client()->CancelEndpointDataWatch(GetEdsResourceName(), + endpoint_watcher_); + } + xds_client_from_channel_.reset(); + } + xds_client_.reset(); +} + +void EdsLb::UpdateLocked(UpdateArgs args) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] Received update", this); + } + const bool is_initial_update = args_ == nullptr; + // Update config. + auto old_config = std::move(config_); + config_ = std::move(args.config); + // Update fallback address list. + fallback_backend_addresses_ = std::move(args.addresses); + // Update args. + grpc_channel_args_destroy(args_); + args_ = args.args; + args.args = nullptr; + // Update the existing fallback policy. The fallback policy config and/or the + // fallback addresses may be new. + if (fallback_policy_ != nullptr) UpdateFallbackPolicyLocked(); + if (is_initial_update) { + // Initialize XdsClient. + if (xds_client_from_channel_ == nullptr) { + grpc_error* error = GRPC_ERROR_NONE; + xds_client_ = MakeOrphanable( + combiner(), interested_parties(), GetEdsResourceName(), + nullptr /* service config watcher */, *args_, &error); + // TODO(roth): If we decide that we care about fallback mode, add + // proper error handling here. + GPR_ASSERT(error == GRPC_ERROR_NONE); + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] Created xds client %p", this, + xds_client_.get()); + } + } + // Start fallback-at-startup checks. + grpc_millis deadline = ExecCtx::Get()->Now() + lb_fallback_timeout_ms_; + Ref(DEBUG_LOCATION, "on_fallback_timer").release(); // Held by closure + GRPC_CLOSURE_INIT(&lb_on_fallback_, &EdsLb::OnFallbackTimer, this, + grpc_schedule_on_exec_ctx); + fallback_at_startup_checks_pending_ = true; + grpc_timer_init(&lb_fallback_timer_, deadline, &lb_on_fallback_); + } + // Update drop stats for load reporting if needed. + if (is_initial_update || config_->lrs_load_reporting_server_name() != + old_config->lrs_load_reporting_server_name()) { + drop_stats_.reset(); + if (config_->lrs_load_reporting_server_name().has_value()) { + const auto key = GetLrsClusterKey(); + drop_stats_ = xds_client()->AddClusterDropStats( + config_->lrs_load_reporting_server_name().value(), + key.first /*cluster_name*/, key.second /*eds_service_name*/); + } + MaybeUpdateDropPickerLocked(); + } + // Update child policy if needed. + // Note that this comes after updating drop_stats_, since we want that + // to be used by any new picker we create here. + if (child_policy_ != nullptr) UpdateChildPolicyLocked(); + // Create endpoint watcher if needed. + if (is_initial_update) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] starting xds watch for %s", this, + std::string(GetEdsResourceName()).c_str()); + } + auto watcher = absl::make_unique( + Ref(DEBUG_LOCATION, "EndpointWatcher")); + endpoint_watcher_ = watcher.get(); + xds_client()->WatchEndpointData(GetEdsResourceName(), std::move(watcher)); + } +} + +void EdsLb::ResetBackoffLocked() { + // When the XdsClient is instantiated in the resolver instead of in this + // LB policy, this is done via the resolver, so we don't need to do it + // for xds_client_from_channel_ here. + if (xds_client_ != nullptr) xds_client_->ResetBackoff(); + if (child_policy_ != nullptr) { + child_policy_->ResetBackoffLocked(); + } + if (fallback_policy_ != nullptr) { + fallback_policy_->ResetBackoffLocked(); + } +} + +// +// child policy-related methods +// + +void EdsLb::UpdatePriorityList( + XdsApi::PriorityListUpdate priority_list_update) { + // Build some maps from locality to child number and the reverse from + // the old data in priority_list_update_ and priority_child_numbers_. + std::map + locality_child_map; + std::map> child_locality_map; + for (uint32_t priority = 0; priority < priority_list_update_.size(); + ++priority) { + auto* locality_map = priority_list_update_.Find(priority); + GPR_ASSERT(locality_map != nullptr); + size_t child_number = priority_child_numbers_[priority]; + for (const auto& p : locality_map->localities) { + XdsLocalityName* locality_name = p.first.get(); + locality_child_map[locality_name] = child_number; + child_locality_map[child_number].insert(locality_name); + } + } + // Construct new list of children. + std::vector priority_child_numbers; + for (uint32_t priority = 0; priority < priority_list_update.size(); + ++priority) { + auto* locality_map = priority_list_update.Find(priority); + GPR_ASSERT(locality_map != nullptr); + absl::optional child_number; + // If one of the localities in this priority already existed, reuse its + // child number. + for (const auto& p : locality_map->localities) { + XdsLocalityName* locality_name = p.first.get(); + if (!child_number.has_value()) { + auto it = locality_child_map.find(locality_name); + if (it != locality_child_map.end()) { + child_number = it->second; + locality_child_map.erase(it); + // Remove localities that *used* to be in this child number, so + // that we don't incorrectly reuse this child number for a + // subsequent priority. + for (XdsLocalityName* old_locality : + child_locality_map[*child_number]) { + locality_child_map.erase(old_locality); + } + } + } else { + // Remove all localities that are now in this child number, so + // that we don't accidentally reuse this child number for a + // subsequent priority. + locality_child_map.erase(locality_name); + } + } + // If we didn't find an existing child number, assign a new one. + if (!child_number.has_value()) { + for (child_number = 0; + child_locality_map.find(*child_number) != child_locality_map.end(); + ++(*child_number)) + ; + // Add entry so we know that the child number is in use. + // (Don't need to add the list of localities, since we won't use them.) + child_locality_map[*child_number]; + } + priority_child_numbers.push_back(*child_number); + } + // Save update. + priority_list_update_ = std::move(priority_list_update); + priority_child_numbers_ = std::move(priority_child_numbers); + // Update child policy. + UpdateChildPolicyLocked(); +} + +ServerAddressList EdsLb::CreateChildPolicyAddressesLocked() { + ServerAddressList addresses; + for (uint32_t priority = 0; priority < priority_list_update_.size(); + ++priority) { + std::string priority_child_name = + absl::StrCat("child", priority_child_numbers_[priority]); + const auto* locality_map = priority_list_update_.Find(priority); + GPR_ASSERT(locality_map != nullptr); + for (const auto& p : locality_map->localities) { + const auto& locality_name = p.first; + const auto& locality = p.second; + std::vector hierarchical_path = { + priority_child_name, locality_name->AsHumanReadableString()}; + for (size_t i = 0; i < locality.serverlist.size(); ++i) { + const ServerAddress& address = locality.serverlist[i]; + grpc_arg new_arg = MakeHierarchicalPathArg(hierarchical_path); + grpc_channel_args* args = + grpc_channel_args_copy_and_add(address.args(), &new_arg, 1); + addresses.emplace_back(address.address(), args); + } + } + } + return addresses; +} + +RefCountedPtr +EdsLb::CreateChildPolicyConfigLocked() { + Json::Object priority_children; + Json::Array priority_priorities; + for (uint32_t priority = 0; priority < priority_list_update_.size(); + ++priority) { + const auto* locality_map = priority_list_update_.Find(priority); + GPR_ASSERT(locality_map != nullptr); + Json::Object weighted_targets; + for (const auto& p : locality_map->localities) { + XdsLocalityName* locality_name = p.first.get(); + const auto& locality = p.second; + // Construct JSON object containing locality name. + Json::Object locality_name_json; + if (!locality_name->region().empty()) { + locality_name_json["region"] = locality_name->region(); + } + if (!locality_name->zone().empty()) { + locality_name_json["zone"] = locality_name->zone(); + } + if (!locality_name->sub_zone().empty()) { + locality_name_json["subzone"] = locality_name->sub_zone(); + } + // Construct endpoint-picking policy. + // Wrap it in the LRS policy if load reporting is enabled. + Json endpoint_picking_policy; + if (config_->lrs_load_reporting_server_name().has_value()) { + const auto key = GetLrsClusterKey(); + Json::Object lrs_config = { + {"clusterName", std::string(key.first)}, + {"locality", std::move(locality_name_json)}, + {"lrsLoadReportingServerName", + config_->lrs_load_reporting_server_name().value()}, + {"childPolicy", config_->endpoint_picking_policy()}, + }; + if (!key.second.empty()) { + lrs_config["edsServiceName"] = std::string(key.second); + } + endpoint_picking_policy = Json::Array{Json::Object{ + {"lrs_experimental", std::move(lrs_config)}, + }}; + } else { + endpoint_picking_policy = config_->endpoint_picking_policy(); + } + // Add weighted target entry. + weighted_targets[locality_name->AsHumanReadableString()] = Json::Object{ + {"weight", locality.lb_weight}, + {"childPolicy", std::move(endpoint_picking_policy)}, + }; + } + // Add priority entry. + const size_t child_number = priority_child_numbers_[priority]; + std::string child_name = absl::StrCat("child", child_number); + priority_priorities.emplace_back(child_name); + Json locality_picking_config = config_->locality_picking_policy(); + Json::Object& config = + *(*locality_picking_config.mutable_array())[0].mutable_object(); + auto it = config.begin(); + GPR_ASSERT(it != config.end()); + (*it->second.mutable_object())["targets"] = std::move(weighted_targets); + priority_children[child_name] = Json::Object{ + {"config", std::move(locality_picking_config)}, + }; + } + Json json = Json::Array{Json::Object{ + {"priority_experimental", + Json::Object{ + {"children", std::move(priority_children)}, + {"priorities", std::move(priority_priorities)}, + }}, + }}; + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + std::string json_str = json.Dump(/*indent=*/1); + gpr_log(GPR_INFO, "[edslb %p] generated config for child policy: %s", this, + json_str.c_str()); + } + grpc_error* error = GRPC_ERROR_NONE; + RefCountedPtr config = + LoadBalancingPolicyRegistry::ParseLoadBalancingConfig(json, &error); + if (error != GRPC_ERROR_NONE) { + // This should never happen, but if it does, we basically have no + // way to fix it, so we put the channel in TRANSIENT_FAILURE. + gpr_log(GPR_ERROR, + "[edslb %p] error parsing generated child policy config -- " + "will put channel in TRANSIENT_FAILURE: %s", + this, grpc_error_string(error)); + error = grpc_error_set_int( + grpc_error_add_child( + GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "eds LB policy: error parsing generated child policy config"), + error), + GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_INTERNAL); + channel_control_helper()->UpdateState( + GRPC_CHANNEL_TRANSIENT_FAILURE, + absl::make_unique(error)); + return nullptr; + } + return config; +} + +void EdsLb::UpdateChildPolicyLocked() { + if (shutting_down_) return; + UpdateArgs update_args; + update_args.config = CreateChildPolicyConfigLocked(); + if (update_args.config == nullptr) return; + update_args.addresses = CreateChildPolicyAddressesLocked(); + update_args.args = CreateChildPolicyArgsLocked(args_); + if (child_policy_ == nullptr) { + child_policy_ = CreateChildPolicyLocked(update_args.args); + } + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] Updating child policy %p", this, + child_policy_.get()); + } + child_policy_->UpdateLocked(std::move(update_args)); +} + +grpc_channel_args* EdsLb::CreateChildPolicyArgsLocked( + const grpc_channel_args* args) { + absl::InlinedVector args_to_add = { + // A channel arg indicating if the target is a backend inferred from an + // xds load balancer. + grpc_channel_arg_integer_create( + const_cast(GRPC_ARG_ADDRESS_IS_BACKEND_FROM_XDS_LOAD_BALANCER), + 1), + // Inhibit client-side health checking, since the balancer does + // this for us. + grpc_channel_arg_integer_create( + const_cast(GRPC_ARG_INHIBIT_HEALTH_CHECKING), 1), + }; + if (xds_client_from_channel_ == nullptr) { + args_to_add.emplace_back(xds_client_->MakeChannelArg()); + } + return grpc_channel_args_copy_and_add(args, args_to_add.data(), + args_to_add.size()); +} + +OrphanablePtr EdsLb::CreateChildPolicyLocked( + const grpc_channel_args* args) { + LoadBalancingPolicy::Args lb_policy_args; + lb_policy_args.combiner = combiner(); + lb_policy_args.args = args; + lb_policy_args.channel_control_helper = + absl::make_unique(Ref(DEBUG_LOCATION, "Helper")); + OrphanablePtr lb_policy = + LoadBalancingPolicyRegistry::CreateLoadBalancingPolicy( + "priority_experimental", std::move(lb_policy_args)); + if (GPR_UNLIKELY(lb_policy == nullptr)) { + gpr_log(GPR_ERROR, "[edslb %p] failure creating child policy", this); + return nullptr; + } + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p]: Created new child policy %p", this, + lb_policy.get()); + } + // Add our interested_parties pollset_set to that of the newly created + // child policy. This will make the child policy progress upon activity on + // this policy, which in turn is tied to the application's call. + grpc_pollset_set_add_pollset_set(lb_policy->interested_parties(), + interested_parties()); + return lb_policy; +} + +void EdsLb::MaybeUpdateDropPickerLocked() { + // If we are in fallback mode, don't override the picker. + if (fallback_policy_ != nullptr) return; + // If we're dropping all calls, report READY, regardless of what (or + // whether) the child has reported. + if (drop_config_ != nullptr && drop_config_->drop_all()) { + channel_control_helper()->UpdateState(GRPC_CHANNEL_READY, + absl::make_unique(this)); + return; + } + // Update only if we have a child picker. + if (child_picker_ != nullptr) { + channel_control_helper()->UpdateState(child_state_, + absl::make_unique(this)); + } +} + +// +// fallback-related methods +// + +void EdsLb::MaybeCancelFallbackAtStartupChecks() { + if (!fallback_at_startup_checks_pending_) return; + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] Cancelling fallback timer", this); + } + grpc_timer_cancel(&lb_fallback_timer_); + fallback_at_startup_checks_pending_ = false; +} + +void EdsLb::OnFallbackTimer(void* arg, grpc_error* error) { + EdsLb* edslb_policy = static_cast(arg); + edslb_policy->combiner()->Run( + GRPC_CLOSURE_INIT(&edslb_policy->lb_on_fallback_, + &EdsLb::OnFallbackTimerLocked, edslb_policy, nullptr), + GRPC_ERROR_REF(error)); +} + +void EdsLb::OnFallbackTimerLocked(void* arg, grpc_error* error) { + EdsLb* edslb_policy = static_cast(arg); + // If some fallback-at-startup check is done after the timer fires but before + // this callback actually runs, don't fall back. + if (edslb_policy->fallback_at_startup_checks_pending_ && + !edslb_policy->shutting_down_ && error == GRPC_ERROR_NONE) { + gpr_log(GPR_INFO, + "[edslb %p] Child policy not ready after fallback timeout; " + "entering fallback mode", + edslb_policy); + edslb_policy->fallback_at_startup_checks_pending_ = false; + edslb_policy->UpdateFallbackPolicyLocked(); + } + edslb_policy->Unref(DEBUG_LOCATION, "on_fallback_timer"); +} + +void EdsLb::UpdateFallbackPolicyLocked() { + if (shutting_down_) return; + // Create policy if needed. + if (fallback_policy_ == nullptr) { + fallback_policy_ = CreateFallbackPolicyLocked(args_); + } + // Construct update args. + UpdateArgs update_args; + update_args.addresses = fallback_backend_addresses_; + update_args.config = config_->fallback_policy(); + update_args.args = grpc_channel_args_copy(args_); + // Update the policy. + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] Updating fallback child policy handler %p", + this, fallback_policy_.get()); + } + fallback_policy_->UpdateLocked(std::move(update_args)); +} + +OrphanablePtr EdsLb::CreateFallbackPolicyLocked( + const grpc_channel_args* args) { + LoadBalancingPolicy::Args lb_policy_args; + lb_policy_args.combiner = combiner(); + lb_policy_args.args = args; + lb_policy_args.channel_control_helper = + absl::make_unique(Ref(DEBUG_LOCATION, "FallbackHelper")); + OrphanablePtr lb_policy = + MakeOrphanable(std::move(lb_policy_args), + &grpc_lb_eds_trace); + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] Created new fallback child policy handler %p", + this, lb_policy.get()); + } + // Add our interested_parties pollset_set to that of the newly created + // child policy. This will make the child policy progress upon activity on + // this policy, which in turn is tied to the application's call. + grpc_pollset_set_add_pollset_set(lb_policy->interested_parties(), + interested_parties()); + return lb_policy; +} + +void EdsLb::MaybeExitFallbackMode() { + if (fallback_policy_ == nullptr) return; + gpr_log(GPR_INFO, "[edslb %p] Exiting fallback mode", this); + fallback_policy_.reset(); +} + +// +// factory +// + +class EdsLbFactory : public LoadBalancingPolicyFactory { + public: + explicit EdsLbFactory(const char* name) : name_(name) {} + + OrphanablePtr CreateLoadBalancingPolicy( + LoadBalancingPolicy::Args args) const override { + return MakeOrphanable(std::move(args), &grpc_lb_eds_trace, + name_); + } + + const char* name() const override { return name_; } + + RefCountedPtr ParseLoadBalancingConfig( + const Json& json, grpc_error** error) const override { + GPR_DEBUG_ASSERT(error != nullptr && *error == GRPC_ERROR_NONE); + if (json.type() == Json::Type::JSON_NULL) { + // xds was mentioned as a policy in the deprecated loadBalancingPolicy + // field or in the client API. + *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:loadBalancingPolicy error:eds policy requires configuration. " + "Please use loadBalancingConfig field of service config instead."); + return nullptr; + } + std::vector error_list; + // EDS service name. + std::string eds_service_name; + auto it = json.object_value().find("edsServiceName"); + if (it != json.object_value().end()) { + if (it->second.type() != Json::Type::STRING) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:edsServiceName error:type should be string")); + } else { + eds_service_name = it->second.string_value(); + } + } + // Cluster name. + std::string cluster_name; + if (name_ == kEds) { + it = json.object_value().find("clusterName"); + if (it == json.object_value().end()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:clusterName error:required field missing")); + } else if (it->second.type() != Json::Type::STRING) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:clusterName error:type should be string")); + } else { + cluster_name = it->second.string_value(); + } + } else { + // For xds policy, this field does not exist in the config, so it + // will always be set to the same value as edsServiceName. + cluster_name = eds_service_name; + } + // LRS load reporting server name. + absl::optional lrs_load_reporting_server_name; + it = json.object_value().find("lrsLoadReportingServerName"); + if (it != json.object_value().end()) { + if (it->second.type() != Json::Type::STRING) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:lrsLoadReportingServerName error:type should be string")); + } else { + lrs_load_reporting_server_name.emplace(it->second.string_value()); + } + } + // Locality-picking policy. Not supported for xds policy. + Json locality_picking_policy = Json::Array{ + Json::Object{ + {"weighted_target_experimental", + Json::Object{ + {"targets", Json::Object()}, + }}, + }, + }; + if (name_ == kEds) { + it = json.object_value().find("localityPickingPolicy"); + if (it != json.object_value().end()) { + locality_picking_policy = it->second; + } + } + grpc_error* parse_error = GRPC_ERROR_NONE; + if (LoadBalancingPolicyRegistry::ParseLoadBalancingConfig( + locality_picking_policy, &parse_error) == nullptr) { + GPR_DEBUG_ASSERT(parse_error != GRPC_ERROR_NONE); + error_list.push_back(GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( + "localityPickingPolicy", &parse_error, 1)); + GRPC_ERROR_UNREF(parse_error); + } + // Endpoint-picking policy. Called "childPolicy" for xds policy. + const char* field_name = + name_ == kEds ? "endpointPickingPolicy" : "childPolicy"; + Json endpoint_picking_policy; + it = json.object_value().find(field_name); + if (it == json.object_value().end()) { + endpoint_picking_policy = Json::Array{ + Json::Object{ + {"round_robin", Json::Object()}, + }, + }; + } else { + endpoint_picking_policy = it->second; + } + parse_error = GRPC_ERROR_NONE; + if (LoadBalancingPolicyRegistry::ParseLoadBalancingConfig( + endpoint_picking_policy, &parse_error) == nullptr) { + GPR_DEBUG_ASSERT(parse_error != GRPC_ERROR_NONE); + error_list.push_back(GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( + field_name, &parse_error, 1)); + GRPC_ERROR_UNREF(parse_error); + } + // Fallback policy. + Json fallback_policy_config; + it = json.object_value().find("fallbackPolicy"); + if (it == json.object_value().end()) { + fallback_policy_config = Json::Array{Json::Object{ + {"round_robin", Json::Object()}, + }}; + } else { + fallback_policy_config = it->second; + } + parse_error = GRPC_ERROR_NONE; + RefCountedPtr fallback_policy = + LoadBalancingPolicyRegistry::ParseLoadBalancingConfig( + fallback_policy_config, &parse_error); + if (fallback_policy == nullptr) { + GPR_DEBUG_ASSERT(parse_error != GRPC_ERROR_NONE); + error_list.push_back(GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( + "fallbackPolicy", &parse_error, 1)); + GRPC_ERROR_UNREF(parse_error); + error_list.push_back(parse_error); + } + if (error_list.empty()) { + return MakeRefCounted( + name_, std::move(cluster_name), std::move(eds_service_name), + std::move(lrs_load_reporting_server_name), + std::move(locality_picking_policy), + std::move(endpoint_picking_policy), std::move(fallback_policy)); + } else { + *error = GRPC_ERROR_CREATE_FROM_VECTOR( + "eds_experimental LB policy config", &error_list); + return nullptr; + } + } + + private: + class EdsChildHandler : public ChildPolicyHandler { + public: + EdsChildHandler(Args args, TraceFlag* tracer, const char* name) + : ChildPolicyHandler(std::move(args), tracer), name_(name) {} + + bool ConfigChangeRequiresNewPolicyInstance( + LoadBalancingPolicy::Config* old_config, + LoadBalancingPolicy::Config* new_config) const override { + GPR_ASSERT(old_config->name() == name_); + GPR_ASSERT(new_config->name() == name_); + EdsLbConfig* old_eds_config = static_cast(old_config); + EdsLbConfig* new_eds_config = static_cast(new_config); + return old_eds_config->cluster_name() != new_eds_config->cluster_name() || + old_eds_config->eds_service_name() != + new_eds_config->eds_service_name(); + } + + OrphanablePtr CreateLoadBalancingPolicy( + const char* name, LoadBalancingPolicy::Args args) const override { + return MakeOrphanable(name_, std::move(args)); + } + + private: + const char* name_; + }; + + const char* name_; +}; + +} // namespace + +} // namespace grpc_core + +// +// Plugin registration +// + +void grpc_lb_policy_eds_init() { + grpc_core::LoadBalancingPolicyRegistry::Builder:: + RegisterLoadBalancingPolicyFactory( + absl::make_unique(grpc_core::kEds)); + // TODO(roth): This is here just for backward compatibility with some + // old tests we have internally. Remove this once they are upgraded + // to use the new policy name and config. + grpc_core::LoadBalancingPolicyRegistry::Builder:: + RegisterLoadBalancingPolicyFactory( + absl::make_unique(grpc_core::kXds)); +} + +void grpc_lb_policy_eds_shutdown() {} diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc b/src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc new file mode 100644 index 00000000000..b91d1b3bb9d --- /dev/null +++ b/src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc @@ -0,0 +1,524 @@ +// +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include + +#include + +#include "src/core/ext/filters/client_channel/lb_policy.h" +#include "src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h" +#include "src/core/ext/filters/client_channel/lb_policy_factory.h" +#include "src/core/ext/filters/client_channel/lb_policy_registry.h" +#include "src/core/ext/filters/client_channel/xds/xds_client.h" +#include "src/core/ext/filters/client_channel/xds/xds_client_stats.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/gprpp/orphanable.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/iomgr/combiner.h" + +namespace grpc_core { + +TraceFlag grpc_lb_lrs_trace(false, "lrs_lb"); + +namespace { + +constexpr char kLrs[] = "lrs_experimental"; + +// Config for LRS LB policy. +class LrsLbConfig : public LoadBalancingPolicy::Config { + public: + LrsLbConfig(RefCountedPtr child_policy, + std::string cluster_name, std::string eds_service_name, + std::string lrs_load_reporting_server_name, + RefCountedPtr locality_name) + : child_policy_(std::move(child_policy)), + cluster_name_(std::move(cluster_name)), + eds_service_name_(std::move(eds_service_name)), + lrs_load_reporting_server_name_( + std::move(lrs_load_reporting_server_name)), + locality_name_(std::move(locality_name)) {} + + const char* name() const override { return kLrs; } + + RefCountedPtr child_policy() const { + return child_policy_; + } + const std::string& cluster_name() const { return cluster_name_; } + const std::string& eds_service_name() const { return eds_service_name_; } + const std::string& lrs_load_reporting_server_name() const { + return lrs_load_reporting_server_name_; + }; + RefCountedPtr locality_name() const { + return locality_name_; + } + + private: + RefCountedPtr child_policy_; + std::string cluster_name_; + std::string eds_service_name_; + std::string lrs_load_reporting_server_name_; + RefCountedPtr locality_name_; +}; + +// LRS LB policy. +class LrsLb : public LoadBalancingPolicy { + public: + LrsLb(RefCountedPtr xds_client, Args args); + + const char* name() const override { return kLrs; } + + void UpdateLocked(UpdateArgs args) override; + void ExitIdleLocked() override; + void ResetBackoffLocked() override; + + private: + // A simple wrapper for ref-counting a picker from the child policy. + class RefCountedPicker : public RefCounted { + public: + explicit RefCountedPicker(std::unique_ptr picker) + : picker_(std::move(picker)) {} + PickResult Pick(PickArgs args) { return picker_->Pick(args); } + + private: + std::unique_ptr picker_; + }; + + // A picker that wraps the picker from the child to perform load reporting. + class LoadReportingPicker : public SubchannelPicker { + public: + LoadReportingPicker(RefCountedPtr picker, + RefCountedPtr locality_stats) + : picker_(std::move(picker)), + locality_stats_(std::move(locality_stats)) {} + + PickResult Pick(PickArgs args); + + private: + RefCountedPtr picker_; + RefCountedPtr locality_stats_; + }; + + class Helper : public ChannelControlHelper { + public: + explicit Helper(RefCountedPtr lrs_policy) + : lrs_policy_(std::move(lrs_policy)) {} + + ~Helper() { lrs_policy_.reset(DEBUG_LOCATION, "Helper"); } + + RefCountedPtr CreateSubchannel( + const grpc_channel_args& args) override; + void UpdateState(grpc_connectivity_state state, + std::unique_ptr picker) override; + void RequestReresolution() override; + void AddTraceEvent(TraceSeverity severity, StringView message) override; + + private: + RefCountedPtr lrs_policy_; + }; + + ~LrsLb(); + + void ShutdownLocked() override; + + OrphanablePtr CreateChildPolicyLocked( + const grpc_channel_args* args); + void UpdateChildPolicyLocked(ServerAddressList addresses, + const grpc_channel_args* args); + + void MaybeUpdatePickerLocked(); + + // Current config from the resolver. + RefCountedPtr config_; + + // Internal state. + bool shutting_down_ = false; + + // The xds client. + RefCountedPtr xds_client_; + + // The stats for client-side load reporting. + RefCountedPtr locality_stats_; + + OrphanablePtr child_policy_; + + // Latest state and picker reported by the child policy. + grpc_connectivity_state state_ = GRPC_CHANNEL_IDLE; + RefCountedPtr picker_; +}; + +// +// LrsLb::LoadReportingPicker +// + +LoadBalancingPolicy::PickResult LrsLb::LoadReportingPicker::Pick( + LoadBalancingPolicy::PickArgs args) { + // Forward the pick to the picker returned from the child policy. + PickResult result = picker_->Pick(args); + if (result.type == PickResult::PICK_COMPLETE && + result.subchannel != nullptr) { + // Record a call started. + locality_stats_->AddCallStarted(); + // Intercept the recv_trailing_metadata op to record call completion. + XdsClusterLocalityStats* locality_stats = + locality_stats_->Ref(DEBUG_LOCATION, "LocalityStats+call").release(); + result.recv_trailing_metadata_ready = + // Note: This callback does not run in either the control plane + // combiner or in the data plane mutex. + [locality_stats](grpc_error* error, MetadataInterface* /*metadata*/, + CallState* /*call_state*/) { + const bool call_failed = error != GRPC_ERROR_NONE; + locality_stats->AddCallFinished(call_failed); + locality_stats->Unref(DEBUG_LOCATION, "LocalityStats+call"); + }; + } + return result; +} + +// +// LrsLb +// + +LrsLb::LrsLb(RefCountedPtr xds_client, Args args) + : LoadBalancingPolicy(std::move(args)), xds_client_(std::move(xds_client)) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_lrs_trace)) { + gpr_log(GPR_INFO, "[lrs_lb %p] created -- using xds client %p from channel", + this, xds_client_.get()); + } +} + +LrsLb::~LrsLb() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_lrs_trace)) { + gpr_log(GPR_INFO, "[lrs_lb %p] destroying xds LB policy", this); + } +} + +void LrsLb::ShutdownLocked() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_lrs_trace)) { + gpr_log(GPR_INFO, "[lrs_lb %p] shutting down", this); + } + shutting_down_ = true; + // Remove the child policy's interested_parties pollset_set from the + // xDS policy. + if (child_policy_ != nullptr) { + grpc_pollset_set_del_pollset_set(child_policy_->interested_parties(), + interested_parties()); + child_policy_.reset(); + } + // Drop our ref to the child's picker, in case it's holding a ref to + // the child. + picker_.reset(); + locality_stats_.reset(); + xds_client_.reset(); +} + +void LrsLb::ExitIdleLocked() { + if (child_policy_ != nullptr) child_policy_->ExitIdleLocked(); +} + +void LrsLb::ResetBackoffLocked() { + // The XdsClient will have its backoff reset by the xds resolver, so we + // don't need to do it here. + if (child_policy_ != nullptr) child_policy_->ResetBackoffLocked(); +} + +void LrsLb::UpdateLocked(UpdateArgs args) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_lrs_trace)) { + gpr_log(GPR_INFO, "[lrs_lb %p] Received update", this); + } + // Update config. + auto old_config = std::move(config_); + config_ = std::move(args.config); + // Update load reporting if needed. + if (old_config == nullptr || + config_->lrs_load_reporting_server_name() != + old_config->lrs_load_reporting_server_name() || + config_->cluster_name() != old_config->cluster_name() || + config_->eds_service_name() != old_config->eds_service_name() || + *config_->locality_name() != *old_config->locality_name()) { + locality_stats_ = xds_client_->AddClusterLocalityStats( + config_->lrs_load_reporting_server_name(), config_->cluster_name(), + config_->eds_service_name(), config_->locality_name()); + MaybeUpdatePickerLocked(); + } + // Update child policy. + UpdateChildPolicyLocked(std::move(args.addresses), args.args); + args.args = nullptr; // Ownership passed to UpdateChildPolicyLocked(). +} + +void LrsLb::MaybeUpdatePickerLocked() { + if (picker_ != nullptr) { + auto lrs_picker = + absl::make_unique(picker_, locality_stats_); + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_lrs_trace)) { + gpr_log(GPR_INFO, "[lrs_lb %p] updating connectivity: state=%s picker=%p", + this, ConnectivityStateName(state_), lrs_picker.get()); + } + channel_control_helper()->UpdateState(state_, std::move(lrs_picker)); + } +} + +OrphanablePtr LrsLb::CreateChildPolicyLocked( + const grpc_channel_args* args) { + LoadBalancingPolicy::Args lb_policy_args; + lb_policy_args.combiner = combiner(); + lb_policy_args.args = args; + lb_policy_args.channel_control_helper = + absl::make_unique(Ref(DEBUG_LOCATION, "Helper")); + OrphanablePtr lb_policy = + MakeOrphanable(std::move(lb_policy_args), + &grpc_lb_lrs_trace); + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_lrs_trace)) { + gpr_log(GPR_INFO, "[lrs_lb %p] Created new child policy handler %p", this, + lb_policy.get()); + } + // Add our interested_parties pollset_set to that of the newly created + // child policy. This will make the child policy progress upon activity on + // this policy, which in turn is tied to the application's call. + grpc_pollset_set_add_pollset_set(lb_policy->interested_parties(), + interested_parties()); + return lb_policy; +} + +void LrsLb::UpdateChildPolicyLocked(ServerAddressList addresses, + const grpc_channel_args* args) { + // Create policy if needed. + if (child_policy_ == nullptr) { + child_policy_ = CreateChildPolicyLocked(args); + } + // Construct update args. + UpdateArgs update_args; + update_args.addresses = std::move(addresses); + update_args.config = config_->child_policy(); + update_args.args = args; + // Update the policy. + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_lrs_trace)) { + gpr_log(GPR_INFO, "[lrs_lb %p] Updating child policy handler %p", this, + child_policy_.get()); + } + child_policy_->UpdateLocked(std::move(update_args)); +} + +// +// LrsLb::Helper +// + +RefCountedPtr LrsLb::Helper::CreateSubchannel( + const grpc_channel_args& args) { + if (lrs_policy_->shutting_down_) return nullptr; + return lrs_policy_->channel_control_helper()->CreateSubchannel(args); +} + +void LrsLb::Helper::UpdateState(grpc_connectivity_state state, + std::unique_ptr picker) { + if (lrs_policy_->shutting_down_) return; + if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_lrs_trace)) { + gpr_log(GPR_INFO, + "[lrs_lb %p] child connectivity state update: state=%s picker=%p", + lrs_policy_.get(), ConnectivityStateName(state), picker.get()); + } + // Save the state and picker. + lrs_policy_->state_ = state; + lrs_policy_->picker_ = MakeRefCounted(std::move(picker)); + // Wrap the picker and return it to the channel. + lrs_policy_->MaybeUpdatePickerLocked(); +} + +void LrsLb::Helper::RequestReresolution() { + if (lrs_policy_->shutting_down_) return; + lrs_policy_->channel_control_helper()->RequestReresolution(); +} + +void LrsLb::Helper::AddTraceEvent(TraceSeverity severity, StringView message) { + if (lrs_policy_->shutting_down_) return; + lrs_policy_->channel_control_helper()->AddTraceEvent(severity, message); +} + +// +// factory +// + +class LrsLbFactory : public LoadBalancingPolicyFactory { + public: + OrphanablePtr CreateLoadBalancingPolicy( + LoadBalancingPolicy::Args args) const override { + RefCountedPtr xds_client = + XdsClient::GetFromChannelArgs(*args.args); + if (xds_client == nullptr) { + gpr_log(GPR_ERROR, + "XdsClient not present in channel args -- cannot instantiate " + "lrs LB policy"); + return nullptr; + } + return MakeOrphanable(std::move(xds_client), std::move(args)); + } + + const char* name() const override { return kLrs; } + + RefCountedPtr ParseLoadBalancingConfig( + const Json& json, grpc_error** error) const override { + GPR_DEBUG_ASSERT(error != nullptr && *error == GRPC_ERROR_NONE); + if (json.type() == Json::Type::JSON_NULL) { + // lrs was mentioned as a policy in the deprecated loadBalancingPolicy + // field or in the client API. + *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:loadBalancingPolicy error:lrs policy requires configuration. " + "Please use loadBalancingConfig field of service config instead."); + return nullptr; + } + std::vector error_list; + // Child policy. + RefCountedPtr child_policy; + auto it = json.object_value().find("childPolicy"); + if (it == json.object_value().end()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:childPolicy error:required field missing")); + } else { + grpc_error* parse_error = GRPC_ERROR_NONE; + child_policy = LoadBalancingPolicyRegistry::ParseLoadBalancingConfig( + it->second, &parse_error); + if (child_policy == nullptr) { + GPR_DEBUG_ASSERT(parse_error != GRPC_ERROR_NONE); + std::vector child_errors; + child_errors.push_back(parse_error); + error_list.push_back( + GRPC_ERROR_CREATE_FROM_VECTOR("field:childPolicy", &child_errors)); + } + } + // Cluster name. + std::string cluster_name; + it = json.object_value().find("clusterName"); + if (it == json.object_value().end()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:clusterName error:required field missing")); + } else if (it->second.type() != Json::Type::STRING) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:clusterName error:type should be string")); + } else { + cluster_name = it->second.string_value(); + } + // EDS service name. + std::string eds_service_name; + it = json.object_value().find("edsServiceName"); + if (it != json.object_value().end()) { + if (it->second.type() != Json::Type::STRING) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:edsServiceName error:type should be string")); + } else { + eds_service_name = it->second.string_value(); + } + } + // Locality. + RefCountedPtr locality_name; + it = json.object_value().find("locality"); + if (it == json.object_value().end()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:locality error:required field missing")); + } else { + std::vector child_errors = + ParseLocality(it->second, &locality_name); + if (!child_errors.empty()) { + error_list.push_back( + GRPC_ERROR_CREATE_FROM_VECTOR("field:locality", &child_errors)); + } + } + // LRS load reporting server name. + std::string lrs_load_reporting_server_name; + it = json.object_value().find("lrsLoadReportingServerName"); + if (it == json.object_value().end()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:lrsLoadReportingServerName error:required field missing")); + } else if (it->second.type() != Json::Type::STRING) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:lrsLoadReportingServerName error:type should be string")); + } else { + lrs_load_reporting_server_name = it->second.string_value(); + } + if (!error_list.empty()) { + *error = GRPC_ERROR_CREATE_FROM_VECTOR( + "lrs_experimental LB policy config", &error_list); + return nullptr; + } + return MakeRefCounted( + std::move(child_policy), std::move(cluster_name), + std::move(eds_service_name), std::move(lrs_load_reporting_server_name), + std::move(locality_name)); + } + + private: + static std::vector ParseLocality( + const Json& json, RefCountedPtr* name) { + std::vector error_list; + if (json.type() != Json::Type::OBJECT) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "locality field is not an object")); + return error_list; + } + std::string region; + auto it = json.object_value().find("region"); + if (it != json.object_value().end()) { + if (it->second.type() != Json::Type::STRING) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "\"region\" field is not a string")); + } else { + region = it->second.string_value(); + } + } + std::string zone; + it = json.object_value().find("zone"); + if (it != json.object_value().end()) { + if (it->second.type() != Json::Type::STRING) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "\"zone\" field is not a string")); + } else { + zone = it->second.string_value(); + } + } + std::string subzone; + it = json.object_value().find("subzone"); + if (it != json.object_value().end()) { + if (it->second.type() != Json::Type::STRING) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "\"subzone\" field is not a string")); + } else { + subzone = it->second.string_value(); + } + } + if (region.empty() && zone.empty() && subzone.empty()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "at least one of region, zone, or subzone must be set")); + } + if (error_list.empty()) { + *name = MakeRefCounted(region, zone, subzone); + } + return error_list; + } +}; + +} // namespace + +} // namespace grpc_core + +// +// Plugin registration +// + +void grpc_lb_policy_lrs_init() { + grpc_core::LoadBalancingPolicyRegistry::Builder:: + RegisterLoadBalancingPolicyFactory( + absl::make_unique()); +} + +void grpc_lb_policy_lrs_shutdown() {} diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc b/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc deleted file mode 100644 index 9196bf628ca..00000000000 --- a/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc +++ /dev/null @@ -1,1754 +0,0 @@ -/* - * - * Copyright 2018 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#include - -#include "src/core/lib/iomgr/sockaddr.h" -#include "src/core/lib/iomgr/socket_utils.h" - -#include -#include -#include - -#include "absl/types/optional.h" - -#include -#include -#include -#include - -#include "src/core/ext/filters/client_channel/client_channel.h" -#include "src/core/ext/filters/client_channel/lb_policy.h" -#include "src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h" -#include "src/core/ext/filters/client_channel/lb_policy/xds/xds.h" -#include "src/core/ext/filters/client_channel/lb_policy_factory.h" -#include "src/core/ext/filters/client_channel/lb_policy_registry.h" -#include "src/core/ext/filters/client_channel/parse_address.h" -#include "src/core/ext/filters/client_channel/server_address.h" -#include "src/core/ext/filters/client_channel/service_config.h" -#include "src/core/ext/filters/client_channel/xds/xds_client.h" -#include "src/core/ext/filters/client_channel/xds/xds_client_stats.h" -#include "src/core/lib/backoff/backoff.h" -#include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/channel/channel_stack.h" -#include "src/core/lib/gpr/string.h" -#include "src/core/lib/gprpp/manual_constructor.h" -#include "src/core/lib/gprpp/map.h" -#include "src/core/lib/gprpp/memory.h" -#include "src/core/lib/gprpp/orphanable.h" -#include "src/core/lib/gprpp/ref_counted_ptr.h" -#include "src/core/lib/gprpp/sync.h" -#include "src/core/lib/iomgr/combiner.h" -#include "src/core/lib/iomgr/sockaddr.h" -#include "src/core/lib/iomgr/sockaddr_utils.h" -#include "src/core/lib/iomgr/timer.h" -#include "src/core/lib/slice/slice_hash_table.h" -#include "src/core/lib/slice/slice_internal.h" -#include "src/core/lib/slice/slice_string_helpers.h" -#include "src/core/lib/surface/call.h" -#include "src/core/lib/surface/channel.h" -#include "src/core/lib/surface/channel_init.h" -#include "src/core/lib/transport/static_metadata.h" - -#define GRPC_XDS_DEFAULT_FALLBACK_TIMEOUT_MS 10000 -#define GRPC_XDS_DEFAULT_LOCALITY_RETENTION_INTERVAL_MS (15 * 60 * 1000) -#define GRPC_XDS_DEFAULT_FAILOVER_TIMEOUT_MS 10000 - -namespace grpc_core { - -TraceFlag grpc_lb_xds_trace(false, "xds_lb"); - -namespace { - -constexpr char kXds[] = "xds_experimental"; - -class XdsConfig : public LoadBalancingPolicy::Config { - public: - XdsConfig(RefCountedPtr child_policy, - RefCountedPtr fallback_policy, - std::string eds_service_name, - absl::optional lrs_load_reporting_server_name) - : child_policy_(std::move(child_policy)), - fallback_policy_(std::move(fallback_policy)), - eds_service_name_(std::move(eds_service_name)), - lrs_load_reporting_server_name_( - std::move(lrs_load_reporting_server_name)) {} - - const char* name() const override { return kXds; } - - RefCountedPtr child_policy() const { - return child_policy_; - } - - RefCountedPtr fallback_policy() const { - return fallback_policy_; - } - - const char* eds_service_name() const { - return eds_service_name_.empty() ? nullptr : eds_service_name_.c_str(); - }; - - const absl::optional& lrs_load_reporting_server_name() const { - return lrs_load_reporting_server_name_; - }; - - private: - RefCountedPtr child_policy_; - RefCountedPtr fallback_policy_; - std::string eds_service_name_; - absl::optional lrs_load_reporting_server_name_; -}; - -class XdsLb : public LoadBalancingPolicy { - public: - explicit XdsLb(Args args); - - const char* name() const override { return kXds; } - - void UpdateLocked(UpdateArgs args) override; - void ResetBackoffLocked() override; - - private: - class EndpointWatcher; - - // A simple wrapper to convert the picker returned from a locality's child - // policy as a unique_ptr<> to a RefCountedPtr<>. This allows it to be - // referenced by both the picker and the locality. - class RefCountedEndpointPicker : public RefCounted { - public: - explicit RefCountedEndpointPicker(std::unique_ptr picker) - : picker_(std::move(picker)) {} - PickResult Pick(PickArgs args) { return picker_->Pick(args); } - - private: - std::unique_ptr picker_; - }; - - // A picker that wraps the RefCountedEndpointPicker and performs load - // reporting for the locality. - class LoadReportingPicker : public RefCounted { - public: - LoadReportingPicker(RefCountedPtr picker, - RefCountedPtr locality_stats) - : picker_(std::move(picker)), - locality_stats_(std::move(locality_stats)) {} - - PickResult Pick(PickArgs args); - - RefCountedEndpointPicker* picker() const { return picker_.get(); } - XdsClusterLocalityStats* locality_stats() const { - return locality_stats_.get(); - } - - private: - RefCountedPtr picker_; - RefCountedPtr locality_stats_; - }; - - // A picker that uses a stateless weighting algorithm to pick the locality - // to use for each request. - class LocalityPicker : public SubchannelPicker { - public: - // Maintains a weighted list of pickers from each locality that is in ready - // state. The first element in the pair represents the end of a range - // proportional to the locality's weight. The start of the range is the - // previous value in the vector and is 0 for the first element. - using PickerList = - InlinedVector>, - 1>; - LocalityPicker(XdsLb* xds_policy, PickerList pickers) - : drop_stats_(xds_policy->drop_stats_), - drop_config_(xds_policy->drop_config_), - pickers_(std::move(pickers)) {} - - PickResult Pick(PickArgs args) override; - - private: - // Calls the picker of the locality that the key falls within. - PickResult PickFromLocality(const uint32_t key, PickArgs args); - - RefCountedPtr drop_stats_; - RefCountedPtr drop_config_; - PickerList pickers_; - }; - - class FallbackHelper : public ChannelControlHelper { - public: - explicit FallbackHelper(RefCountedPtr parent) - : parent_(std::move(parent)) {} - - ~FallbackHelper() { parent_.reset(DEBUG_LOCATION, "FallbackHelper"); } - - RefCountedPtr CreateSubchannel( - const grpc_channel_args& args) override; - void UpdateState(grpc_connectivity_state state, - std::unique_ptr picker) override; - void RequestReresolution() override; - void AddTraceEvent(TraceSeverity severity, StringView message) override; - - private: - RefCountedPtr parent_; - }; - - // Each LocalityMap holds a ref to the XdsLb. - class LocalityMap : public InternallyRefCounted { - public: - // Each Locality holds a ref to the LocalityMap it is in. - class Locality : public InternallyRefCounted { - public: - Locality(RefCountedPtr locality_map, - RefCountedPtr name); - ~Locality(); - - void UpdateLocked(uint32_t locality_weight, ServerAddressList serverlist, - bool update_locality_stats); - void ShutdownLocked(); - void ResetBackoffLocked(); - void DeactivateLocked(); - void Orphan() override; - - uint32_t weight() const { return weight_; } - - grpc_connectivity_state connectivity_state() const { - return connectivity_state_; - } - - RefCountedPtr GetLoadReportingPicker() { - // Recreate load reporting picker if stats object has changed. - if (load_reporting_picker_ == nullptr || - load_reporting_picker_->picker() != picker_wrapper_.get() || - load_reporting_picker_->locality_stats() != stats_.get()) { - load_reporting_picker_ = - MakeRefCounted(picker_wrapper_, stats_); - } - return load_reporting_picker_; - } - - void set_locality_map(RefCountedPtr locality_map) { - locality_map_ = std::move(locality_map); - } - - private: - class Helper : public ChannelControlHelper { - public: - explicit Helper(RefCountedPtr locality) - : locality_(std::move(locality)) {} - - ~Helper() { locality_.reset(DEBUG_LOCATION, "Helper"); } - - RefCountedPtr CreateSubchannel( - const grpc_channel_args& args) override; - void UpdateState(grpc_connectivity_state state, - std::unique_ptr picker) override; - // This is a no-op, because we get the addresses from the xds - // client, which is a watch-based API. - void RequestReresolution() override {} - void AddTraceEvent(TraceSeverity severity, StringView message) override; - - private: - RefCountedPtr locality_; - }; - - // Methods for dealing with the child policy. - OrphanablePtr CreateChildPolicyLocked( - const grpc_channel_args* args); - grpc_channel_args* CreateChildPolicyArgsLocked( - const grpc_channel_args* args); - - void UpdateLocalityStats(); - - static void OnDelayedRemovalTimer(void* arg, grpc_error* error); - static void OnDelayedRemovalTimerLocked(void* arg, grpc_error* error); - - XdsLb* xds_policy() const { return locality_map_->xds_policy(); } - - // The owning locality map. - RefCountedPtr locality_map_; - - RefCountedPtr name_; - RefCountedPtr stats_; - OrphanablePtr child_policy_; - RefCountedPtr picker_wrapper_; - RefCountedPtr load_reporting_picker_; - grpc_connectivity_state connectivity_state_ = GRPC_CHANNEL_IDLE; - uint32_t weight_; - - // States for delayed removal. - grpc_timer delayed_removal_timer_; - grpc_closure on_delayed_removal_timer_; - bool delayed_removal_timer_callback_pending_ = false; - bool shutdown_ = false; - }; - - LocalityMap(RefCountedPtr xds_policy, uint32_t priority); - - ~LocalityMap() { xds_policy_.reset(DEBUG_LOCATION, "LocalityMap"); } - - void UpdateLocked( - const XdsApi::PriorityListUpdate::LocalityMap& priority_update, - bool update_locality_stats); - void ResetBackoffLocked(); - void UpdateXdsPickerLocked(); - OrphanablePtr ExtractLocalityLocked( - const RefCountedPtr& name); - void DeactivateLocked(); - // Returns true if this locality map becomes the currently used one (i.e., - // its priority is selected) after reactivation. - bool MaybeReactivateLocked(); - void MaybeCancelFailoverTimerLocked(); - - void Orphan() override; - - XdsLb* xds_policy() const { return xds_policy_.get(); } - uint32_t priority() const { return priority_; } - grpc_connectivity_state connectivity_state() const { - return connectivity_state_; - } - bool failover_timer_callback_pending() const { - return failover_timer_callback_pending_; - } - - private: - void OnLocalityStateUpdateLocked(); - void UpdateConnectivityStateLocked(); - static void OnDelayedRemovalTimer(void* arg, grpc_error* error); - static void OnFailoverTimer(void* arg, grpc_error* error); - static void OnDelayedRemovalTimerLocked(void* arg, grpc_error* error); - static void OnFailoverTimerLocked(void* arg, grpc_error* error); - - const XdsApi::PriorityListUpdate& priority_list_update() const { - return xds_policy_->priority_list_update_; - } - const XdsApi::PriorityListUpdate::LocalityMap* locality_map_update() const { - return xds_policy_->priority_list_update_.Find(priority_); - } - - RefCountedPtr xds_policy_; - - std::map, OrphanablePtr, - XdsLocalityName::Less> - localities_; - const uint32_t priority_; - grpc_connectivity_state connectivity_state_ = GRPC_CHANNEL_IDLE; - - // States for delayed removal. - grpc_timer delayed_removal_timer_; - grpc_closure on_delayed_removal_timer_; - bool delayed_removal_timer_callback_pending_ = false; - - // States of failover. - grpc_timer failover_timer_; - grpc_closure on_failover_timer_; - bool failover_timer_callback_pending_ = false; - }; - - ~XdsLb(); - - void ShutdownLocked() override; - - const char* eds_service_name() const { - if (config_ != nullptr && config_->eds_service_name() != nullptr) { - return config_->eds_service_name(); - } - return server_name_.c_str(); - } - - XdsClient* xds_client() const { - return xds_client_from_channel_ != nullptr ? xds_client_from_channel_.get() - : xds_client_.get(); - } - - void UpdatePrioritiesLocked(bool update_locality_stats); - void UpdateXdsPickerLocked(); - void MaybeCreateLocalityMapLocked(uint32_t priority); - void FailoverOnConnectionFailureLocked(); - void FailoverOnDisconnectionLocked(uint32_t failed_priority); - void SwitchToHigherPriorityLocked(uint32_t priority); - void DeactivatePrioritiesLowerThan(uint32_t priority); - OrphanablePtr ExtractLocalityLocked( - const RefCountedPtr& name, uint32_t exclude_priority); - // Callers should make sure the priority list is non-empty. - uint32_t LowestPriority() const { - return static_cast(priorities_.size()) - 1; - } - bool Contains(uint32_t priority) { return priority < priorities_.size(); } - - // Methods for dealing with fallback state. - void MaybeCancelFallbackAtStartupChecks(); - static void OnFallbackTimer(void* arg, grpc_error* error); - static void OnFallbackTimerLocked(void* arg, grpc_error* error); - void UpdateFallbackPolicyLocked(); - OrphanablePtr CreateFallbackPolicyLocked( - const grpc_channel_args* args); - void MaybeExitFallbackMode(); - - // Server name from target URI. - std::string server_name_; - - // Current channel args and config from the resolver. - const grpc_channel_args* args_ = nullptr; - RefCountedPtr config_; - - // Internal state. - bool shutting_down_ = false; - - // The xds client and endpoint watcher. - // If we get the XdsClient from the channel, we store it in - // xds_client_from_channel_; if we create it ourselves, we store it in - // xds_client_. - RefCountedPtr xds_client_from_channel_; - OrphanablePtr xds_client_; - // A pointer to the endpoint watcher, to be used when cancelling the watch. - // Note that this is not owned, so this pointer must never be derefernced. - EndpointWatcher* endpoint_watcher_ = nullptr; - - // Whether the checks for fallback at startup are ALL pending. There are - // several cases where this can be reset: - // 1. The fallback timer fires, we enter fallback mode. - // 2. Before the fallback timer fires, the endpoint watcher reports an - // error, we enter fallback mode. - // 3. Before the fallback timer fires, if any child policy in the locality map - // becomes READY, we cancel the fallback timer. - bool fallback_at_startup_checks_pending_ = false; - // Timeout in milliseconds for before using fallback backend addresses. - // 0 means not using fallback. - const grpc_millis lb_fallback_timeout_ms_; - // The backend addresses from the resolver. - ServerAddressList fallback_backend_addresses_; - // Fallback timer. - grpc_timer lb_fallback_timer_; - grpc_closure lb_on_fallback_; - - // Non-null iff we are in fallback mode. - OrphanablePtr fallback_policy_; - - const grpc_millis locality_retention_interval_ms_; - const grpc_millis locality_map_failover_timeout_ms_; - // The list of locality maps, indexed by priority. P0 is the highest - // priority. - InlinedVector, 2> priorities_; - // The priority that is being used. - uint32_t current_priority_ = UINT32_MAX; - // The update for priority_list_. - XdsApi::PriorityListUpdate priority_list_update_; - - // The config for dropping calls. - RefCountedPtr drop_config_; - - // Drop stats for client-side load reporting. - RefCountedPtr drop_stats_; -}; - -// -// XdsLb::LoadReportingPicker -// - -LoadBalancingPolicy::PickResult XdsLb::LoadReportingPicker::Pick( - LoadBalancingPolicy::PickArgs args) { - // Forward the pick to the picker returned from the child policy. - PickResult result = picker_->Pick(args); - if (result.type != PickResult::PICK_COMPLETE || - result.subchannel == nullptr || locality_stats_ == nullptr) { - return result; - } - // Record a call started. - locality_stats_->AddCallStarted(); - // Intercept the recv_trailing_metadata op to record call completion. - XdsClusterLocalityStats* locality_stats = - locality_stats_->Ref(DEBUG_LOCATION, "LocalityStats+call").release(); - result.recv_trailing_metadata_ready = - // Note: This callback does not run in either the control plane - // combiner or in the data plane mutex. - [locality_stats](grpc_error* error, MetadataInterface* /*metadata*/, - CallState* /*call_state*/) { - const bool call_failed = error != GRPC_ERROR_NONE; - locality_stats->AddCallFinished(call_failed); - locality_stats->Unref(DEBUG_LOCATION, "LocalityStats+call"); - }; - return result; -} - -// -// XdsLb::LocalityPicker -// - -XdsLb::PickResult XdsLb::LocalityPicker::Pick(PickArgs args) { - // Handle drop. - const std::string* drop_category; - if (drop_config_->ShouldDrop(&drop_category)) { - if (drop_stats_ != nullptr) drop_stats_->AddCallDropped(*drop_category); - PickResult result; - result.type = PickResult::PICK_COMPLETE; - return result; - } - // If we didn't drop, we better have some localities to pick from. - if (pickers_.empty()) { // Should never happen. - PickResult result; - result.type = PickResult::PICK_FAILED; - result.error = - grpc_error_set_int(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "xds picker not given any localities"), - GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_INTERNAL); - return result; - } - // Generate a random number in [0, total weight). - const uint32_t key = rand() % pickers_[pickers_.size() - 1].first; - // Forward pick to whichever locality maps to the range in which the - // random number falls in. - return PickFromLocality(key, args); -} - -XdsLb::PickResult XdsLb::LocalityPicker::PickFromLocality(const uint32_t key, - PickArgs args) { - size_t mid = 0; - size_t start_index = 0; - size_t end_index = pickers_.size() - 1; - size_t index = 0; - while (end_index > start_index) { - mid = (start_index + end_index) / 2; - if (pickers_[mid].first > key) { - end_index = mid; - } else if (pickers_[mid].first < key) { - start_index = mid + 1; - } else { - index = mid + 1; - break; - } - } - if (index == 0) index = start_index; - GPR_ASSERT(pickers_[index].first > key); - return pickers_[index].second->Pick(args); -} - -// -// XdsLb::FallbackHelper -// - -RefCountedPtr XdsLb::FallbackHelper::CreateSubchannel( - const grpc_channel_args& args) { - if (parent_->shutting_down_) return nullptr; - return parent_->channel_control_helper()->CreateSubchannel(args); -} - -void XdsLb::FallbackHelper::UpdateState( - grpc_connectivity_state state, std::unique_ptr picker) { - if (parent_->shutting_down_) return; - parent_->channel_control_helper()->UpdateState(state, std::move(picker)); -} - -void XdsLb::FallbackHelper::RequestReresolution() { - if (parent_->shutting_down_) return; - parent_->channel_control_helper()->RequestReresolution(); -} - -void XdsLb::FallbackHelper::AddTraceEvent(TraceSeverity severity, - StringView message) { - if (parent_->shutting_down_) return; - parent_->channel_control_helper()->AddTraceEvent(severity, message); -} - -// -// XdsLb::EndpointWatcher -// - -class XdsLb::EndpointWatcher : public XdsClient::EndpointWatcherInterface { - public: - explicit EndpointWatcher(RefCountedPtr xds_policy) - : xds_policy_(std::move(xds_policy)) {} - - ~EndpointWatcher() { xds_policy_.reset(DEBUG_LOCATION, "EndpointWatcher"); } - - void OnEndpointChanged(XdsApi::EdsUpdate update) override { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] Received EDS update from xds client", - xds_policy_.get()); - } - // If the balancer tells us to drop all the calls, we should exit fallback - // mode immediately. - if (update.drop_config->drop_all()) xds_policy_->MaybeExitFallbackMode(); - // Update the drop config. - const bool drop_config_changed = - xds_policy_->drop_config_ == nullptr || - *xds_policy_->drop_config_ != *update.drop_config; - xds_policy_->drop_config_ = std::move(update.drop_config); - // Ignore identical locality update. - if (xds_policy_->priority_list_update_ == update.priority_list_update) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, - "[xdslb %p] Incoming locality update identical to current, " - "ignoring. (drop_config_changed=%d)", - xds_policy_.get(), drop_config_changed); - } - if (drop_config_changed) { - xds_policy_->UpdateXdsPickerLocked(); - } - return; - } - // Update the priority list. - xds_policy_->priority_list_update_ = std::move(update.priority_list_update); - xds_policy_->UpdatePrioritiesLocked(false /*update_locality_stats*/); - } - - void OnError(grpc_error* error) override { - // If the fallback-at-startup checks are pending, go into fallback mode - // immediately. This short-circuits the timeout for the - // fallback-at-startup case. - if (xds_policy_->fallback_at_startup_checks_pending_) { - gpr_log(GPR_INFO, - "[xdslb %p] xds watcher reported error; entering fallback " - "mode: %s", - xds_policy_.get(), grpc_error_string(error)); - xds_policy_->fallback_at_startup_checks_pending_ = false; - grpc_timer_cancel(&xds_policy_->lb_fallback_timer_); - xds_policy_->UpdateFallbackPolicyLocked(); - // If the xds call failed, request re-resolution. - // TODO(roth): We check the error string contents here to - // differentiate between the xds call failing and the xds channel - // going into TRANSIENT_FAILURE. This is a pretty ugly hack, - // but it's okay for now, since we're not yet sure whether we will - // continue to support the current fallback functionality. If we - // decide to keep the fallback approach, then we should either - // find a cleaner way to expose the difference between these two - // cases or decide that we're okay re-resolving in both cases. - // Note that even if we do keep the current fallback functionality, - // this re-resolution will only be necessary if we are going to be - // using this LB policy with resolvers other than the xds resolver. - if (strstr(grpc_error_string(error), "xds call failed")) { - xds_policy_->channel_control_helper()->RequestReresolution(); - } - } else if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] xds watcher reported error (ignoring): %s", - xds_policy_.get(), grpc_error_string(error)); - } - GRPC_ERROR_UNREF(error); - } - - private: - RefCountedPtr xds_policy_; -}; - -// -// ctor and dtor -// - -XdsLb::XdsLb(Args args) - : LoadBalancingPolicy(std::move(args)), - xds_client_from_channel_(XdsClient::GetFromChannelArgs(*args.args)), - lb_fallback_timeout_ms_(grpc_channel_args_find_integer( - args.args, GRPC_ARG_XDS_FALLBACK_TIMEOUT_MS, - {GRPC_XDS_DEFAULT_FALLBACK_TIMEOUT_MS, 0, INT_MAX})), - locality_retention_interval_ms_(grpc_channel_args_find_integer( - args.args, GRPC_ARG_LOCALITY_RETENTION_INTERVAL_MS, - {GRPC_XDS_DEFAULT_LOCALITY_RETENTION_INTERVAL_MS, 0, INT_MAX})), - locality_map_failover_timeout_ms_(grpc_channel_args_find_integer( - args.args, GRPC_ARG_XDS_FAILOVER_TIMEOUT_MS, - {GRPC_XDS_DEFAULT_FAILOVER_TIMEOUT_MS, 0, INT_MAX})) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] created -- xds client from channel: %p", this, - xds_client_from_channel_.get()); - } - // Record server name. - const grpc_arg* arg = grpc_channel_args_find(args.args, GRPC_ARG_SERVER_URI); - const char* server_uri = grpc_channel_arg_get_string(arg); - GPR_ASSERT(server_uri != nullptr); - grpc_uri* uri = grpc_uri_parse(server_uri, true); - GPR_ASSERT(uri->path[0] != '\0'); - server_name_ = uri->path[0] == '/' ? uri->path + 1 : uri->path; - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] server name from channel: %s", this, - server_name_.c_str()); - } - grpc_uri_destroy(uri); -} - -XdsLb::~XdsLb() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] destroying xds LB policy", this); - } - grpc_channel_args_destroy(args_); -} - -void XdsLb::ShutdownLocked() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] shutting down", this); - } - shutting_down_ = true; - MaybeCancelFallbackAtStartupChecks(); - priorities_.clear(); - drop_stats_.reset(); - if (fallback_policy_ != nullptr) { - grpc_pollset_set_del_pollset_set(fallback_policy_->interested_parties(), - interested_parties()); - fallback_policy_.reset(); - } - // Cancel the endpoint watch here instead of in our dtor if we are using the - // XdsResolver, because the watcher holds a ref to us and we might not be - // destroying the Xds client leading to a situation where the Xds lb policy is - // never destroyed. - if (xds_client_from_channel_ != nullptr) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] cancelling watch for %s", this, - eds_service_name()); - } - xds_client()->CancelEndpointDataWatch(StringView(eds_service_name()), - endpoint_watcher_); - xds_client_from_channel_.reset(); - } - xds_client_.reset(); -} - -// -// public methods -// - -void XdsLb::ResetBackoffLocked() { - // When the XdsClient is instantiated in the resolver instead of in this - // LB policy, this is done via the resolver, so we don't need to do it - // for xds_client_from_channel_ here. - if (xds_client_ != nullptr) xds_client_->ResetBackoff(); - for (size_t i = 0; i < priorities_.size(); ++i) { - priorities_[i]->ResetBackoffLocked(); - } - if (fallback_policy_ != nullptr) { - fallback_policy_->ResetBackoffLocked(); - } -} - -void XdsLb::UpdateLocked(UpdateArgs args) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] Received update", this); - } - const bool is_initial_update = args_ == nullptr; - // Update config. - auto old_config = std::move(config_); - config_ = std::move(args.config); - // Update fallback address list. - fallback_backend_addresses_ = std::move(args.addresses); - // Update args. - grpc_channel_args_destroy(args_); - args_ = args.args; - args.args = nullptr; - // Update the existing fallback policy. The fallback policy config and/or the - // fallback addresses may be new. - if (fallback_policy_ != nullptr) UpdateFallbackPolicyLocked(); - if (is_initial_update) { - // Initialize XdsClient. - if (xds_client_from_channel_ == nullptr) { - grpc_error* error = GRPC_ERROR_NONE; - xds_client_ = MakeOrphanable( - combiner(), interested_parties(), StringView(eds_service_name()), - nullptr /* service config watcher */, *args_, &error); - // TODO(roth): If we decide that we care about fallback mode, add - // proper error handling here. - GPR_ASSERT(error == GRPC_ERROR_NONE); - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] Created xds client %p", this, - xds_client_.get()); - } - } - // Start fallback-at-startup checks. - grpc_millis deadline = ExecCtx::Get()->Now() + lb_fallback_timeout_ms_; - Ref(DEBUG_LOCATION, "on_fallback_timer").release(); // Held by closure - GRPC_CLOSURE_INIT(&lb_on_fallback_, &XdsLb::OnFallbackTimer, this, - grpc_schedule_on_exec_ctx); - fallback_at_startup_checks_pending_ = true; - grpc_timer_init(&lb_fallback_timer_, deadline, &lb_on_fallback_); - } - // Update drop stats for load reporting if needed. - if (is_initial_update || config_->lrs_load_reporting_server_name() != - old_config->lrs_load_reporting_server_name()) { - drop_stats_.reset(); - if (config_->lrs_load_reporting_server_name().has_value()) { - drop_stats_ = xds_client()->AddClusterDropStats( - config_->lrs_load_reporting_server_name().value(), - // TODO(roth): We currently hard-code the assumption that - // cluster name and EDS service name are the same. Fix this - // as part of refectoring this LB policy. - eds_service_name(), eds_service_name()); - } - } - // On the initial update, create the endpoint watcher. - if (is_initial_update) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] starting watch for %s", this, - eds_service_name()); - } - auto watcher = absl::make_unique( - Ref(DEBUG_LOCATION, "EndpointWatcher")); - endpoint_watcher_ = watcher.get(); - xds_client()->WatchEndpointData(StringView(eds_service_name()), - std::move(watcher)); - } else { - // Update priority list. - // Note that this comes after updating drop_stats_, since we want that - // to be used by any new picker we create here. - // No need to do this on the initial update, since there won't be any - // priorities to update yet. - const bool update_locality_stats = - config_->lrs_load_reporting_server_name() != - old_config->lrs_load_reporting_server_name(); - UpdatePrioritiesLocked(update_locality_stats); - } -} - -// -// fallback-related methods -// - -void XdsLb::MaybeCancelFallbackAtStartupChecks() { - if (!fallback_at_startup_checks_pending_) return; - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] Cancelling fallback timer", this); - } - grpc_timer_cancel(&lb_fallback_timer_); - fallback_at_startup_checks_pending_ = false; -} - -void XdsLb::OnFallbackTimer(void* arg, grpc_error* error) { - XdsLb* xdslb_policy = static_cast(arg); - xdslb_policy->combiner()->Run( - GRPC_CLOSURE_INIT(&xdslb_policy->lb_on_fallback_, - &XdsLb::OnFallbackTimerLocked, xdslb_policy, nullptr), - GRPC_ERROR_REF(error)); -} - -void XdsLb::OnFallbackTimerLocked(void* arg, grpc_error* error) { - XdsLb* xdslb_policy = static_cast(arg); - // If some fallback-at-startup check is done after the timer fires but before - // this callback actually runs, don't fall back. - if (xdslb_policy->fallback_at_startup_checks_pending_ && - !xdslb_policy->shutting_down_ && error == GRPC_ERROR_NONE) { - gpr_log(GPR_INFO, - "[xdslb %p] Child policy not ready after fallback timeout; " - "entering fallback mode", - xdslb_policy); - xdslb_policy->fallback_at_startup_checks_pending_ = false; - xdslb_policy->UpdateFallbackPolicyLocked(); - } - xdslb_policy->Unref(DEBUG_LOCATION, "on_fallback_timer"); -} - -void XdsLb::UpdateFallbackPolicyLocked() { - if (shutting_down_) return; - // Create policy if needed. - if (fallback_policy_ == nullptr) { - fallback_policy_ = CreateFallbackPolicyLocked(args_); - GPR_ASSERT(fallback_policy_ != nullptr); - } - // Perform update. - UpdateArgs update_args; - update_args.addresses = fallback_backend_addresses_; - update_args.config = config_->fallback_policy(); - update_args.args = grpc_channel_args_copy(args_); - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] Updating fallback child policy handler %p", - this, fallback_policy_.get()); - } - fallback_policy_->UpdateLocked(std::move(update_args)); -} - -OrphanablePtr XdsLb::CreateFallbackPolicyLocked( - const grpc_channel_args* args) { - LoadBalancingPolicy::Args lb_policy_args; - lb_policy_args.combiner = combiner(); - lb_policy_args.args = args; - lb_policy_args.channel_control_helper = - absl::make_unique(Ref(DEBUG_LOCATION, "FallbackHelper")); - OrphanablePtr lb_policy = - MakeOrphanable(std::move(lb_policy_args), - &grpc_lb_xds_trace); - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, - "[xdslb %p] Created new fallback child policy handler (%p)", this, - lb_policy.get()); - } - // Add the xDS's interested_parties pollset_set to that of the newly created - // child policy. This will make the child policy progress upon activity on xDS - // LB, which in turn is tied to the application's call. - grpc_pollset_set_add_pollset_set(lb_policy->interested_parties(), - interested_parties()); - return lb_policy; -} - -void XdsLb::MaybeExitFallbackMode() { - if (fallback_policy_ == nullptr) return; - gpr_log(GPR_INFO, "[xdslb %p] Exiting fallback mode", this); - fallback_policy_.reset(); -} - -// -// priority list-related methods -// - -void XdsLb::UpdatePrioritiesLocked(bool update_locality_stats) { - // 1. Remove from the priority list the priorities that are not in the update. - DeactivatePrioritiesLowerThan(priority_list_update_.LowestPriority()); - // 2. Update all the existing priorities. - for (uint32_t priority = 0; priority < priorities_.size(); ++priority) { - LocalityMap* locality_map = priorities_[priority].get(); - const auto* locality_map_update = priority_list_update_.Find(priority); - // If we have more current priorities than exist in the update, stop here. - if (locality_map_update == nullptr) break; - // Propagate locality_map_update. - // TODO(juanlishen): Find a clean way to skip duplicate update for a - // priority. - locality_map->UpdateLocked(*locality_map_update, update_locality_stats); - } - // 3. Only create a new locality map if all the existing ones have failed. - if (priorities_.empty() || - !priorities_[priorities_.size() - 1]->failover_timer_callback_pending()) { - const uint32_t new_priority = static_cast(priorities_.size()); - // Create a new locality map. Note that in some rare cases (e.g., the - // locality map reports TRANSIENT_FAILURE synchronously due to subchannel - // sharing), the following invocation may result in multiple locality maps - // to be created. - MaybeCreateLocalityMapLocked(new_priority); - } - // 4. If we updated locality stats and we already have at least one - // priority, update the picker to start using the new stats object(s). - if (update_locality_stats && !priorities_.empty()) { - UpdateXdsPickerLocked(); - } -} - -void XdsLb::UpdateXdsPickerLocked() { - // If we are in fallback mode, don't generate an xds picker from localities. - if (fallback_policy_ != nullptr) return; - // If we're dropping all calls, report READY, even though we won't - // have a selected priority. - if (drop_config_ != nullptr && drop_config_->drop_all()) { - channel_control_helper()->UpdateState( - GRPC_CHANNEL_READY, - absl::make_unique(this, LocalityPicker::PickerList{})); - return; - } - // If we don't have a selected priority, report TRANSIENT_FAILURE. - if (current_priority_ == UINT32_MAX) { - if (fallback_policy_ == nullptr) { - grpc_error* error = grpc_error_set_int( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("no ready locality map"), - GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE); - channel_control_helper()->UpdateState( - GRPC_CHANNEL_TRANSIENT_FAILURE, - absl::make_unique(error)); - } - return; - } - priorities_[current_priority_]->UpdateXdsPickerLocked(); -} - -void XdsLb::MaybeCreateLocalityMapLocked(uint32_t priority) { - // Exhausted priorities in the update. - if (!priority_list_update_.Contains(priority)) return; - auto new_locality_map = - new LocalityMap(Ref(DEBUG_LOCATION, "LocalityMap"), priority); - priorities_.emplace_back(OrphanablePtr(new_locality_map)); - new_locality_map->UpdateLocked(*priority_list_update_.Find(priority), - false /*update_locality_stats*/); -} - -void XdsLb::FailoverOnConnectionFailureLocked() { - const uint32_t failed_priority = LowestPriority(); - // If we're failing over from the lowest priority, report TRANSIENT_FAILURE. - if (failed_priority == priority_list_update_.LowestPriority()) { - UpdateXdsPickerLocked(); - } - MaybeCreateLocalityMapLocked(failed_priority + 1); -} - -void XdsLb::FailoverOnDisconnectionLocked(uint32_t failed_priority) { - current_priority_ = UINT32_MAX; - for (uint32_t next_priority = failed_priority + 1; - next_priority <= priority_list_update_.LowestPriority(); - ++next_priority) { - if (!Contains(next_priority)) { - MaybeCreateLocalityMapLocked(next_priority); - return; - } - if (priorities_[next_priority]->MaybeReactivateLocked()) return; - } -} - -void XdsLb::SwitchToHigherPriorityLocked(uint32_t priority) { - current_priority_ = priority; - DeactivatePrioritiesLowerThan(current_priority_); - UpdateXdsPickerLocked(); -} - -void XdsLb::DeactivatePrioritiesLowerThan(uint32_t priority) { - if (priorities_.empty()) return; - // Deactivate the locality maps from the lowest priority. - for (uint32_t p = LowestPriority(); p > priority; --p) { - if (locality_retention_interval_ms_ == 0) { - priorities_.pop_back(); - } else { - priorities_[p]->DeactivateLocked(); - } - } -} - -OrphanablePtr XdsLb::ExtractLocalityLocked( - const RefCountedPtr& name, uint32_t exclude_priority) { - for (uint32_t priority = 0; priority < priorities_.size(); ++priority) { - if (priority == exclude_priority) continue; - LocalityMap* locality_map = priorities_[priority].get(); - auto locality = locality_map->ExtractLocalityLocked(name); - if (locality != nullptr) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, - "[xdslb %p] moving locality %p %s to new priority (%" PRIu32 - " -> %" PRIu32 ")", - this, locality.get(), name->AsHumanReadableString(), - exclude_priority, priority); - } - return locality; - } - } - return nullptr; -} - -// -// XdsLb::LocalityMap -// - -XdsLb::LocalityMap::LocalityMap(RefCountedPtr xds_policy, - uint32_t priority) - : xds_policy_(std::move(xds_policy)), priority_(priority) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] Creating priority %" PRIu32, - xds_policy_.get(), priority_); - } - GRPC_CLOSURE_INIT(&on_failover_timer_, OnFailoverTimer, this, - grpc_schedule_on_exec_ctx); - // Start the failover timer. - Ref(DEBUG_LOCATION, "LocalityMap+OnFailoverTimerLocked").release(); - grpc_timer_init( - &failover_timer_, - ExecCtx::Get()->Now() + xds_policy_->locality_map_failover_timeout_ms_, - &on_failover_timer_); - failover_timer_callback_pending_ = true; - // This is the first locality map ever created, report CONNECTING. - if (priority_ == 0 && xds_policy_->fallback_policy_ == nullptr) { - xds_policy_->channel_control_helper()->UpdateState( - GRPC_CHANNEL_CONNECTING, - absl::make_unique( - xds_policy_->Ref(DEBUG_LOCATION, "QueuePicker"))); - } -} - -void XdsLb::LocalityMap::UpdateLocked( - const XdsApi::PriorityListUpdate::LocalityMap& priority_update, - bool update_locality_stats) { - if (xds_policy_->shutting_down_) return; - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] Start Updating priority %" PRIu32, - xds_policy(), priority_); - } - // Maybe reactivate the locality map in case all the active locality maps have - // failed. - MaybeReactivateLocked(); - // Remove (later) the localities not in priority_update. - for (auto iter = localities_.begin(); iter != localities_.end();) { - const auto& name = iter->first; - Locality* locality = iter->second.get(); - if (priority_update.Contains(name)) { - ++iter; - continue; - } - if (xds_policy()->locality_retention_interval_ms_ == 0) { - iter = localities_.erase(iter); - } else { - locality->DeactivateLocked(); - ++iter; - } - } - // Add or update the localities in priority_update. - for (const auto& p : priority_update.localities) { - const auto& name = p.first; - const auto& locality_update = p.second; - OrphanablePtr& locality = localities_[name]; - if (locality == nullptr) { - // Move from another locality map if possible. - locality = xds_policy_->ExtractLocalityLocked(name, priority_); - if (locality != nullptr) { - locality->set_locality_map( - Ref(DEBUG_LOCATION, "LocalityMap+Locality_move")); - } else { - locality = MakeOrphanable( - Ref(DEBUG_LOCATION, "LocalityMap+Locality"), name); - } - } - // Keep a copy of serverlist in the update so that we can compare it - // with the future ones. - locality->UpdateLocked(locality_update.lb_weight, - locality_update.serverlist, update_locality_stats); - } - // If this is the current priority and we removed all of the READY - // localities, go into state CONNECTING. - // TODO(roth): Ideally, we should model this as a graceful policy - // switch: we should keep using the old localities for a short period - // of time, long enough to give the new localities a chance to get - // connected. As part of refactoring this policy, we should try to - // fix that. - if (priority_ == xds_policy()->current_priority_) { - bool found_ready = false; - for (auto& p : localities_) { - const auto& locality_name = p.first; - Locality* locality = p.second.get(); - if (!locality_map_update()->Contains(locality_name)) continue; - if (locality->connectivity_state() == GRPC_CHANNEL_READY) { - found_ready = true; - break; - } - } - if (!found_ready) { - xds_policy_->channel_control_helper()->UpdateState( - GRPC_CHANNEL_CONNECTING, - absl::make_unique( - xds_policy_->Ref(DEBUG_LOCATION, "QueuePicker"))); - xds_policy_->current_priority_ = UINT32_MAX; - } - } -} - -void XdsLb::LocalityMap::ResetBackoffLocked() { - for (auto& p : localities_) p.second->ResetBackoffLocked(); -} - -void XdsLb::LocalityMap::UpdateXdsPickerLocked() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] constructing new picker", xds_policy()); - } - // Construct a new xds picker which maintains a map of all locality pickers - // that are ready. Each locality is represented by a portion of the range - // proportional to its weight, such that the total range is the sum of the - // weights of all localities. - LocalityPicker::PickerList picker_list; - uint32_t end = 0; - for (auto& p : localities_) { - const auto& locality_name = p.first; - Locality* locality = p.second.get(); - // Skip the localities that are not in the latest locality map update. - const auto* locality_update = locality_map_update(); - if (locality_update == nullptr) continue; - if (!locality_update->Contains(locality_name)) continue; - if (locality->connectivity_state() != GRPC_CHANNEL_READY) continue; - end += locality->weight(); - picker_list.push_back( - std::make_pair(end, locality->GetLoadReportingPicker())); - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] locality=%s weight=%d picker=%p", - xds_policy(), locality_name->AsHumanReadableString(), - locality->weight(), picker_list.back().second.get()); - } - } - xds_policy()->channel_control_helper()->UpdateState( - GRPC_CHANNEL_READY, - absl::make_unique(xds_policy(), std::move(picker_list))); -} - -OrphanablePtr -XdsLb::LocalityMap::ExtractLocalityLocked( - const RefCountedPtr& name) { - for (auto iter = localities_.begin(); iter != localities_.end(); ++iter) { - const auto& name_in_map = iter->first; - if (*name_in_map == *name) { - auto locality = std::move(iter->second); - localities_.erase(iter); - return locality; - } - } - return nullptr; -} - -void XdsLb::LocalityMap::DeactivateLocked() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] deactivating priority %" PRIu32, xds_policy(), - priority_); - } - // If already deactivated, don't do it again. - if (delayed_removal_timer_callback_pending_) return; - MaybeCancelFailoverTimerLocked(); - // Start a timer to delete the locality. - Ref(DEBUG_LOCATION, "LocalityMap+timer").release(); - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, - "[xdslb %p] Will remove priority %" PRIu32 " in %" PRId64 " ms.", - xds_policy(), priority_, - xds_policy()->locality_retention_interval_ms_); - } - GRPC_CLOSURE_INIT(&on_delayed_removal_timer_, OnDelayedRemovalTimer, this, - grpc_schedule_on_exec_ctx); - grpc_timer_init( - &delayed_removal_timer_, - ExecCtx::Get()->Now() + xds_policy()->locality_retention_interval_ms_, - &on_delayed_removal_timer_); - delayed_removal_timer_callback_pending_ = true; -} - -bool XdsLb::LocalityMap::MaybeReactivateLocked() { - // Don't reactivate a priority that is not higher than the current one. - if (priority_ >= xds_policy_->current_priority_) return false; - // Reactivate this priority by cancelling deletion timer. - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] reactivating priority %" PRIu32, xds_policy(), - priority_); - } - if (delayed_removal_timer_callback_pending_) { - grpc_timer_cancel(&delayed_removal_timer_); - } - // Switch to this higher priority if it's READY. - if (connectivity_state_ != GRPC_CHANNEL_READY) return false; - xds_policy_->SwitchToHigherPriorityLocked(priority_); - return true; -} - -void XdsLb::LocalityMap::MaybeCancelFailoverTimerLocked() { - if (failover_timer_callback_pending_) grpc_timer_cancel(&failover_timer_); -} - -void XdsLb::LocalityMap::Orphan() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] Priority %" PRIu32 " orphaned.", xds_policy(), - priority_); - } - MaybeCancelFailoverTimerLocked(); - if (delayed_removal_timer_callback_pending_) { - grpc_timer_cancel(&delayed_removal_timer_); - } - localities_.clear(); - Unref(DEBUG_LOCATION, "LocalityMap+Orphan"); -} - -void XdsLb::LocalityMap::OnLocalityStateUpdateLocked() { - UpdateConnectivityStateLocked(); - // Ignore priorities not in priority_list_update. - if (!priority_list_update().Contains(priority_)) return; - const uint32_t current_priority = xds_policy_->current_priority_; - // Ignore lower-than-current priorities. - if (priority_ > current_priority) return; - // Maybe update fallback state. - if (connectivity_state_ == GRPC_CHANNEL_READY) { - xds_policy_->MaybeCancelFallbackAtStartupChecks(); - xds_policy_->MaybeExitFallbackMode(); - } - // Update is for a higher-than-current priority. (Special case: update is for - // any active priority if there is no current priority.) - if (priority_ < current_priority) { - if (connectivity_state_ == GRPC_CHANNEL_READY) { - MaybeCancelFailoverTimerLocked(); - // If a higher-than-current priority becomes READY, switch to use it. - xds_policy_->SwitchToHigherPriorityLocked(priority_); - } else if (connectivity_state_ == GRPC_CHANNEL_TRANSIENT_FAILURE) { - // If a higher-than-current priority becomes TRANSIENT_FAILURE, only - // handle it if it's the priority that is still in failover timeout. - if (failover_timer_callback_pending_) { - MaybeCancelFailoverTimerLocked(); - xds_policy_->FailoverOnConnectionFailureLocked(); - } - } - return; - } - // Update is for current priority. - if (connectivity_state_ != GRPC_CHANNEL_READY) { - // Fail over if it's no longer READY. - xds_policy_->FailoverOnDisconnectionLocked(priority_); - } - // At this point, one of the following things has happened to the current - // priority. - // 1. It remained the same (but received picker update from its localities). - // 2. It changed to a lower priority due to failover. - // 3. It became invalid because failover didn't yield a READY priority. - // In any case, update the xds picker. - xds_policy_->UpdateXdsPickerLocked(); -} - -void XdsLb::LocalityMap::UpdateConnectivityStateLocked() { - size_t num_ready = 0; - size_t num_connecting = 0; - size_t num_idle = 0; - size_t num_transient_failures = 0; - for (const auto& p : localities_) { - const auto& locality_name = p.first; - const Locality* locality = p.second.get(); - // Skip the localities that are not in the latest locality map update. - if (!locality_map_update()->Contains(locality_name)) continue; - switch (locality->connectivity_state()) { - case GRPC_CHANNEL_READY: { - ++num_ready; - break; - } - case GRPC_CHANNEL_CONNECTING: { - ++num_connecting; - break; - } - case GRPC_CHANNEL_IDLE: { - ++num_idle; - break; - } - case GRPC_CHANNEL_TRANSIENT_FAILURE: { - ++num_transient_failures; - break; - } - default: - GPR_UNREACHABLE_CODE(return ); - } - } - if (num_ready > 0) { - connectivity_state_ = GRPC_CHANNEL_READY; - } else if (num_connecting > 0) { - connectivity_state_ = GRPC_CHANNEL_CONNECTING; - } else if (num_idle > 0) { - connectivity_state_ = GRPC_CHANNEL_IDLE; - } else { - connectivity_state_ = GRPC_CHANNEL_TRANSIENT_FAILURE; - } - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, - "[xdslb %p] Priority %" PRIu32 " (%p) connectivity changed to %s", - xds_policy(), priority_, this, - ConnectivityStateName(connectivity_state_)); - } -} - -void XdsLb::LocalityMap::OnDelayedRemovalTimer(void* arg, grpc_error* error) { - LocalityMap* self = static_cast(arg); - self->xds_policy_->combiner()->Run( - GRPC_CLOSURE_INIT(&self->on_delayed_removal_timer_, - OnDelayedRemovalTimerLocked, self, nullptr), - GRPC_ERROR_REF(error)); -} - -void XdsLb::LocalityMap::OnDelayedRemovalTimerLocked(void* arg, - grpc_error* error) { - LocalityMap* self = static_cast(arg); - self->delayed_removal_timer_callback_pending_ = false; - if (error == GRPC_ERROR_NONE && !self->xds_policy_->shutting_down_) { - const bool keep = self->priority_list_update().Contains(self->priority_) && - self->priority_ <= self->xds_policy_->current_priority_; - if (!keep) { - // This check is to make sure we always delete the locality maps from - // the lowest priority even if the closures of the back-to-back timers - // are not run in FIFO order. - // TODO(juanlishen): Eliminate unnecessary maintenance overhead for some - // deactivated locality maps when out-of-order closures are run. - // TODO(juanlishen): Check the timer implementation to see if this - // defense is necessary. - if (self->priority_ == self->xds_policy_->LowestPriority()) { - self->xds_policy_->priorities_.pop_back(); - } else { - gpr_log(GPR_ERROR, - "[xdslb %p] Priority %" PRIu32 - " is not the lowest priority (highest numeric value) but is " - "attempted to be deleted.", - self->xds_policy(), self->priority_); - } - } - } - self->Unref(DEBUG_LOCATION, "LocalityMap+timer"); -} - -void XdsLb::LocalityMap::OnFailoverTimer(void* arg, grpc_error* error) { - LocalityMap* self = static_cast(arg); - self->xds_policy_->combiner()->Run( - GRPC_CLOSURE_INIT(&self->on_failover_timer_, OnFailoverTimerLocked, self, - nullptr), - GRPC_ERROR_REF(error)); -} - -void XdsLb::LocalityMap::OnFailoverTimerLocked(void* arg, grpc_error* error) { - LocalityMap* self = static_cast(arg); - self->failover_timer_callback_pending_ = false; - if (error == GRPC_ERROR_NONE && !self->xds_policy_->shutting_down_) { - self->xds_policy_->FailoverOnConnectionFailureLocked(); - } - self->Unref(DEBUG_LOCATION, "LocalityMap+OnFailoverTimerLocked"); -} - -// -// XdsLb::LocalityMap::Locality -// - -XdsLb::LocalityMap::Locality::Locality(RefCountedPtr locality_map, - RefCountedPtr name) - : locality_map_(std::move(locality_map)), name_(std::move(name)) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] created Locality %p for %s", xds_policy(), - this, name_->AsHumanReadableString()); - } - // Initialize locality stats if load reporting is enabled. - UpdateLocalityStats(); -} - -XdsLb::LocalityMap::Locality::~Locality() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] Locality %p %s: destroying locality", - xds_policy(), this, name_->AsHumanReadableString()); - } - locality_map_.reset(DEBUG_LOCATION, "Locality"); -} - -void XdsLb::LocalityMap::Locality::UpdateLocalityStats() { - stats_.reset(); - if (xds_policy()->config_->lrs_load_reporting_server_name().has_value()) { - stats_ = xds_policy()->xds_client()->AddClusterLocalityStats( - xds_policy()->config_->lrs_load_reporting_server_name().value(), - // TODO(roth): We currently hard-code the assumption that - // cluster name and EDS service name are the same. Fix this - // as part of refectoring this LB policy. - xds_policy()->eds_service_name(), xds_policy()->eds_service_name(), - name_); - } -} - -grpc_channel_args* XdsLb::LocalityMap::Locality::CreateChildPolicyArgsLocked( - const grpc_channel_args* args_in) { - const grpc_arg args_to_add[] = { - // A channel arg indicating if the target is a backend inferred from a - // grpclb load balancer. - grpc_channel_arg_integer_create( - const_cast(GRPC_ARG_ADDRESS_IS_BACKEND_FROM_XDS_LOAD_BALANCER), - 1), - // Inhibit client-side health checking, since the balancer does - // this for us. - grpc_channel_arg_integer_create( - const_cast(GRPC_ARG_INHIBIT_HEALTH_CHECKING), 1), - }; - return grpc_channel_args_copy_and_add(args_in, args_to_add, - GPR_ARRAY_SIZE(args_to_add)); -} - -OrphanablePtr -XdsLb::LocalityMap::Locality::CreateChildPolicyLocked( - const grpc_channel_args* args) { - LoadBalancingPolicy::Args lb_policy_args; - lb_policy_args.combiner = xds_policy()->combiner(); - lb_policy_args.args = args; - lb_policy_args.channel_control_helper = - absl::make_unique(this->Ref(DEBUG_LOCATION, "Helper")); - OrphanablePtr lb_policy = - MakeOrphanable(std::move(lb_policy_args), - &grpc_lb_xds_trace); - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, - "[xdslb %p] Locality %p %s: Created new child policy handler (%p)", - xds_policy(), this, name_->AsHumanReadableString(), - lb_policy.get()); - } - // Add the xDS's interested_parties pollset_set to that of the newly created - // child policy. This will make the child policy progress upon activity on - // xDS LB, which in turn is tied to the application's call. - grpc_pollset_set_add_pollset_set(lb_policy->interested_parties(), - xds_policy()->interested_parties()); - return lb_policy; -} - -void XdsLb::LocalityMap::Locality::UpdateLocked(uint32_t locality_weight, - ServerAddressList serverlist, - bool update_locality_stats) { - if (xds_policy()->shutting_down_) return; - // Update locality weight. - weight_ = locality_weight; - if (delayed_removal_timer_callback_pending_) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] Locality %p %s: reactivating", xds_policy(), - this, name_->AsHumanReadableString()); - } - grpc_timer_cancel(&delayed_removal_timer_); - } - // Update locality stats. - if (update_locality_stats) UpdateLocalityStats(); - // Construct update args. - UpdateArgs update_args; - update_args.addresses = std::move(serverlist); - update_args.config = xds_policy()->config_->child_policy(); - update_args.args = CreateChildPolicyArgsLocked(xds_policy()->args_); - // Create child policy if needed. - if (child_policy_ == nullptr) { - child_policy_ = CreateChildPolicyLocked(update_args.args); - GPR_ASSERT(child_policy_ != nullptr); - } - // Update the policy. - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, - "[xdslb %p] Locality %p %s: Updating child policy handler %p", - xds_policy(), this, name_->AsHumanReadableString(), - child_policy_.get()); - } - child_policy_->UpdateLocked(std::move(update_args)); -} - -void XdsLb::LocalityMap::Locality::ShutdownLocked() { - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] Locality %p %s: shutting down locality", - xds_policy(), this, name_->AsHumanReadableString()); - } - stats_.reset(); - // Remove the child policy's interested_parties pollset_set from the - // xDS policy. - grpc_pollset_set_del_pollset_set(child_policy_->interested_parties(), - xds_policy()->interested_parties()); - child_policy_.reset(); - // Drop our ref to the child's picker, in case it's holding a ref to - // the child. - load_reporting_picker_.reset(); - picker_wrapper_.reset(); - if (delayed_removal_timer_callback_pending_) { - grpc_timer_cancel(&delayed_removal_timer_); - } - shutdown_ = true; -} - -void XdsLb::LocalityMap::Locality::ResetBackoffLocked() { - child_policy_->ResetBackoffLocked(); -} - -void XdsLb::LocalityMap::Locality::Orphan() { - ShutdownLocked(); - Unref(); -} - -void XdsLb::LocalityMap::Locality::DeactivateLocked() { - // If already deactivated, don't do that again. - if (weight_ == 0) return; - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, "[xdslb %p] Locality %p %s: deactivating", xds_policy(), - this, name_->AsHumanReadableString()); - } - // Set the locality weight to 0 so that future xds picker won't contain this - // locality. - weight_ = 0; - // Start a timer to delete the locality. - Ref(DEBUG_LOCATION, "Locality+timer").release(); - GRPC_CLOSURE_INIT(&on_delayed_removal_timer_, OnDelayedRemovalTimer, this, - grpc_schedule_on_exec_ctx); - grpc_timer_init( - &delayed_removal_timer_, - ExecCtx::Get()->Now() + xds_policy()->locality_retention_interval_ms_, - &on_delayed_removal_timer_); - delayed_removal_timer_callback_pending_ = true; -} - -void XdsLb::LocalityMap::Locality::OnDelayedRemovalTimer(void* arg, - grpc_error* error) { - Locality* self = static_cast(arg); - self->xds_policy()->combiner()->Run( - GRPC_CLOSURE_INIT(&self->on_delayed_removal_timer_, - OnDelayedRemovalTimerLocked, self, nullptr), - GRPC_ERROR_REF(error)); -} - -void XdsLb::LocalityMap::Locality::OnDelayedRemovalTimerLocked( - void* arg, grpc_error* error) { - Locality* self = static_cast(arg); - self->delayed_removal_timer_callback_pending_ = false; - if (error == GRPC_ERROR_NONE && !self->shutdown_ && self->weight_ == 0) { - self->locality_map_->localities_.erase(self->name_); - } - self->Unref(DEBUG_LOCATION, "Locality+timer"); -} - -// -// XdsLb::LocalityMap::Locality::Helper -// - -RefCountedPtr -XdsLb::LocalityMap::Locality::Helper::CreateSubchannel( - const grpc_channel_args& args) { - if (locality_->xds_policy()->shutting_down_) return nullptr; - return locality_->xds_policy()->channel_control_helper()->CreateSubchannel( - args); -} - -void XdsLb::LocalityMap::Locality::Helper::UpdateState( - grpc_connectivity_state state, std::unique_ptr picker) { - if (locality_->xds_policy()->shutting_down_) return; - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_xds_trace)) { - gpr_log(GPR_INFO, - "[xdslb %p helper %p] child policy handler %p reports state=%s", - locality_->xds_policy(), this, locality_->child_policy_.get(), - ConnectivityStateName(state)); - } - // Cache the state and picker in the locality. - locality_->connectivity_state_ = state; - locality_->picker_wrapper_ = - MakeRefCounted(std::move(picker)); - // Notify the locality map. - locality_->locality_map_->OnLocalityStateUpdateLocked(); -} - -void XdsLb::LocalityMap::Locality::Helper::AddTraceEvent(TraceSeverity severity, - StringView message) { - if (locality_->xds_policy()->shutting_down_) return; - locality_->xds_policy()->channel_control_helper()->AddTraceEvent(severity, - message); -} - -// -// factory -// - -class XdsFactory : public LoadBalancingPolicyFactory { - public: - OrphanablePtr CreateLoadBalancingPolicy( - LoadBalancingPolicy::Args args) const override { - return MakeOrphanable(std::move(args), &grpc_lb_xds_trace); - } - - const char* name() const override { return kXds; } - - RefCountedPtr ParseLoadBalancingConfig( - const Json& json, grpc_error** error) const override { - GPR_DEBUG_ASSERT(error != nullptr && *error == GRPC_ERROR_NONE); - if (json.type() == Json::Type::JSON_NULL) { - // xds was mentioned as a policy in the deprecated loadBalancingPolicy - // field or in the client API. - *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "field:loadBalancingPolicy error:xds policy requires configuration. " - "Please use loadBalancingConfig field of service config instead."); - return nullptr; - } - std::vector error_list; - // Child policy. - Json json_tmp; - const Json* child_policy_json; - auto it = json.object_value().find("childPolicy"); - if (it == json.object_value().end()) { - json_tmp = Json::Array{Json::Object{ - {"round_robin", Json::Object()}, - }}; - child_policy_json = &json_tmp; - } else { - child_policy_json = &it->second; - } - grpc_error* parse_error = GRPC_ERROR_NONE; - RefCountedPtr child_policy = - LoadBalancingPolicyRegistry::ParseLoadBalancingConfig( - *child_policy_json, &parse_error); - if (child_policy == nullptr) { - GPR_DEBUG_ASSERT(parse_error != GRPC_ERROR_NONE); - std::vector child_errors; - child_errors.push_back(parse_error); - error_list.push_back( - GRPC_ERROR_CREATE_FROM_VECTOR("field:childPolicy", &child_errors)); - } - // Fallback policy. - const Json* fallback_policy_json; - it = json.object_value().find("fallbackPolicy"); - if (it == json.object_value().end()) { - json_tmp = Json::Array{Json::Object{ - {"round_robin", Json::Object()}, - }}; - fallback_policy_json = &json_tmp; - } else { - fallback_policy_json = &it->second; - } - RefCountedPtr fallback_policy = - LoadBalancingPolicyRegistry::ParseLoadBalancingConfig( - *fallback_policy_json, &parse_error); - if (fallback_policy == nullptr) { - GPR_DEBUG_ASSERT(parse_error != GRPC_ERROR_NONE); - std::vector child_errors; - child_errors.push_back(parse_error); - error_list.push_back( - GRPC_ERROR_CREATE_FROM_VECTOR("field:fallbackPolicy", &child_errors)); - } - // EDS service name. - const char* eds_service_name = nullptr; - it = json.object_value().find("edsServiceName"); - if (it != json.object_value().end()) { - if (it->second.type() != Json::Type::STRING) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "field:edsServiceName error:type should be string")); - } else { - eds_service_name = it->second.string_value().c_str(); - } - } - // LRS load reporting server name. - const char* lrs_load_reporting_server_name = nullptr; - it = json.object_value().find("lrsLoadReportingServerName"); - if (it != json.object_value().end()) { - if (it->second.type() != Json::Type::STRING) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "field:lrsLoadReportingServerName error:type should be string")); - } else { - lrs_load_reporting_server_name = it->second.string_value().c_str(); - } - } - if (error_list.empty()) { - absl::optional optional_lrs_load_reporting_server_name; - if (lrs_load_reporting_server_name != nullptr) { - optional_lrs_load_reporting_server_name.emplace( - std::string(lrs_load_reporting_server_name)); - } - return MakeRefCounted( - std::move(child_policy), std::move(fallback_policy), - eds_service_name == nullptr ? "" : eds_service_name, - std::move(optional_lrs_load_reporting_server_name)); - } else { - *error = GRPC_ERROR_CREATE_FROM_VECTOR("Xds Parser", &error_list); - return nullptr; - } - } - - private: - class XdsChildHandler : public ChildPolicyHandler { - public: - XdsChildHandler(Args args, TraceFlag* tracer) - : ChildPolicyHandler(std::move(args), tracer) {} - - bool ConfigChangeRequiresNewPolicyInstance( - LoadBalancingPolicy::Config* old_config, - LoadBalancingPolicy::Config* new_config) const override { - GPR_ASSERT(old_config->name() == kXds); - GPR_ASSERT(new_config->name() == kXds); - XdsConfig* old_xds_config = static_cast(old_config); - XdsConfig* new_xds_config = static_cast(new_config); - const char* old_eds_service_name = - old_xds_config->eds_service_name() == nullptr - ? "" - : old_xds_config->eds_service_name(); - const char* new_eds_service_name = - new_xds_config->eds_service_name() == nullptr - ? "" - : new_xds_config->eds_service_name(); - return strcmp(old_eds_service_name, new_eds_service_name) != 0; - } - - OrphanablePtr CreateLoadBalancingPolicy( - const char* name, LoadBalancingPolicy::Args args) const override { - return MakeOrphanable(std::move(args)); - } - }; -}; - -} // namespace - -} // namespace grpc_core - -// -// Plugin registration -// - -void grpc_lb_policy_xds_init() { - grpc_core::LoadBalancingPolicyRegistry::Builder:: - RegisterLoadBalancingPolicyFactory( - absl::make_unique()); -} - -void grpc_lb_policy_xds_shutdown() {} diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/xds.h b/src/core/ext/filters/client_channel/lb_policy/xds/xds.h index 13d3435da34..1de3472a3a0 100644 --- a/src/core/ext/filters/client_channel/lb_policy/xds/xds.h +++ b/src/core/ext/filters/client_channel/lb_policy/xds/xds.h @@ -29,5 +29,4 @@ #define GRPC_ARG_ADDRESS_IS_BACKEND_FROM_XDS_LOAD_BALANCER \ "grpc.address_is_backend_from_xds_load_balancer" -#endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_XDS_XDS_H \ - */ +#endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_XDS_XDS_H */ diff --git a/src/core/ext/filters/client_channel/lb_policy_registry.cc b/src/core/ext/filters/client_channel/lb_policy_registry.cc index 79e5f2f4926..498b158900e 100644 --- a/src/core/ext/filters/client_channel/lb_policy_registry.cc +++ b/src/core/ext/filters/client_channel/lb_policy_registry.cc @@ -35,6 +35,8 @@ class RegistryState { void RegisterLoadBalancingPolicyFactory( std::unique_ptr factory) { + gpr_log(GPR_DEBUG, "registering LB policy factory for \"%s\"", + factory->name()); for (size_t i = 0; i < factories_.size(); ++i) { GPR_ASSERT(strcmp(factories_[i]->name(), factory->name()) != 0); } diff --git a/src/core/ext/filters/client_channel/xds/xds_client_stats.h b/src/core/ext/filters/client_channel/xds/xds_client_stats.h index ab592ae6954..e0dd29d75ff 100644 --- a/src/core/ext/filters/client_channel/xds/xds_client_stats.h +++ b/src/core/ext/filters/client_channel/xds/xds_client_stats.h @@ -42,11 +42,7 @@ class XdsLocalityName : public RefCounted { struct Less { bool operator()(const XdsLocalityName* lhs, const XdsLocalityName* rhs) const { - int cmp_result = lhs->region_.compare(rhs->region_); - if (cmp_result != 0) return cmp_result < 0; - cmp_result = lhs->zone_.compare(rhs->zone_); - if (cmp_result != 0) return cmp_result < 0; - return lhs->sub_zone_.compare(rhs->sub_zone_) < 0; + return lhs->Compare(*rhs) < 0; } bool operator()(const RefCountedPtr& lhs, @@ -65,6 +61,18 @@ class XdsLocalityName : public RefCounted { sub_zone_ == other.sub_zone_; } + bool operator!=(const XdsLocalityName& other) const { + return !(*this == other); + } + + int Compare(const XdsLocalityName& other) const { + int cmp_result = region_.compare(other.region_); + if (cmp_result != 0) return cmp_result; + cmp_result = zone_.compare(other.zone_); + if (cmp_result != 0) return cmp_result; + return sub_zone_.compare(other.sub_zone_); + } + const std::string& region() const { return region_; } const std::string& zone() const { return zone_; } const std::string& sub_zone() const { return sub_zone_; } diff --git a/src/core/plugin_registry/grpc_plugin_registry.cc b/src/core/plugin_registry/grpc_plugin_registry.cc index 20ad526d837..b52c5be1719 100644 --- a/src/core/plugin_registry/grpc_plugin_registry.cc +++ b/src/core/plugin_registry/grpc_plugin_registry.cc @@ -36,8 +36,14 @@ void grpc_lb_policy_grpclb_init(void); void grpc_lb_policy_grpclb_shutdown(void); void grpc_lb_policy_cds_init(void); void grpc_lb_policy_cds_shutdown(void); -void grpc_lb_policy_xds_init(void); -void grpc_lb_policy_xds_shutdown(void); +void grpc_lb_policy_eds_init(void); +void grpc_lb_policy_eds_shutdown(void); +void grpc_lb_policy_lrs_init(void); +void grpc_lb_policy_lrs_shutdown(void); +void grpc_lb_policy_priority_init(void); +void grpc_lb_policy_priority_shutdown(void); +void grpc_lb_policy_weighted_target_init(void); +void grpc_lb_policy_weighted_target_shutdown(void); void grpc_lb_policy_pick_first_init(void); void grpc_lb_policy_pick_first_shutdown(void); void grpc_lb_policy_round_robin_init(void); @@ -78,8 +84,14 @@ void grpc_register_built_in_plugins(void) { grpc_lb_policy_grpclb_shutdown); grpc_register_plugin(grpc_lb_policy_cds_init, grpc_lb_policy_cds_shutdown); - grpc_register_plugin(grpc_lb_policy_xds_init, - grpc_lb_policy_xds_shutdown); + grpc_register_plugin(grpc_lb_policy_eds_init, + grpc_lb_policy_eds_shutdown); + grpc_register_plugin(grpc_lb_policy_lrs_init, + grpc_lb_policy_lrs_shutdown); + grpc_register_plugin(grpc_lb_policy_priority_init, + grpc_lb_policy_priority_shutdown); + grpc_register_plugin(grpc_lb_policy_weighted_target_init, + grpc_lb_policy_weighted_target_shutdown); grpc_register_plugin(grpc_lb_policy_pick_first_init, grpc_lb_policy_pick_first_shutdown); grpc_register_plugin(grpc_lb_policy_round_robin_init, diff --git a/src/core/plugin_registry/grpc_unsecure_plugin_registry.cc b/src/core/plugin_registry/grpc_unsecure_plugin_registry.cc index bfed2e22ddd..b1e442a07c0 100644 --- a/src/core/plugin_registry/grpc_unsecure_plugin_registry.cc +++ b/src/core/plugin_registry/grpc_unsecure_plugin_registry.cc @@ -44,8 +44,14 @@ void grpc_lb_policy_grpclb_init(void); void grpc_lb_policy_grpclb_shutdown(void); void grpc_lb_policy_cds_init(void); void grpc_lb_policy_cds_shutdown(void); -void grpc_lb_policy_xds_init(void); -void grpc_lb_policy_xds_shutdown(void); +void grpc_lb_policy_eds_init(void); +void grpc_lb_policy_eds_shutdown(void); +void grpc_lb_policy_lrs_init(void); +void grpc_lb_policy_lrs_shutdown(void); +void grpc_lb_policy_priority_init(void); +void grpc_lb_policy_priority_shutdown(void); +void grpc_lb_policy_weighted_target_init(void); +void grpc_lb_policy_weighted_target_shutdown(void); void grpc_lb_policy_pick_first_init(void); void grpc_lb_policy_pick_first_shutdown(void); void grpc_lb_policy_round_robin_init(void); @@ -86,8 +92,14 @@ void grpc_register_built_in_plugins(void) { grpc_lb_policy_grpclb_shutdown); grpc_register_plugin(grpc_lb_policy_cds_init, grpc_lb_policy_cds_shutdown); - grpc_register_plugin(grpc_lb_policy_xds_init, - grpc_lb_policy_xds_shutdown); + grpc_register_plugin(grpc_lb_policy_eds_init, + grpc_lb_policy_eds_shutdown); + grpc_register_plugin(grpc_lb_policy_lrs_init, + grpc_lb_policy_lrs_shutdown); + grpc_register_plugin(grpc_lb_policy_priority_init, + grpc_lb_policy_priority_shutdown); + grpc_register_plugin(grpc_lb_policy_weighted_target_init, + grpc_lb_policy_weighted_target_shutdown); grpc_register_plugin(grpc_lb_policy_pick_first_init, grpc_lb_policy_pick_first_shutdown); grpc_register_plugin(grpc_lb_policy_round_robin_init, diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 46cce67f77e..6c2dfed4ae3 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -28,6 +28,7 @@ CORE_SOURCE_FILES = [ 'src/core/ext/filters/client_channel/http_connect_handshaker.cc', 'src/core/ext/filters/client_channel/http_proxy.cc', 'src/core/ext/filters/client_channel/lb_policy.cc', + 'src/core/ext/filters/client_channel/lb_policy/address_filtering.cc', 'src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc', @@ -36,9 +37,12 @@ CORE_SOURCE_FILES = [ 'src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc', 'src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc', 'src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc', + 'src/core/ext/filters/client_channel/lb_policy/priority/priority.cc', 'src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc', + 'src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc', 'src/core/ext/filters/client_channel/lb_policy/xds/cds.cc', - 'src/core/ext/filters/client_channel/lb_policy/xds/xds.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/eds.cc', + 'src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc', 'src/core/ext/filters/client_channel/lb_policy_registry.cc', 'src/core/ext/filters/client_channel/local_subchannel_pool.cc', 'src/core/ext/filters/client_channel/parse_address.cc', diff --git a/test/core/client_channel/service_config_test.cc b/test/core/client_channel/service_config_test.cc index d659fbeb51d..5cdb51341ab 100644 --- a/test/core/client_channel/service_config_test.cc +++ b/test/core/client_channel/service_config_test.cc @@ -464,7 +464,7 @@ TEST_F(ClientChannelParserTest, ValidLoadBalancingConfigXds) { "{\n" " \"loadBalancingConfig\":[\n" " { \"does_not_exist\":{} },\n" - " { \"xds_experimental\":{ \"balancerName\": \"fake:///lb\" } }\n" + " { \"eds_experimental\":{ \"clusterName\": \"foo\" } }\n" " ]\n" "}"; grpc_error* error = GRPC_ERROR_NONE; @@ -474,7 +474,7 @@ TEST_F(ClientChannelParserTest, ValidLoadBalancingConfigXds) { static_cast( svc_cfg->GetGlobalParsedConfig(0)); auto lb_config = parsed_config->parsed_lb_config(); - EXPECT_STREQ(lb_config->name(), "xds_experimental"); + EXPECT_STREQ(lb_config->name(), "eds_experimental"); } TEST_F(ClientChannelParserTest, UnknownLoadBalancingConfig) { @@ -544,14 +544,14 @@ TEST_F(ClientChannelParserTest, UnknownLoadBalancingPolicy) { } TEST_F(ClientChannelParserTest, LoadBalancingPolicyXdsNotAllowed) { - const char* test_json = "{\"loadBalancingPolicy\":\"xds_experimental\"}"; + const char* test_json = "{\"loadBalancingPolicy\":\"eds_experimental\"}"; grpc_error* error = GRPC_ERROR_NONE; auto svc_cfg = ServiceConfig::Create(test_json, &error); std::regex regex( "Service config parsing error.*referenced_errors.*" "Global Params.*referenced_errors.*" "Client channel global parser.*referenced_errors.*" - "field:loadBalancingPolicy error:xds_experimental requires " + "field:loadBalancingPolicy error:eds_experimental requires " "a config. Please use loadBalancingConfig instead."); VerifyRegexMatch(error, regex); } diff --git a/test/cpp/end2end/xds_end2end_test.cc b/test/cpp/end2end/xds_end2end_test.cc index e84692d2812..2efbc38a364 100644 --- a/test/cpp/end2end/xds_end2end_test.cc +++ b/test/cpp/end2end/xds_end2end_test.cc @@ -1153,7 +1153,7 @@ class XdsEnd2endTest : public ::testing::TestWithParam { args.SetInt(GRPC_ARG_XDS_FALLBACK_TIMEOUT_MS, fallback_timeout); } if (failover_timeout > 0) { - args.SetInt(GRPC_ARG_XDS_FAILOVER_TIMEOUT_MS, failover_timeout); + args.SetInt(GRPC_ARG_PRIORITY_FAILOVER_TIMEOUT_MS, failover_timeout); } if (xds_resource_does_not_exist_timeout > 0) { args.SetInt(GRPC_ARG_XDS_RESOURCE_DOES_NOT_EXIST_TIMEOUT_MS, @@ -1285,7 +1285,8 @@ class XdsEnd2endTest : public ::testing::TestWithParam { : kDefaultServiceConfigWithoutLoadReporting_; result.service_config = grpc_core::ServiceConfig::Create(service_config_json, &error); - GRPC_ERROR_UNREF(error); + ASSERT_NE(result.service_config.get(), nullptr); + ASSERT_EQ(error, GRPC_ERROR_NONE) << grpc_error_string(error); grpc_arg arg = grpc_core::FakeResolverResponseGenerator::MakeChannelArg( lb_channel_response_generator == nullptr ? lb_channel_response_generator_.get() @@ -1317,7 +1318,8 @@ class XdsEnd2endTest : public ::testing::TestWithParam { grpc_error* error = GRPC_ERROR_NONE; result.service_config = grpc_core::ServiceConfig::Create(service_config_json, &error); - GRPC_ERROR_UNREF(error); + ASSERT_NE(result.service_config.get(), nullptr); + ASSERT_EQ(error, GRPC_ERROR_NONE) << grpc_error_string(error); } if (lb_channel_response_generator == nullptr) { lb_channel_response_generator = lb_channel_response_generator_.get(); diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index edbe236dc17..60e99d7b5a0 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -1091,6 +1091,8 @@ src/core/ext/filters/client_channel/http_proxy.cc \ src/core/ext/filters/client_channel/http_proxy.h \ src/core/ext/filters/client_channel/lb_policy.cc \ src/core/ext/filters/client_channel/lb_policy.h \ +src/core/ext/filters/client_channel/lb_policy/address_filtering.cc \ +src/core/ext/filters/client_channel/lb_policy/address_filtering.h \ src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc \ src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h \ src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc \ @@ -1106,10 +1108,13 @@ src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h \ src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h \ src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc \ +src/core/ext/filters/client_channel/lb_policy/priority/priority.cc \ src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc \ src/core/ext/filters/client_channel/lb_policy/subchannel_list.h \ +src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc \ src/core/ext/filters/client_channel/lb_policy/xds/cds.cc \ -src/core/ext/filters/client_channel/lb_policy/xds/xds.cc \ +src/core/ext/filters/client_channel/lb_policy/xds/eds.cc \ +src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc \ src/core/ext/filters/client_channel/lb_policy/xds/xds.h \ src/core/ext/filters/client_channel/lb_policy_factory.h \ src/core/ext/filters/client_channel/lb_policy_registry.cc \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 5c03ac61231..fef1fd11d4a 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -888,6 +888,8 @@ src/core/ext/filters/client_channel/http_proxy.cc \ src/core/ext/filters/client_channel/http_proxy.h \ src/core/ext/filters/client_channel/lb_policy.cc \ src/core/ext/filters/client_channel/lb_policy.h \ +src/core/ext/filters/client_channel/lb_policy/address_filtering.cc \ +src/core/ext/filters/client_channel/lb_policy/address_filtering.h \ src/core/ext/filters/client_channel/lb_policy/child_policy_handler.cc \ src/core/ext/filters/client_channel/lb_policy/child_policy_handler.h \ src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc \ @@ -903,10 +905,13 @@ src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.h \ src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc \ src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.h \ src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc \ +src/core/ext/filters/client_channel/lb_policy/priority/priority.cc \ src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc \ src/core/ext/filters/client_channel/lb_policy/subchannel_list.h \ +src/core/ext/filters/client_channel/lb_policy/weighted_target/weighted_target.cc \ src/core/ext/filters/client_channel/lb_policy/xds/cds.cc \ -src/core/ext/filters/client_channel/lb_policy/xds/xds.cc \ +src/core/ext/filters/client_channel/lb_policy/xds/eds.cc \ +src/core/ext/filters/client_channel/lb_policy/xds/lrs.cc \ src/core/ext/filters/client_channel/lb_policy/xds/xds.h \ src/core/ext/filters/client_channel/lb_policy_factory.h \ src/core/ext/filters/client_channel/lb_policy_registry.cc \ diff --git a/tools/internal_ci/linux/grpc_xds_bazel_python_test_in_docker.sh b/tools/internal_ci/linux/grpc_xds_bazel_python_test_in_docker.sh index 6ed8856bf0d..a9a74eef377 100755 --- a/tools/internal_ci/linux/grpc_xds_bazel_python_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_xds_bazel_python_test_in_docker.sh @@ -48,7 +48,7 @@ touch "$TOOLS_DIR"/src/proto/grpc/testing/__init__.py bazel build //src/python/grpcio_tests/tests_py3_only/interop:xds_interop_client -GRPC_VERBOSITY=debug GRPC_TRACE=xds_client,xds_resolver,cds_lb,xds_lb "$PYTHON" \ +GRPC_VERBOSITY=debug GRPC_TRACE=xds_client,xds_resolver,cds_lb,eds_lb,priority_lb,weighted_target_lb,lrs_lb "$PYTHON" \ tools/run_tests/run_xds_tests.py \ --test_case=all \ --project_id=grpc-testing \ diff --git a/tools/internal_ci/linux/grpc_xds_bazel_test_in_docker.sh b/tools/internal_ci/linux/grpc_xds_bazel_test_in_docker.sh index d0a1338817b..943ba2b1981 100755 --- a/tools/internal_ci/linux/grpc_xds_bazel_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_xds_bazel_test_in_docker.sh @@ -48,7 +48,7 @@ touch "$TOOLS_DIR"/src/proto/grpc/testing/__init__.py bazel build test/cpp/interop:xds_interop_client -GRPC_VERBOSITY=debug GRPC_TRACE=xds_client,xds_resolver,cds_lb,xds_lb "$PYTHON" \ +GRPC_VERBOSITY=debug GRPC_TRACE=xds_client,xds_resolver,cds_lb,eds_lb,priority_lb,weighted_target_lb,lrs_lb "$PYTHON" \ tools/run_tests/run_xds_tests.py \ --test_case=all \ --project_id=grpc-testing \ From 1c1321ddea3fe5f2357872654d48ea4d1f1af54e Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Tue, 7 Apr 2020 16:10:20 -0700 Subject: [PATCH 32/73] Indentation --- tools/run_tests/helper_scripts/build_python.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index cfcb151bcce..e79a8896092 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -224,8 +224,8 @@ pip_install_dir "$ROOT/src/python/grpcio_testing" # Build/install tests pip_install coverage==4.4 oauth2client==4.1.0 \ - google-auth==1.0.0 requests==2.14.2 \ - googleapis-common-protos==1.5.5 + google-auth==1.0.0 requests==2.14.2 \ + googleapis-common-protos==1.5.5 $VENV_PYTHON "$ROOT/src/python/grpcio_tests/setup.py" preprocess $VENV_PYTHON "$ROOT/src/python/grpcio_tests/setup.py" build_package_protos pip_install_dir "$ROOT/src/python/grpcio_tests" From bc831fec51a1e669b48627f9ce1276a6af93a7ca Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Wed, 1 Apr 2020 16:54:43 -0700 Subject: [PATCH 33/73] Use grpcHealthCheck and targetGrpcProxy --- tools/run_tests/run_xds_tests.py | 118 +++++++++++++++++-------------- 1 file changed, 66 insertions(+), 52 deletions(-) diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index 3e165da4f1d..016eed9a115 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -122,6 +122,13 @@ argp.add_argument( help= 'If provided, uses this file instead of retrieving via the GCP discovery ' 'API') +argp.add_argument( + '--alpha_compute_discovery_document', + default=None, + type=str, + help= + 'If provided, uses this file instead of retrieving via the alpha GCP ' + 'discovery API') argp.add_argument('--network', default='global/networks/default', help='GCP network to use') @@ -577,14 +584,14 @@ def add_instance_group(gcp, zone, name, size): def create_health_check(gcp, name): config = { 'name': name, - 'type': 'TCP', - 'tcpHealthCheck': { - 'portName': 'grpc' + 'type': 'GRPC', + 'grpcHealthCheck': { + 'portSpecification': 'USE_SERVING_PORT' } } logger.debug('Sending GCP request with body=%s', config) - result = gcp.compute.healthChecks().insert(project=gcp.project, - body=config).execute() + result = gcp.alpha_compute.healthChecks().insert(project=gcp.project, + body=config).execute() wait_for_global_operation(gcp, result['name']) gcp.health_check = GcpResource(config['name'], result['targetLink']) @@ -613,11 +620,11 @@ def add_backend_service(gcp, name): 'loadBalancingScheme': 'INTERNAL_SELF_MANAGED', 'healthChecks': [gcp.health_check.url], 'portName': 'grpc', - 'protocol': 'HTTP2' + 'protocol': 'GRPC' } logger.debug('Sending GCP request with body=%s', config) - result = gcp.compute.backendServices().insert(project=gcp.project, - body=config).execute() + result = gcp.alpha_compute.backendServices().insert(project=gcp.project, + body=config).execute() wait_for_global_operation(gcp, result['name']) backend_service = GcpResource(config['name'], result['targetLink']) gcp.backend_services.append(backend_service) @@ -644,33 +651,45 @@ def create_url_map(gcp, name, backend_service, host_name): gcp.url_map = GcpResource(config['name'], result['targetLink']) -def create_target_http_proxy(gcp, name): +def create_target_grpc_proxy(gcp, name): config = { 'name': name, 'url_map': gcp.url_map.url, + # TODO(ericgribkoff) This requires forwarding rule's ip=0.0.0.0 + # 'validate_for_proxyless': True, } logger.debug('Sending GCP request with body=%s', config) - result = gcp.compute.targetHttpProxies().insert(project=gcp.project, - body=config).execute() + result = gcp.alpha_compute.targetGrpcProxies().insert( + project=gcp.project, body=config).execute() wait_for_global_operation(gcp, result['name']) - gcp.target_http_proxy = GcpResource(config['name'], result['targetLink']) - - -def create_global_forwarding_rule(gcp, name, ip, port): - config = { - 'name': name, - 'loadBalancingScheme': 'INTERNAL_SELF_MANAGED', - 'portRange': str(port), - 'IPAddress': ip, - 'network': args.network, - 'target': gcp.target_http_proxy.url, - } - logger.debug('Sending GCP request with body=%s', config) - result = gcp.compute.globalForwardingRules().insert(project=gcp.project, - body=config).execute() - wait_for_global_operation(gcp, result['name']) - gcp.global_forwarding_rule = GcpResource(config['name'], - result['targetLink']) + gcp.target_grpc_proxy = GcpResource(config['name'], result['targetLink']) + + +def create_global_forwarding_rule(gcp, name, potential_ips, potential_ports): + for port in potential_ports: + for ip in potential_ips: + try: + config = { + 'name': name, + 'loadBalancingScheme': 'INTERNAL_SELF_MANAGED', + 'portRange': str(port), + 'IPAddress': ip, + 'network': args.network, + 'target': gcp.target_grpc_proxy.url, + } + logger.debug('Sending GCP request with body=%s', config) + result = gcp.alpha_compute.globalForwardingRules().insert( + project=gcp.project, body=config).execute() + wait_for_global_operation(gcp, result['name']) + gcp.global_forwarding_rule = GcpResource( + config['name'], result['targetLink']) + gcp.service_port = port + return + except googleapiclient.errors.HttpError as http_error: + logger.warning( + 'Got error %s when attempting to create forwarding rule to ' + '%s:%d. Retrying with another ip:port.' % + (http_error, ip, port)) def delete_global_forwarding_rule(gcp): @@ -683,11 +702,11 @@ def delete_global_forwarding_rule(gcp): logger.info('Delete failed: %s', http_error) -def delete_target_http_proxy(gcp): +def delete_target_grpc_proxy(gcp): try: - result = gcp.compute.targetHttpProxies().delete( + result = gcp.alpha_compute.targetGrpcProxies().delete( project=gcp.project, - targetHttpProxy=gcp.target_http_proxy.name).execute() + targetGrpcProxy=gcp.target_grpc_proxy.name).execute() wait_for_global_operation(gcp, result['name']) except googleapiclient.errors.HttpError as http_error: logger.info('Delete failed: %s', http_error) @@ -769,7 +788,7 @@ def patch_backend_instances(gcp, } for instance_group in instance_groups], } logger.debug('Sending GCP request with body=%s', config) - result = gcp.compute.backendServices().patch( + result = gcp.alpha_compute.backendServices().patch( project=gcp.project, backendService=backend_service.name, body=config).execute() wait_for_global_operation(gcp, result['name']) @@ -910,8 +929,8 @@ def start_xds_client(cmd): def clean_up(gcp): if gcp.global_forwarding_rule: delete_global_forwarding_rule(gcp) - if gcp.target_http_proxy: - delete_target_http_proxy(gcp) + if gcp.target_grpc_proxy: + delete_target_grpc_proxy(gcp) if gcp.url_map: delete_url_map(gcp) delete_backend_services(gcp) @@ -941,14 +960,15 @@ class GcpResource(object): class GcpState(object): - def __init__(self, compute, project): + def __init__(self, compute, alpha_compute, project): self.compute = compute + self.alpha_compute = alpha_compute self.project = project self.health_check = None self.health_check_firewall_rule = None self.backend_services = [] self.url_map = None - self.target_http_proxy = None + self.target_grpc_proxy = None self.global_forwarding_rule = None self.service_port = None self.instance_template = None @@ -959,18 +979,22 @@ if args.compute_discovery_document: with open(args.compute_discovery_document, 'r') as discovery_doc: compute = googleapiclient.discovery.build_from_document( discovery_doc.read()) + with open(args.alpha_compute_discovery_document, 'r') as discovery_doc: + alpha_compute = googleapiclient.discovery.build_from_document( + discovery_doc.read()) else: compute = googleapiclient.discovery.build('compute', 'v1') + alpha_compute = googleapiclient.discovery.build('compute', 'alpha') try: - gcp = GcpState(compute, args.project_id) + gcp = GcpState(compute, alpha_compute, args.project_id) health_check_name = _BASE_HEALTH_CHECK_NAME + args.gcp_suffix firewall_name = _BASE_FIREWALL_RULE_NAME + args.gcp_suffix backend_service_name = _BASE_BACKEND_SERVICE_NAME + args.gcp_suffix alternate_backend_service_name = _BASE_BACKEND_SERVICE_NAME + '-alternate' + args.gcp_suffix url_map_name = _BASE_URL_MAP_NAME + args.gcp_suffix service_host_name = _BASE_SERVICE_HOST + args.gcp_suffix - target_http_proxy_name = _BASE_TARGET_PROXY_NAME + args.gcp_suffix + target_grpc_proxy_name = _BASE_TARGET_PROXY_NAME + args.gcp_suffix forwarding_rule_name = _BASE_FORWARDING_RULE_NAME + args.gcp_suffix template_name = _BASE_TEMPLATE_NAME + args.gcp_suffix instance_group_name = _BASE_INSTANCE_GROUP_NAME + args.gcp_suffix @@ -984,7 +1008,7 @@ try: alternate_backend_service = add_backend_service( gcp, alternate_backend_service_name) create_url_map(gcp, url_map_name, backend_service, service_host_name) - create_target_http_proxy(gcp, target_http_proxy_name) + create_target_grpc_proxy(gcp, target_grpc_proxy_name) potential_service_ports = list(args.service_port_range) random.shuffle(potential_service_ports) if args.forwarding_rule_ip_prefix == '': @@ -994,18 +1018,8 @@ try: args.forwarding_rule_ip_prefix + str(x) for x in range(256) ] random.shuffle(potential_ips) - for port in potential_service_ports: - for ip in potential_ips: - try: - create_global_forwarding_rule(gcp, forwarding_rule_name, ip, - port) - gcp.service_port = port - break - except googleapiclient.errors.HttpError as http_error: - logger.warning( - 'Got error %s when attempting to create forwarding rule to ' - '%s:%d. Retrying with another ip:port.' % - (http_error, ip, port)) + create_global_forwarding_rule(gcp, forwarding_rule_name, potential_ips, + potential_service_ports) if not gcp.service_port: raise Exception( 'Failed to find a valid ip:port for the forwarding rule') From d9c6b001c00fb715258328702c855a0fc362ce51 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Thu, 2 Apr 2020 15:30:44 -0700 Subject: [PATCH 34/73] Use validate-for-proxyless --- tools/run_tests/run_xds_tests.py | 98 ++++++++++++++------------------ 1 file changed, 42 insertions(+), 56 deletions(-) diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index 016eed9a115..3dc66ab7424 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -126,23 +126,18 @@ argp.add_argument( '--alpha_compute_discovery_document', default=None, type=str, - help= - 'If provided, uses this file instead of retrieving via the alpha GCP ' + help='If provided, uses this file instead of retrieving via the alpha GCP ' 'discovery API') argp.add_argument('--network', default='global/networks/default', help='GCP network to use') argp.add_argument('--service_port_range', - default='80', + default='8080:8110', type=parse_port_range, help='Listening port for created gRPC backends. Specified as ' 'either a single int or as a range in the format min:max, in ' 'which case an available port p will be chosen s.t. min <= p ' '<= max') -argp.add_argument('--forwarding_rule_ip_prefix', - default='172.16.0.', - help='If set, an available IP with this prefix followed by ' - '0-255 will be used for the generated forwarding rule.') argp.add_argument( '--stats_port', default=8079, @@ -651,12 +646,25 @@ def create_url_map(gcp, name, backend_service, host_name): gcp.url_map = GcpResource(config['name'], result['targetLink']) +def patch_url_map_host_rule(gcp, name, backend_service, host_name): + config = { + 'hostRules': [{ + 'hosts': ['%s:%d' % (host_name, gcp.service_port)], + 'pathMatcher': _PATH_MATCHER_NAME + }] + } + logger.debug('Sending GCP request with body=%s', config) + result = gcp.compute.urlMaps().patch(project=gcp.project, + urlMap=name, + body=config).execute() + wait_for_global_operation(gcp, result['name']) + + def create_target_grpc_proxy(gcp, name): config = { 'name': name, 'url_map': gcp.url_map.url, - # TODO(ericgribkoff) This requires forwarding rule's ip=0.0.0.0 - # 'validate_for_proxyless': True, + 'validate_for_proxyless': True, } logger.debug('Sending GCP request with body=%s', config) result = gcp.alpha_compute.targetGrpcProxies().insert( @@ -665,31 +673,29 @@ def create_target_grpc_proxy(gcp, name): gcp.target_grpc_proxy = GcpResource(config['name'], result['targetLink']) -def create_global_forwarding_rule(gcp, name, potential_ips, potential_ports): +def create_global_forwarding_rule(gcp, name, potential_ports): for port in potential_ports: - for ip in potential_ips: - try: - config = { - 'name': name, - 'loadBalancingScheme': 'INTERNAL_SELF_MANAGED', - 'portRange': str(port), - 'IPAddress': ip, - 'network': args.network, - 'target': gcp.target_grpc_proxy.url, - } - logger.debug('Sending GCP request with body=%s', config) - result = gcp.alpha_compute.globalForwardingRules().insert( - project=gcp.project, body=config).execute() - wait_for_global_operation(gcp, result['name']) - gcp.global_forwarding_rule = GcpResource( - config['name'], result['targetLink']) - gcp.service_port = port - return - except googleapiclient.errors.HttpError as http_error: - logger.warning( - 'Got error %s when attempting to create forwarding rule to ' - '%s:%d. Retrying with another ip:port.' % - (http_error, ip, port)) + try: + config = { + 'name': name, + 'loadBalancingScheme': 'INTERNAL_SELF_MANAGED', + 'portRange': str(port), + 'IPAddress': '0.0.0.0', + 'network': args.network, + 'target': gcp.target_grpc_proxy.url, + } + logger.debug('Sending GCP request with body=%s', config) + result = gcp.alpha_compute.globalForwardingRules().insert( + project=gcp.project, body=config).execute() + wait_for_global_operation(gcp, result['name']) + gcp.global_forwarding_rule = GcpResource(config['name'], + result['targetLink']) + gcp.service_port = port + return + except googleapiclient.errors.HttpError as http_error: + logger.warning( + 'Got error %s when attempting to create forwarding rule to ' + '0.0.0.0:%d. Retrying with another port.' % (http_error, port)) def delete_global_forwarding_rule(gcp): @@ -911,21 +917,6 @@ def get_instance_names(gcp, instance_group): return instance_names -def start_xds_client(cmd): - bootstrap_path = None - with tempfile.NamedTemporaryFile(delete=False) as bootstrap_file: - bootstrap_file.write( - _BOOTSTRAP_TEMPLATE.format( - node_id=socket.gethostname()).encode('utf-8')) - bootstrap_path = bootstrap_file.name - - client_process = subprocess.Popen(shlex.split(cmd), - env=dict( - os.environ, - GRPC_XDS_BOOTSTRAP=bootstrap_path)) - return client_process - - def clean_up(gcp): if gcp.global_forwarding_rule: delete_global_forwarding_rule(gcp) @@ -1011,18 +1002,13 @@ try: create_target_grpc_proxy(gcp, target_grpc_proxy_name) potential_service_ports = list(args.service_port_range) random.shuffle(potential_service_ports) - if args.forwarding_rule_ip_prefix == '': - potential_ips = ['0.0.0.0'] - else: - potential_ips = [ - args.forwarding_rule_ip_prefix + str(x) for x in range(256) - ] - random.shuffle(potential_ips) - create_global_forwarding_rule(gcp, forwarding_rule_name, potential_ips, + create_global_forwarding_rule(gcp, forwarding_rule_name, potential_service_ports) if not gcp.service_port: raise Exception( 'Failed to find a valid ip:port for the forwarding rule') + patch_url_map_host_rule(gcp, url_map_name, backend_service, + service_host_name) startup_script = get_startup_script(args.path_to_server_binary, gcp.service_port) create_instance_template(gcp, template_name, args.network, From 652ab6c49797639714b20d5c9326ec9ce7d15f27 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Tue, 7 Apr 2020 19:41:33 -0700 Subject: [PATCH 35/73] Clamp the max recv flow control window to 2^31 - 1 --- src/core/ext/transport/chttp2/transport/flow_control.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/flow_control.cc b/src/core/ext/transport/chttp2/transport/flow_control.cc index d53475a1b61..babe564d39d 100644 --- a/src/core/ext/transport/chttp2/transport/flow_control.cc +++ b/src/core/ext/transport/chttp2/transport/flow_control.cc @@ -284,8 +284,8 @@ void StreamFlowControl::IncomingByteStreamUpdate(size_t max_size_hint, [GRPC_CHTTP2_SETTINGS_INITIAL_WINDOW_SIZE]; /* clamp max recv hint to an allowable size */ - if (max_size_hint >= UINT32_MAX - sent_init_window) { - max_recv_bytes = UINT32_MAX - sent_init_window; + if (max_size_hint >= kMaxWindowUpdateSize - sent_init_window) { + max_recv_bytes = kMaxWindowUpdateSize - sent_init_window; } else { max_recv_bytes = static_cast(max_size_hint); } @@ -298,7 +298,7 @@ void StreamFlowControl::IncomingByteStreamUpdate(size_t max_size_hint, } /* add some small lookahead to keep pipelines flowing */ - GPR_ASSERT(max_recv_bytes <= UINT32_MAX - sent_init_window); + GPR_DEBUG_ASSERT(max_recv_bytes <= kMaxWindowUpdateSize - sent_init_window); if (local_window_delta_ < max_recv_bytes) { uint32_t add_max_recv_bytes = static_cast(max_recv_bytes - local_window_delta_); From fdbdcb93d2a2e1f28ba98e9990efafd6f9e2b380 Mon Sep 17 00:00:00 2001 From: Ashitha Santhosh Date: Tue, 7 Apr 2020 23:44:40 -0700 Subject: [PATCH 36/73] Remove ABSL_CONST_INIT. --- .../tsi/alts/handshaker/alts_handshaker_client.cc | 8 ++++---- src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc | 13 +++++-------- src/core/tsi/alts/handshaker/alts_tsi_handshaker.h | 6 +++--- .../tsi/alts/handshaker/alts_tsi_handshaker_test.cc | 4 ++-- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/src/core/tsi/alts/handshaker/alts_handshaker_client.cc b/src/core/tsi/alts/handshaker/alts_handshaker_client.cc index 5c5a407f54f..61927276195 100644 --- a/src/core/tsi/alts/handshaker/alts_handshaker_client.cc +++ b/src/core/tsi/alts/handshaker/alts_handshaker_client.cc @@ -508,8 +508,8 @@ static grpc_byte_buffer* get_serialized_start_client( upb_strview_makez(ptr->data)); ptr = ptr->next; } - grpc_gcp_StartClientHandshakeReq_set_max_frame_size(start_client, - client->max_frame_size); + grpc_gcp_StartClientHandshakeReq_set_max_frame_size( + start_client, static_cast(client->max_frame_size)); return get_serialized_handshaker_req(req, arena.ptr()); } @@ -569,8 +569,8 @@ static grpc_byte_buffer* get_serialized_start_server( arena.ptr()); grpc_gcp_RpcProtocolVersions_assign_from_struct( server_version, arena.ptr(), &client->options->rpc_versions); - grpc_gcp_StartServerHandshakeReq_set_max_frame_size(start_server, - client->max_frame_size); + grpc_gcp_StartServerHandshakeReq_set_max_frame_size( + start_server, static_cast(client->max_frame_size)); return get_serialized_handshaker_req(req, arena.ptr()); } diff --git a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc index b2687825d17..2a925182d3f 100644 --- a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc +++ b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc @@ -41,9 +41,6 @@ #include "src/core/tsi/alts/handshaker/alts_tsi_utils.h" #include "src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.h" -const size_t kMinFrameSize = 16 * 1024; -const size_t kMaxFrameSize = 128 * 1024; - /* Main struct for ALTS TSI handshaker. */ struct alts_tsi_handshaker { tsi_handshaker base; @@ -166,17 +163,17 @@ static tsi_result handshaker_result_create_zero_copy_grpc_protector( // In case the peer does not send max frame size (e.g. peer is gRPC Go or // peer uses an old binary), the negotiated frame size is set to - // kMinFrameSize (ignoring max_output_protected_frame_size value if + // kTsiAltsMinFrameSize (ignoring max_output_protected_frame_size value if // present). Otherwise, it is based on peer and user specified max frame // size (if present). - size_t max_frame_size = kMinFrameSize; + size_t max_frame_size = kTsiAltsMinFrameSize; if (result->max_frame_size) { size_t peer_max_frame_size = result->max_frame_size; max_frame_size = std::min(peer_max_frame_size, max_output_protected_frame_size == nullptr - ? kMaxFrameSize + ? kTsiAltsMaxFrameSize : *max_output_protected_frame_size); - max_frame_size = std::max(max_frame_size, kMinFrameSize); + max_frame_size = std::max(max_frame_size, kTsiAltsMinFrameSize); } max_output_protected_frame_size = &max_frame_size; gpr_log(GPR_DEBUG, @@ -622,7 +619,7 @@ tsi_result alts_tsi_handshaker_create( handshaker->options = grpc_alts_credentials_options_copy(options); handshaker->max_frame_size = user_specified_max_frame_size != 0 ? user_specified_max_frame_size - : kMaxFrameSize; + : kTsiAltsMaxFrameSize; handshaker->base.vtable = handshaker->use_dedicated_cq ? &handshaker_vtable_dedicated : &handshaker_vtable; diff --git a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h index 6959c563a5e..e1ae985a84d 100644 --- a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h +++ b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.h @@ -39,9 +39,9 @@ const size_t kTsiAltsNumOfPeerProperties = 5; // Frame size negotiation extends send frame size range to -// [kMinFrameSize, kMaxFrameSize] -ABSL_CONST_INIT extern const size_t kMinFrameSize; -ABSL_CONST_INIT extern const size_t kMaxFrameSize; +// [kTsiAltsMinFrameSize, kTsiAltsMaxFrameSize]. +const size_t kTsiAltsMinFrameSize = 16 * 1024; +const size_t kTsiAltsMaxFrameSize = 128 * 1024; typedef struct alts_tsi_handshaker alts_tsi_handshaker; diff --git a/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc b/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc index 8f38640ef67..2127e980488 100644 --- a/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc +++ b/test/core/tsi/alts/handshaker/alts_tsi_handshaker_test.cc @@ -296,7 +296,7 @@ static void on_client_next_success_cb(tsi_result status, void* user_data, size_t actual_max_frame_size; tsi_zero_copy_grpc_protector_max_frame_size(zero_copy_protector, &actual_max_frame_size); - GPR_ASSERT(actual_max_frame_size == kMaxFrameSize); + GPR_ASSERT(actual_max_frame_size == kTsiAltsMaxFrameSize); tsi_zero_copy_grpc_protector_destroy(zero_copy_protector); /* Validate peer identity. */ tsi_peer peer; @@ -370,7 +370,7 @@ static void on_server_next_success_cb(tsi_result status, void* user_data, size_t actual_max_frame_size; tsi_zero_copy_grpc_protector_max_frame_size(zero_copy_protector, &actual_max_frame_size); - GPR_ASSERT(actual_max_frame_size == kMinFrameSize); + GPR_ASSERT(actual_max_frame_size == kTsiAltsMinFrameSize); tsi_zero_copy_grpc_protector_destroy(zero_copy_protector); /* Validate peer identity. */ tsi_peer peer; From c3fc65c9e00bb630cadfc4e420501d66633f9257 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Wed, 8 Apr 2020 07:29:45 -0700 Subject: [PATCH 37/73] Don't NACK EDS updates with no localities, but report TRANSIENT_FAILURE. --- .../client_channel/lb_policy/xds/eds.cc | 19 ++++++++------ .../ext/filters/client_channel/xds/xds_api.cc | 7 ------ test/cpp/end2end/xds_end2end_test.cc | 25 +++++++++---------- 3 files changed, 23 insertions(+), 28 deletions(-) diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/eds.cc b/src/core/ext/filters/client_channel/lb_policy/xds/eds.cc index 2f1acdb9f80..099fb286223 100644 --- a/src/core/ext/filters/client_channel/lb_policy/xds/eds.cc +++ b/src/core/ext/filters/client_channel/lb_policy/xds/eds.cc @@ -405,19 +405,22 @@ class EdsLb::EndpointWatcher : public XdsClient::EndpointWatcherInterface { } eds_policy_->drop_config_ = std::move(update.drop_config); eds_policy_->MaybeUpdateDropPickerLocked(); + } else if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] Drop config unchanged, ignoring", + eds_policy_.get()); } // Update priority and locality info. - if (eds_policy_->priority_list_update_ == update.priority_list_update) { + if (eds_policy_->child_policy_ == nullptr || + eds_policy_->priority_list_update_ != update.priority_list_update) { if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { - gpr_log(GPR_INFO, - "[edslb %p] Incoming locality update identical to current, " - "ignoring. (drop_config_changed=%d)", - eds_policy_.get(), drop_config_changed); + gpr_log(GPR_INFO, "[edslb %p] Updating priority list", + eds_policy_.get()); } - return; + eds_policy_->UpdatePriorityList(std::move(update.priority_list_update)); + } else if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { + gpr_log(GPR_INFO, "[edslb %p] Priority list unchanged, ignoring", + eds_policy_.get()); } - // Update the child policy with the new priority and endpoint data. - eds_policy_->UpdatePriorityList(std::move(update.priority_list_update)); } void OnError(grpc_error* error) override { diff --git a/src/core/ext/filters/client_channel/xds/xds_api.cc b/src/core/ext/filters/client_channel/xds/xds_api.cc index 2ac824ba08e..4cf5d8c26a2 100644 --- a/src/core/ext/filters/client_channel/xds/xds_api.cc +++ b/src/core/ext/filters/client_channel/xds/xds_api.cc @@ -1420,13 +1420,6 @@ grpc_error* EdsResponseParse( if (error != GRPC_ERROR_NONE) return error; } } - // Validate the update content. - if (eds_update.priority_list_update.empty() && - !eds_update.drop_config->drop_all()) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "EDS response doesn't contain any valid " - "locality but doesn't require to drop all calls."); - } eds_update_map->emplace(std::string(cluster_name.data, cluster_name.size), std::move(eds_update)); } diff --git a/test/cpp/end2end/xds_end2end_test.cc b/test/cpp/end2end/xds_end2end_test.cc index 2efbc38a364..2e643a43670 100644 --- a/test/cpp/end2end/xds_end2end_test.cc +++ b/test/cpp/end2end/xds_end2end_test.cc @@ -2355,19 +2355,6 @@ TEST_P(EdsTest, Timeout) { CheckRpcSendFailure(); } -// Tests that EDS client should send a NACK if the EDS update contains -// no localities but does not say to drop all calls. -TEST_P(EdsTest, NacksNoLocalitiesWithoutDropAll) { - SetNextResolution({}); - SetNextResolutionForLbChannelAllBalancers(); - AdsServiceImpl::EdsResourceArgs args; - balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); - CheckRpcSendFailure(); - EXPECT_EQ(balancers_[0]->ads_service()->eds_response_state(), - AdsServiceImpl::NACKED); -} - // Tests that EDS client should send a NACK if the EDS update contains // sparse priorities. TEST_P(EdsTest, NacksSparsePriorityList) { @@ -2454,6 +2441,18 @@ TEST_P(LocalityMapTest, LocalityContainingNoEndpoints) { kNumRpcs / backends_.size()); } +// EDS update with no localities. +TEST_P(LocalityMapTest, NoLocalities) { + SetNextResolution({}); + SetNextResolutionForLbChannelAllBalancers(); + // EDS response contains 2 localities, one with no endpoints. + balancers_[0]->ads_service()->SetEdsResource( + AdsServiceImpl::BuildEdsResource({}), kDefaultResourceName); + Status status = SendRpc(); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.error_code(), GRPC_STATUS_UNAVAILABLE); +} + // Tests that the locality map can work properly even when it contains a large // number of localities. TEST_P(LocalityMapTest, StressTest) { From f1979cece2324549fd24c3d58de707331ae40e3b Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Wed, 8 Apr 2020 08:41:54 -0700 Subject: [PATCH 38/73] Disable SockPair fixture --- test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc b/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc index d4533e6c78e..d0df2417139 100644 --- a/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc +++ b/test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc @@ -34,8 +34,6 @@ BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, UDS) ->Range(0, 128 * 1024 * 1024); BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, InProcess) ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, SockPair) - ->Range(0, 128 * 1024 * 1024); BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, InProcessCHTTP2) ->Range(0, 128 * 1024 * 1024); BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, TCP) @@ -44,19 +42,15 @@ BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, UDS) ->Range(0, 128 * 1024 * 1024); BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcess) ->Range(0, 128 * 1024 * 1024); -BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, SockPair) - ->Range(0, 128 * 1024 * 1024); BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, InProcessCHTTP2) ->Range(0, 128 * 1024 * 1024); BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, MinTCP)->Arg(0); BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, MinUDS)->Arg(0); BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, MinInProcess)->Arg(0); -BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, MinSockPair)->Arg(0); BENCHMARK_TEMPLATE(BM_PumpStreamClientToServer, MinInProcessCHTTP2)->Arg(0); BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, MinTCP)->Arg(0); BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, MinUDS)->Arg(0); BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, MinInProcess)->Arg(0); -BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, MinSockPair)->Arg(0); BENCHMARK_TEMPLATE(BM_PumpStreamServerToClient, MinInProcessCHTTP2)->Arg(0); } // namespace testing From bb3b5d1d238bb562425f4ba417d5fc9f41799f70 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 8 Apr 2020 11:52:03 -0400 Subject: [PATCH 39/73] switch grpc-dart interop test to dart2.7 --- .../interoptest/grpc_interop_dart/Dockerfile.template | 5 +---- tools/dockerfile/interoptest/grpc_interop_dart/Dockerfile | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_dart/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_dart/Dockerfile.template index dee59335280..7836341a913 100644 --- a/templates/tools/dockerfile/interoptest/grpc_interop_dart/Dockerfile.template +++ b/templates/tools/dockerfile/interoptest/grpc_interop_dart/Dockerfile.template @@ -14,10 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - FROM google/dart:2.3 - - # Upgrade Dart to version 2. - RUN apt-get update && apt-get upgrade -y dart + FROM google/dart:2.7 # Define the default command. CMD ["bash"] diff --git a/tools/dockerfile/interoptest/grpc_interop_dart/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_dart/Dockerfile index 43d8a60daea..dd9b8e55b0c 100644 --- a/tools/dockerfile/interoptest/grpc_interop_dart/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_dart/Dockerfile @@ -12,10 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM google/dart:2.3 - -# Upgrade Dart to version 2. -RUN apt-get update && apt-get upgrade -y dart +FROM google/dart:2.7 # Define the default command. CMD ["bash"] From 527509abc40b74e2e95ed19ec99cd06175fb3fd8 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Wed, 8 Apr 2020 10:09:00 -0700 Subject: [PATCH 40/73] Unmark flaky on streaming_pump --- test/cpp/microbenchmarks/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/test/cpp/microbenchmarks/BUILD b/test/cpp/microbenchmarks/BUILD index 7e8d371fc3c..e8065fa8964 100644 --- a/test/cpp/microbenchmarks/BUILD +++ b/test/cpp/microbenchmarks/BUILD @@ -206,7 +206,6 @@ grpc_cc_test( srcs = [ "bm_fullstack_streaming_pump.cc", ], - flaky = True, # TODO(b/150422385) tags = [ "no_mac", # to emulate "excluded_poll_engines: poll" "no_windows", From 719ec593d05b228828402ea708e93ed9e7760c1b Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Wed, 8 Apr 2020 10:31:33 -0700 Subject: [PATCH 41/73] Regen projects --- Makefile | 2 ++ build_autogenerated.yaml | 1 - tools/run_tests/generated/tests.json | 20 ++++++++++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d1a58c87a8b..4aab6a0adb9 100644 --- a/Makefile +++ b/Makefile @@ -2176,6 +2176,8 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/bm_error || ( echo test bm_error failed ; exit 1 ) $(E) "[RUN] Testing bm_fullstack_streaming_ping_pong" $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_ping_pong || ( echo test bm_fullstack_streaming_ping_pong failed ; exit 1 ) + $(E) "[RUN] Testing bm_fullstack_streaming_pump" + $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump || ( echo test bm_fullstack_streaming_pump failed ; exit 1 ) $(E) "[RUN] Testing bm_fullstack_unary_ping_pong" $(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong || ( echo test bm_fullstack_unary_ping_pong failed ; exit 1 ) $(E) "[RUN] Testing bm_metadata" diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 69e0b124799..3e4cab440ed 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -5127,7 +5127,6 @@ targets: - posix - name: bm_fullstack_streaming_pump build: test - run: false language: c++ headers: - test/cpp/microbenchmarks/fullstack_streaming_pump.h diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index fc5695d84a3..c1707be58da 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -3757,6 +3757,26 @@ ], "uses_polling": true }, + { + "args": [], + "benchmark": true, + "ci_platforms": [ + "linux", + "posix" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c++", + "name": "bm_fullstack_streaming_pump", + "platforms": [ + "linux", + "posix" + ], + "uses_polling": true + }, { "args": [], "benchmark": true, From e4dcc0319c022babd472a461da76c0827f1dc038 Mon Sep 17 00:00:00 2001 From: ZhenLian Date: Wed, 8 Apr 2020 10:50:49 -0700 Subject: [PATCH 42/73] [2/n] Avoid using hardcoded test credentials --- test/core/end2end/fixtures/h2_oauth2.cc | 36 ++++++++++++++---- test/core/end2end/fixtures/h2_ssl.cc | 49 ++++++++++++------------- test/core/end2end/generate_tests.bzl | 14 +++++++ 3 files changed, 65 insertions(+), 34 deletions(-) diff --git a/test/core/end2end/fixtures/h2_oauth2.cc b/test/core/end2end/fixtures/h2_oauth2.cc index 0800d5929f2..22f85a2feea 100644 --- a/test/core/end2end/fixtures/h2_oauth2.cc +++ b/test/core/end2end/fixtures/h2_oauth2.cc @@ -16,22 +16,26 @@ * */ -#include "test/core/end2end/end2end_tests.h" - -#include -#include - #include #include +#include +#include #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/gprpp/host_port.h" #include "src/core/lib/iomgr/iomgr.h" +#include "src/core/lib/iomgr/load_file.h" #include "src/core/lib/security/credentials/credentials.h" -#include "test/core/end2end/data/ssl_test_data.h" +#include "test/core/end2end/end2end_tests.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" +#define CA_CERT_PATH "src/core/tsi/test_creds/ca.pem" +#define CLIENT_CERT_PATH "src/core/tsi/test_creds/client.pem" +#define CLIENT_KEY_PATH "src/core/tsi/test_creds/client.key" +#define SERVER_CERT_PATH "src/core/tsi/test_creds/server1.pem" +#define SERVER_KEY_PATH "src/core/tsi/test_creds/server1.key" + static const char oauth2_md[] = "Bearer aaslkfjs424535asdf"; static const char* client_identity_property_name = "smurf_name"; static const char* client_identity = "Brainy Smurf"; @@ -139,6 +143,11 @@ void chttp2_tear_down_secure_fullstack(grpc_end2end_test_fixture* f) { static void chttp2_init_client_simple_ssl_with_oauth2_secure_fullstack( grpc_end2end_test_fixture* f, grpc_channel_args* client_args) { grpc_core::ExecCtx exec_ctx; + grpc_slice ca_slice; + GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", + grpc_load_file(CA_CERT_PATH, 1, &ca_slice))); + const char* test_root_cert = + reinterpret_cast GRPC_SLICE_START_PTR(ca_slice); grpc_channel_credentials* ssl_creds = grpc_ssl_credentials_create(test_root_cert, nullptr, nullptr, nullptr); grpc_call_credentials* oauth2_creds = grpc_md_only_test_credentials_create( @@ -156,6 +165,7 @@ static void chttp2_init_client_simple_ssl_with_oauth2_secure_fullstack( grpc_channel_args_destroy(new_client_args); grpc_channel_credentials_release(ssl_creds); grpc_call_credentials_release(oauth2_creds); + grpc_slice_unref(ca_slice); } static int fail_server_auth_check(grpc_channel_args* server_args) { @@ -193,13 +203,23 @@ static grpc_auth_metadata_processor test_processor_create(int failing) { static void chttp2_init_server_simple_ssl_secure_fullstack( grpc_end2end_test_fixture* f, grpc_channel_args* server_args) { - grpc_ssl_pem_key_cert_pair pem_key_cert_pair = {test_server1_key, - test_server1_cert}; + grpc_slice cert_slice, key_slice; + GPR_ASSERT(GRPC_LOG_IF_ERROR( + "load_file", grpc_load_file(SERVER_CERT_PATH, 1, &cert_slice))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", + grpc_load_file(SERVER_KEY_PATH, 1, &key_slice))); + const char* server_cert = + reinterpret_cast GRPC_SLICE_START_PTR(cert_slice); + const char* server_key = + reinterpret_cast GRPC_SLICE_START_PTR(key_slice); + grpc_ssl_pem_key_cert_pair pem_key_cert_pair = {server_key, server_cert}; grpc_server_credentials* ssl_creds = grpc_ssl_server_credentials_create( nullptr, &pem_key_cert_pair, 1, 0, nullptr); grpc_server_credentials_set_auth_metadata_processor( ssl_creds, test_processor_create(fail_server_auth_check(server_args))); chttp2_init_server_secure_fullstack(f, server_args, ssl_creds); + grpc_slice_unref(cert_slice); + grpc_slice_unref(key_slice); } /* All test configurations */ diff --git a/test/core/end2end/fixtures/h2_ssl.cc b/test/core/end2end/fixtures/h2_ssl.cc index 2ba2418151d..1e46486dd5e 100644 --- a/test/core/end2end/fixtures/h2_ssl.cc +++ b/test/core/end2end/fixtures/h2_ssl.cc @@ -16,23 +16,26 @@ * */ -#include "test/core/end2end/end2end_tests.h" - -#include -#include - #include #include +#include +#include #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/gpr/string.h" #include "src/core/lib/gpr/tmpfile.h" #include "src/core/lib/gprpp/host_port.h" +#include "src/core/lib/iomgr/load_file.h" #include "src/core/lib/security/credentials/credentials.h" #include "src/core/lib/security/security_connector/ssl_utils_config.h" -#include "test/core/end2end/data/ssl_test_data.h" +#include "test/core/end2end/end2end_tests.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" +#define CA_CERT_PATH "src/core/tsi/test_creds/ca.pem" +#define CLIENT_CERT_PATH "src/core/tsi/test_creds/client.pem" +#define CLIENT_KEY_PATH "src/core/tsi/test_creds/client.key" +#define SERVER_CERT_PATH "src/core/tsi/test_creds/server1.pem" +#define SERVER_KEY_PATH "src/core/tsi/test_creds/server1.key" struct fullstack_secure_fixture_data { grpc_core::UniquePtr localaddr; @@ -124,10 +127,20 @@ static int fail_server_auth_check(grpc_channel_args* server_args) { static void chttp2_init_server_simple_ssl_secure_fullstack( grpc_end2end_test_fixture* f, grpc_channel_args* server_args) { - grpc_ssl_pem_key_cert_pair pem_cert_key_pair = {test_server1_key, - test_server1_cert}; + grpc_slice cert_slice, key_slice; + GPR_ASSERT(GRPC_LOG_IF_ERROR( + "load_file", grpc_load_file(SERVER_CERT_PATH, 1, &cert_slice))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", + grpc_load_file(SERVER_KEY_PATH, 1, &key_slice))); + const char* server_cert = + reinterpret_cast GRPC_SLICE_START_PTR(cert_slice); + const char* server_key = + reinterpret_cast GRPC_SLICE_START_PTR(key_slice); + grpc_ssl_pem_key_cert_pair pem_key_cert_pair = {server_key, server_cert}; grpc_server_credentials* ssl_creds = grpc_ssl_server_credentials_create( - nullptr, &pem_cert_key_pair, 1, 0, nullptr); + nullptr, &pem_key_cert_pair, 1, 0, nullptr); + grpc_slice_unref(cert_slice); + grpc_slice_unref(key_slice); if (fail_server_auth_check(server_args)) { grpc_auth_metadata_processor processor = {process_auth_failure, nullptr, nullptr}; @@ -152,20 +165,9 @@ static grpc_end2end_test_config configs[] = { int main(int argc, char** argv) { size_t i; - FILE* roots_file; - size_t roots_size = strlen(test_root_cert); - char* roots_filename; - grpc::testing::TestEnvironment env(argc, argv); grpc_end2end_tests_pre_init(); - - /* Set the SSL roots env var. */ - roots_file = gpr_tmpfile("chttp2_simple_ssl_fullstack_test", &roots_filename); - GPR_ASSERT(roots_filename != nullptr); - GPR_ASSERT(roots_file != nullptr); - GPR_ASSERT(fwrite(test_root_cert, 1, roots_size, roots_file) == roots_size); - fclose(roots_file); - GPR_GLOBAL_CONFIG_SET(grpc_default_ssl_roots_file_path, roots_filename); + GPR_GLOBAL_CONFIG_SET(grpc_default_ssl_roots_file_path, CA_CERT_PATH); grpc_init(); @@ -174,10 +176,5 @@ int main(int argc, char** argv) { } grpc_shutdown(); - - /* Cleanup. */ - remove(roots_filename); - gpr_free(roots_filename); - return 0; } diff --git a/test/core/end2end/generate_tests.bzl b/test/core/end2end/generate_tests.bzl index 2b73af6f715..e7034222e52 100755 --- a/test/core/end2end/generate_tests.bzl +++ b/test/core/end2end/generate_tests.bzl @@ -429,6 +429,13 @@ def grpc_end2end_tests(): name = "%s_test" % f, srcs = ["fixtures/%s.cc" % f], language = "C++", + data = [ + "//src/core/tsi/test_creds:ca.pem", + "//src/core/tsi/test_creds:client.key", + "//src/core/tsi/test_creds:client.pem", + "//src/core/tsi/test_creds:server1.key", + "//src/core/tsi/test_creds:server1.pem", + ], deps = [ ":end2end_tests", "//test/core/util:grpc_test_util", @@ -499,6 +506,13 @@ def grpc_end2end_nosec_tests(): name = "%s_nosec_test" % f, srcs = ["fixtures/%s.cc" % f], language = "C++", + data = [ + "//src/core/tsi/test_creds:ca.pem", + "//src/core/tsi/test_creds:client.key", + "//src/core/tsi/test_creds:client.pem", + "//src/core/tsi/test_creds:server1.key", + "//src/core/tsi/test_creds:server1.pem", + ], deps = [ ":end2end_nosec_tests", "//test/core/util:grpc_test_util_unsecure", From 06c6fa73b08e7cdb3010891d28e160cfda4306e3 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 9 Apr 2020 08:55:07 -0400 Subject: [PATCH 43/73] avoid too long name for retry_exceeds_buffer_size_in_initial_batch --- test/core/end2end/generate_tests.bzl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/core/end2end/generate_tests.bzl b/test/core/end2end/generate_tests.bzl index e7034222e52..3c1817ff82b 100755 --- a/test/core/end2end/generate_tests.bzl +++ b/test/core/end2end/generate_tests.bzl @@ -282,6 +282,9 @@ END2END_TESTS = { "retry_exceeds_buffer_size_in_initial_batch": _test_options( needs_client_channel = True, proxyable = False, + # TODO(jtattermusch): too long bazel test name makes the test flaky on Windows RBE + # See b/151617965 + short_name = "retry_exceeds_buffer_size_in_init", ), "retry_exceeds_buffer_size_in_subsequent_batch": _test_options( needs_client_channel = True, From 697f411fef03c1ef296f0e4efa686e3621213f05 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 9 Apr 2020 10:05:07 -0400 Subject: [PATCH 44/73] mark client_channel_stress_test as flaky --- Makefile | 2 -- build_autogenerated.yaml | 1 + test/cpp/client/BUILD | 1 + tools/run_tests/generated/tests.json | 22 ---------------------- 4 files changed, 2 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index d1a58c87a8b..3c912074365 100644 --- a/Makefile +++ b/Makefile @@ -2204,8 +2204,6 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/cli_call_test || ( echo test cli_call_test failed ; exit 1 ) $(E) "[RUN] Testing client_callback_end2end_test" $(Q) $(BINDIR)/$(CONFIG)/client_callback_end2end_test || ( echo test client_callback_end2end_test failed ; exit 1 ) - $(E) "[RUN] Testing client_channel_stress_test" - $(Q) $(BINDIR)/$(CONFIG)/client_channel_stress_test || ( echo test client_channel_stress_test failed ; exit 1 ) $(E) "[RUN] Testing client_interceptors_end2end_test" $(Q) $(BINDIR)/$(CONFIG)/client_interceptors_end2end_test || ( echo test client_interceptors_end2end_test failed ; exit 1 ) $(E) "[RUN] Testing codegen_test_full" diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 69e0b124799..3da4665b127 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -5511,6 +5511,7 @@ targets: - name: client_channel_stress_test gtest: true build: test + run: false language: c++ headers: - test/cpp/end2end/test_service_impl.h diff --git a/test/cpp/client/BUILD b/test/cpp/client/BUILD index 73b0aa37457..18e9ce4973f 100644 --- a/test/cpp/client/BUILD +++ b/test/cpp/client/BUILD @@ -35,6 +35,7 @@ grpc_cc_test( grpc_cc_test( name = "client_channel_stress_test", srcs = ["client_channel_stress_test.cc"], + flaky = True, # TODO(b/153136407) # TODO(jtattermusch): test fails frequently on Win RBE, but passes locally # reenable the tests once it works reliably on Win RBE. # TODO(roth): Test disabled on msan and tsan due to variable diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index fc5695d84a3..7bc5d2575a9 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -4077,28 +4077,6 @@ ], "uses_polling": true }, - { - "args": [], - "benchmark": false, - "ci_platforms": [ - "linux", - "mac", - "posix" - ], - "cpu_cost": 1.0, - "exclude_configs": [], - "exclude_iomgrs": [], - "flaky": false, - "gtest": true, - "language": "c++", - "name": "client_channel_stress_test", - "platforms": [ - "linux", - "mac", - "posix" - ], - "uses_polling": true - }, { "args": [], "benchmark": false, From ebe94949cafbad27c841a5ecd20a544d8629eced Mon Sep 17 00:00:00 2001 From: Patrice Chalin Date: Thu, 9 Apr 2020 16:22:37 -0400 Subject: [PATCH 45/73] Removing obsolete C++ tutorial content Contributes to https://github.com/grpc/grpc.io/issues/180 --- examples/cpp/cpptutorial.md | 489 +----------------------------------- 1 file changed, 1 insertion(+), 488 deletions(-) diff --git a/examples/cpp/cpptutorial.md b/examples/cpp/cpptutorial.md index c40676de134..21411f9db98 100644 --- a/examples/cpp/cpptutorial.md +++ b/examples/cpp/cpptutorial.md @@ -1,488 +1 @@ -# gRPC Basics: C++ - -This tutorial provides a basic C++ programmer's introduction to working with -gRPC. By walking through this example you'll learn how to: - -- Define a service in a `.proto` file. -- Generate server and client code using the protocol buffer compiler. -- Use the C++ gRPC API to write a simple client and server for your service. - -It assumes that you are familiar with -[protocol buffers](https://developers.google.com/protocol-buffers/docs/overview). -Note that the example in this tutorial uses the proto3 version of the protocol -buffers language, which is currently in alpha release: you can find out more in -the [proto3 language guide](https://developers.google.com/protocol-buffers/docs/proto3) -and see the [release notes](https://github.com/google/protobuf/releases) for the -new version in the protocol buffers Github repository. - -## Why use gRPC? - -Our example is a simple route mapping application that lets clients get -information about features on their route, create a summary of their route, and -exchange route information such as traffic updates with the server and other -clients. - -With gRPC we can define our service once in a `.proto` file and implement clients -and servers in any of gRPC's supported languages, which in turn can be run in -environments ranging from servers inside Google to your own tablet - all the -complexity of communication between different languages and environments is -handled for you by gRPC. We also get all the advantages of working with protocol -buffers, including efficient serialization, a simple IDL, and easy interface -updating. - -## Example code and setup - -The example code for our tutorial is in [examples/cpp/route_guide](route_guide). -You also should have the relevant tools installed to generate the server and -client interface code - if you don't already, follow the setup instructions in -[BUILDING.md](../../BUILDING.md). - -## Defining the service - -Our first step is to define the gRPC *service* and the method *request* and -*response* types using -[protocol buffers](https://developers.google.com/protocol-buffers/docs/overview). -You can see the complete `.proto` file in -[`examples/protos/route_guide.proto`](../protos/route_guide.proto). - -To define a service, you specify a named `service` in your `.proto` file: - -```protobuf -service RouteGuide { - ... -} -``` - -Then you define `rpc` methods inside your service definition, specifying their -request and response types. gRPC lets you define four kinds of service method, -all of which are used in the `RouteGuide` service: - -- A *simple RPC* where the client sends a request to the server using the stub - and waits for a response to come back, just like a normal function call. - -```protobuf - // Obtains the feature at a given position. - rpc GetFeature(Point) returns (Feature) {} -``` - -- A *server-side streaming RPC* where the client sends a request to the server - and gets a stream to read a sequence of messages back. The client reads from - the returned stream until there are no more messages. As you can see in our - example, you specify a server-side streaming method by placing the `stream` - keyword before the *response* type. - -```protobuf - // Obtains the Features available within the given Rectangle. Results are - // streamed rather than returned at once (e.g. in a response message with a - // repeated field), as the rectangle may cover a large area and contain a - // huge number of features. - rpc ListFeatures(Rectangle) returns (stream Feature) {} -``` - -- A *client-side streaming RPC* where the client writes a sequence of messages - and sends them to the server, again using a provided stream. Once the client - has finished writing the messages, it waits for the server to read them all - and return its response. You specify a client-side streaming method by placing - the `stream` keyword before the *request* type. - -```protobuf - // Accepts a stream of Points on a route being traversed, returning a - // RouteSummary when traversal is completed. - rpc RecordRoute(stream Point) returns (RouteSummary) {} -``` - -- A *bidirectional streaming RPC* where both sides send a sequence of messages - using a read-write stream. The two streams operate independently, so clients - and servers can read and write in whatever order they like: for example, the - server could wait to receive all the client messages before writing its - responses, or it could alternately read a message then write a message, or - some other combination of reads and writes. The order of messages in each - stream is preserved. You specify this type of method by placing the `stream` - keyword before both the request and the response. - -```protobuf - // Accepts a stream of RouteNotes sent while a route is being traversed, - // while receiving other RouteNotes (e.g. from other users). - rpc RouteChat(stream RouteNote) returns (stream RouteNote) {} -``` - -Our `.proto` file also contains protocol buffer message type definitions for all -the request and response types used in our service methods - for example, here's -the `Point` message type: - -```protobuf -// Points are represented as latitude-longitude pairs in the E7 representation -// (degrees multiplied by 10**7 and rounded to the nearest integer). -// Latitudes should be in the range +/- 90 degrees and longitude should be in -// the range +/- 180 degrees (inclusive). -message Point { - int32 latitude = 1; - int32 longitude = 2; -} -``` - -## Generating client and server code - -Next we need to generate the gRPC client and server interfaces from our `.proto` -service definition. We do this using the protocol buffer compiler `protoc` with -a special gRPC C++ plugin. - -For simplicity, we've provided a [Makefile](route_guide/Makefile) that runs -`protoc` for you with the appropriate plugin, input, and output (if you want to -run this yourself, make sure you've installed protoc and followed the gRPC code -[installation instructions](../../BUILDING.md) first): - -```shell -$ make route_guide.grpc.pb.cc route_guide.pb.cc -``` - -which actually runs: - -```shell -$ protoc -I ../../protos --grpc_out=. --plugin=protoc-gen-grpc=`which grpc_cpp_plugin` ../../protos/route_guide.proto -$ protoc -I ../../protos --cpp_out=. ../../protos/route_guide.proto -``` - -Running this command generates the following files in your current directory: -- `route_guide.pb.h`, the header which declares your generated message classes -- `route_guide.pb.cc`, which contains the implementation of your message classes -- `route_guide.grpc.pb.h`, the header which declares your generated service - classes -- `route_guide.grpc.pb.cc`, which contains the implementation of your service - classes - -These contain: -- All the protocol buffer code to populate, serialize, and retrieve our request - and response message types -- A class called `RouteGuide` that contains - - a remote interface type (or *stub*) for clients to call with the methods - defined in the `RouteGuide` service. - - two abstract interfaces for servers to implement, also with the methods - defined in the `RouteGuide` service. - - - -## Creating the server - -First let's look at how we create a `RouteGuide` server. If you're only -interested in creating gRPC clients, you can skip this section and go straight -to [Creating the client](#client) (though you might find it interesting -anyway!). - -There are two parts to making our `RouteGuide` service do its job: -- Implementing the service interface generated from our service definition: - doing the actual "work" of our service. -- Running a gRPC server to listen for requests from clients and return the - service responses. - -You can find our example `RouteGuide` server in -[route_guide/route_guide_server.cc](route_guide/route_guide_server.cc). Let's -take a closer look at how it works. - -### Implementing RouteGuide - -As you can see, our server has a `RouteGuideImpl` class that implements the -generated `RouteGuide::Service` interface: - -```cpp -class RouteGuideImpl final : public RouteGuide::Service { -... -} -``` -In this case we're implementing the *synchronous* version of `RouteGuide`, which -provides our default gRPC server behaviour. It's also possible to implement an -asynchronous interface, `RouteGuide::AsyncService`, which allows you to further -customize your server's threading behaviour, though we won't look at this in -this tutorial. - -`RouteGuideImpl` implements all our service methods. Let's look at the simplest -type first, `GetFeature`, which just gets a `Point` from the client and returns -the corresponding feature information from its database in a `Feature`. - -```cpp - Status GetFeature(ServerContext* context, const Point* point, - Feature* feature) override { - feature->set_name(GetFeatureName(*point, feature_list_)); - feature->mutable_location()->CopyFrom(*point); - return Status::OK; - } -``` - -The method is passed a context object for the RPC, the client's `Point` protocol -buffer request, and a `Feature` protocol buffer to fill in with the response -information. In the method we populate the `Feature` with the appropriate -information, and then `return` with an `OK` status to tell gRPC that we've -finished dealing with the RPC and that the `Feature` can be returned to the -client. - -Now let's look at something a bit more complicated - a streaming RPC. -`ListFeatures` is a server-side streaming RPC, so we need to send back multiple -`Feature`s to our client. - -```cpp -Status ListFeatures(ServerContext* context, const Rectangle* rectangle, - ServerWriter* writer) override { - auto lo = rectangle->lo(); - auto hi = rectangle->hi(); - long left = std::min(lo.longitude(), hi.longitude()); - long right = std::max(lo.longitude(), hi.longitude()); - long top = std::max(lo.latitude(), hi.latitude()); - long bottom = std::min(lo.latitude(), hi.latitude()); - for (const Feature& f : feature_list_) { - if (f.location().longitude() >= left && - f.location().longitude() <= right && - f.location().latitude() >= bottom && - f.location().latitude() <= top) { - writer->Write(f); - } - } - return Status::OK; -} -``` - -As you can see, instead of getting simple request and response objects in our -method parameters, this time we get a request object (the `Rectangle` in which -our client wants to find `Feature`s) and a special `ServerWriter` object. In the -method, we populate as many `Feature` objects as we need to return, writing them -to the `ServerWriter` using its `Write()` method. Finally, as in our simple RPC, -we `return Status::OK` to tell gRPC that we've finished writing responses. - -If you look at the client-side streaming method `RecordRoute` you'll see it's -quite similar, except this time we get a `ServerReader` instead of a request -object and a single response. We use the `ServerReader`s `Read()` method to -repeatedly read in our client's requests to a request object (in this case a -`Point`) until there are no more messages: the server needs to check the return -value of `Read()` after each call. If `true`, the stream is still good and it -can continue reading; if `false` the message stream has ended. - -```cpp -while (stream->Read(&point)) { - ...//process client input -} -``` -Finally, let's look at our bidirectional streaming RPC `RouteChat()`. - -```cpp - Status RouteChat(ServerContext* context, - ServerReaderWriter* stream) override { - std::vector received_notes; - RouteNote note; - while (stream->Read(¬e)) { - for (const RouteNote& n : received_notes) { - if (n.location().latitude() == note.location().latitude() && - n.location().longitude() == note.location().longitude()) { - stream->Write(n); - } - } - received_notes.push_back(note); - } - - return Status::OK; - } -``` - -This time we get a `ServerReaderWriter` that can be used to read *and* write -messages. The syntax for reading and writing here is exactly the same as for our -client-streaming and server-streaming methods. Although each side will always -get the other's messages in the order they were written, both the client and -server can read and write in any order — the streams operate completely -independently. - -### Starting the server - -Once we've implemented all our methods, we also need to start up a gRPC server -so that clients can actually use our service. The following snippet shows how we -do this for our `RouteGuide` service: - -```cpp -void RunServer(const std::string& db_path) { - std::string server_address("0.0.0.0:50051"); - RouteGuideImpl service(db_path); - - ServerBuilder builder; - builder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); - builder.RegisterService(&service); - std::unique_ptr server(builder.BuildAndStart()); - std::cout << "Server listening on " << server_address << std::endl; - server->Wait(); -} -``` -As you can see, we build and start our server using a `ServerBuilder`. To do this, we: - -1. Create an instance of our service implementation class `RouteGuideImpl`. -1. Create an instance of the factory `ServerBuilder` class. -1. Specify the address and port we want to use to listen for client requests - using the builder's `AddListeningPort()` method. -1. Register our service implementation with the builder. -1. Call `BuildAndStart()` on the builder to create and start an RPC server for - our service. -1. Call `Wait()` on the server to do a blocking wait until process is killed or - `Shutdown()` is called. - - -## Creating the client - -In this section, we'll look at creating a C++ client for our `RouteGuide` -service. You can see our complete example client code in -[route_guide/route_guide_client.cc](route_guide/route_guide_client.cc). - -### Creating a stub - -To call service methods, we first need to create a *stub*. - -First we need to create a gRPC *channel* for our stub, specifying the server -address and port we want to connect to without SSL: - -```cpp -grpc::CreateChannel("localhost:50051", grpc::InsecureChannelCredentials()); -``` - -Now we can use the channel to create our stub using the `NewStub` method -provided in the `RouteGuide` class we generated from our `.proto`. - -```cpp -public: - RouteGuideClient(std::shared_ptr channel, const std::string& db) - : stub_(RouteGuide::NewStub(channel)) { - ... - } -``` - -### Calling service methods - -Now let's look at how we call our service methods. Note that in this tutorial -we're calling the *blocking/synchronous* versions of each method: this means -that the RPC call waits for the server to respond, and will either return a -response or raise an exception. - -#### Simple RPC - -Calling the simple RPC `GetFeature` is nearly as straightforward as calling a -local method. - -```cpp - Point point; - Feature feature; - point = MakePoint(409146138, -746188906); - GetOneFeature(point, &feature); - -... - - bool GetOneFeature(const Point& point, Feature* feature) { - ClientContext context; - Status status = stub_->GetFeature(&context, point, feature); - ... - } -``` - -As you can see, we create and populate a request protocol buffer object (in our -case `Point`), and create a response protocol buffer object for the server to -fill in. We also create a `ClientContext` object for our call - you can -optionally set RPC configuration values on this object, such as deadlines, -though for now we'll use the default settings. Note that you cannot reuse this -object between calls. Finally, we call the method on the stub, passing it the -context, request, and response. If the method returns `OK`, then we can read the -response information from the server from our response object. - -```cpp -std::cout << "Found feature called " << feature->name() << " at " - << feature->location().latitude()/kCoordFactor_ << ", " - << feature->location().longitude()/kCoordFactor_ << std::endl; -``` - -#### Streaming RPCs - -Now let's look at our streaming methods. If you've already read [Creating the -server](#server) some of this may look very familiar - streaming RPCs are -implemented in a similar way on both sides. Here's where we call the server-side -streaming method `ListFeatures`, which returns a stream of geographical -`Feature`s: - -```cpp -std::unique_ptr > reader( - stub_->ListFeatures(&context, rect)); -while (reader->Read(&feature)) { - std::cout << "Found feature called " - << feature.name() << " at " - << feature.location().latitude()/kCoordFactor_ << ", " - << feature.location().longitude()/kCoordFactor_ << std::endl; -} -Status status = reader->Finish(); -``` - -Instead of passing the method a context, request, and response, we pass it a -context and request and get a `ClientReader` object back. The client can use the -`ClientReader` to read the server's responses. We use the `ClientReader`s -`Read()` method to repeatedly read in the server's responses to a response -protocol buffer object (in this case a `Feature`) until there are no more -messages: the client needs to check the return value of `Read()` after each -call. If `true`, the stream is still good and it can continue reading; if -`false` the message stream has ended. Finally, we call `Finish()` on the stream -to complete the call and get our RPC status. - -The client-side streaming method `RecordRoute` is similar, except there we pass -the method a context and response object and get back a `ClientWriter`. - -```cpp - std::unique_ptr > writer( - stub_->RecordRoute(&context, &stats)); - for (int i = 0; i < kPoints; i++) { - const Feature& f = feature_list_[feature_distribution(generator)]; - std::cout << "Visiting point " - << f.location().latitude()/kCoordFactor_ << ", " - << f.location().longitude()/kCoordFactor_ << std::endl; - if (!writer->Write(f.location())) { - // Broken stream. - break; - } - std::this_thread::sleep_for(std::chrono::milliseconds( - delay_distribution(generator))); - } - writer->WritesDone(); - Status status = writer->Finish(); - if (status.IsOk()) { - std::cout << "Finished trip with " << stats.point_count() << " points\n" - << "Passed " << stats.feature_count() << " features\n" - << "Travelled " << stats.distance() << " meters\n" - << "It took " << stats.elapsed_time() << " seconds" - << std::endl; - } else { - std::cout << "RecordRoute rpc failed." << std::endl; - } -``` - -Once we've finished writing our client's requests to the stream using `Write()`, -we need to call `WritesDone()` on the stream to let gRPC know that we've -finished writing, then `Finish()` to complete the call and get our RPC status. -If the status is `OK`, our response object that we initially passed to -`RecordRoute()` will be populated with the server's response. - -Finally, let's look at our bidirectional streaming RPC `RouteChat()`. In this -case, we just pass a context to the method and get back a `ClientReaderWriter`, -which we can use to both write and read messages. - -```cpp -std::shared_ptr > stream( - stub_->RouteChat(&context)); -``` - -The syntax for reading and writing here is exactly the same as for our -client-streaming and server-streaming methods. Although each side will always -get the other's messages in the order they were written, both the client and -server can read and write in any order — the streams operate completely -independently. - -## Try it out! - -Build client and server: -```shell -$ make -``` -Run the server, which will listen on port 50051: -```shell -$ ./route_guide_server -``` -Run the client (in a different terminal): -```shell -$ ./route_guide_client -``` +The content of this page has moved to [gRPC Basics - C++](https://grpc.io/docs/tutorials/basic/cpp). From df5b521e079e250fab89fb95a12a372baedb0200 Mon Sep 17 00:00:00 2001 From: Patrice Chalin Date: Thu, 9 Apr 2020 16:56:10 -0400 Subject: [PATCH 46/73] Ignore `cmake/build/` anywhere in the repo Ignore `cmake/build/` anywhere in the repo, not just at the root level. This is helpful when working on the examples. --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index bd9a5139fe2..383fc25b360 100644 --- a/.gitignore +++ b/.gitignore @@ -136,7 +136,7 @@ bm_diff_old/ bm_*.json # cmake build files -/cmake/build +**/cmake/build/ # Visual Studio Code artifacts .vscode/* From 33082023e936d03f89ed2226d6617208449e412c Mon Sep 17 00:00:00 2001 From: Patrice Chalin Date: Thu, 9 Apr 2020 17:00:33 -0400 Subject: [PATCH 47/73] C++ examples README cleanup Contributes to https://github.com/grpc/grpc.io/issues/180. cc @ejona86 @jtattermusch @srini100 --- examples/cpp/README.md | 51 +++++++++--------------------------------- 1 file changed, 10 insertions(+), 41 deletions(-) diff --git a/examples/cpp/README.md b/examples/cpp/README.md index 0e358bf9a22..a0ff629b1a0 100644 --- a/examples/cpp/README.md +++ b/examples/cpp/README.md @@ -1,44 +1,13 @@ -# gRPC in 3 minutes (C++) +# gRPC C++ Examples -## Installation +- **[Hello World][]!** Eager to run your first gRPC example? You'll find + instructions for building gRPC and running a simple "Hello World" app in [Quick Start][]. +- **[Route Guide][].** For a basic tutorial on gRPC see [gRPC Basics][]. -To install gRPC on your system, follow the instructions to build from source -[here](../../BUILDING.md). This also installs the protocol buffer compiler -`protoc` (if you don't have it already), and the C++ gRPC plugin for `protoc`. +For information about the other examples in this directory, see their respective +README files. -## Hello C++ gRPC! - -Here's how to build and run the C++ implementation of the [Hello -World](../protos/helloworld.proto) example used in [Getting started](..). - -### Client and server implementations - -The client implementation is at [greeter_client.cc](helloworld/greeter_client.cc). - -The server implementation is at [greeter_server.cc](helloworld/greeter_server.cc). - -### Try it! -Build client and server: - -```sh -$ make -``` - -Run the server, which will listen on port 50051: - -```sh -$ ./greeter_server -``` - -Run the client (in a different terminal): - -```sh -$ ./greeter_client -``` - -If things go smoothly, you will see the "Greeter received: Hello world" in the -client side output. - -## Tutorial - -You can find a more detailed tutorial in [gRPC Basics: C++](cpptutorial.md) +[gRPC Basics]: https://grpc.io/docs/tutorials/basic/cpp +[Hello World]: helloworld +[Quick Start]: https://grpc.io/docs/quickstart/cpp +[Route Guide]: route_guide From a3714f5b08b5b9a164d89ba93505b8d0c084167c Mon Sep 17 00:00:00 2001 From: Patrice Chalin Date: Thu, 9 Apr 2020 17:20:48 -0400 Subject: [PATCH 48/73] C++ Hello World: refer reader to Quick Start --- examples/cpp/helloworld/README.md | 266 +----------------------------- 1 file changed, 4 insertions(+), 262 deletions(-) diff --git a/examples/cpp/helloworld/README.md b/examples/cpp/helloworld/README.md index 71718645645..e13c83281a7 100644 --- a/examples/cpp/helloworld/README.md +++ b/examples/cpp/helloworld/README.md @@ -1,264 +1,6 @@ -# gRPC C++ Hello World Tutorial +# gRPC C++ Hello World Example -### Install gRPC -Make sure you have installed gRPC on your system. Follow the -[BUILDING.md](../../../BUILDING.md) instructions. +You can find a complete set of instructions for building gRPC and running the +Hello World app in the [C++ Quick Start][]. -### Get the tutorial source code - -The example code for this and our other examples lives in the `examples` -directory. Clone this repository at the [latest stable release tag](https://github.com/grpc/grpc/releases) -to your local machine by running the following command: - - -```sh -$ git clone -b RELEASE_TAG_HERE https://github.com/grpc/grpc -``` - -Change your current directory to examples/cpp/helloworld - -```sh -$ cd examples/cpp/helloworld/ -``` - -### Defining a service - -The first step in creating our example is to define a *service*: an RPC -service specifies the methods that can be called remotely with their parameters -and return types. As you saw in the -[overview](#protocolbuffers) above, gRPC does this using [protocol -buffers](https://developers.google.com/protocol-buffers/docs/overview). We -use the protocol buffers interface definition language (IDL) to define our -service methods, and define the parameters and return -types as protocol buffer message types. Both the client and the -server use interface code generated from the service definition. - -Here's our example service definition, defined using protocol buffers IDL in -[helloworld.proto](../../protos/helloworld.proto). The `Greeting` -service has one method, `hello`, that lets the server receive a single -`HelloRequest` -message from the remote client containing the user's name, then send back -a greeting in a single `HelloReply`. This is the simplest type of RPC you -can specify in gRPC - we'll look at some other types later in this document. - -```protobuf -syntax = "proto3"; - -option java_package = "ex.grpc"; - -package helloworld; - -// The greeting service definition. -service Greeter { - // Sends a greeting - rpc SayHello (HelloRequest) returns (HelloReply) {} -} - -// The request message containing the user's name. -message HelloRequest { - string name = 1; -} - -// The response message containing the greetings -message HelloReply { - string message = 1; -} - -``` - - -### Generating gRPC code - -Once we've defined our service, we use the protocol buffer compiler -`protoc` to generate the special client and server code we need to create -our application. The generated code contains both stub code for clients to -use and an abstract interface for servers to implement, both with the method -defined in our `Greeting` service. - -To generate the client and server side interfaces: - -```sh -$ make helloworld.grpc.pb.cc helloworld.pb.cc -``` -Which internally invokes the proto-compiler as: - -```sh -$ protoc -I ../../protos/ --grpc_out=. --plugin=protoc-gen-grpc=grpc_cpp_plugin ../../protos/helloworld.proto -$ protoc -I ../../protos/ --cpp_out=. ../../protos/helloworld.proto -``` - -### Writing a client - -- Create a channel. A channel is a logical connection to an endpoint. A gRPC - channel can be created with the target address, credentials to use and - arguments as follows - - ```cpp - auto channel = CreateChannel("localhost:50051", InsecureChannelCredentials()); - ``` - -- Create a stub. A stub implements the rpc methods of a service and in the - generated code, a method is provided to create a stub with a channel: - - ```cpp - auto stub = helloworld::Greeter::NewStub(channel); - ``` - -- Make a unary rpc, with `ClientContext` and request/response proto messages. - - ```cpp - ClientContext context; - HelloRequest request; - request.set_name("hello"); - HelloReply reply; - Status status = stub->SayHello(&context, request, &reply); - ``` - -- Check returned status and response. - - ```cpp - if (status.ok()) { - // check reply.message() - } else { - // rpc failed. - } - ``` - -For a working example, refer to [greeter_client.cc](greeter_client.cc). - -### Writing a server - -- Implement the service interface - - ```cpp - class GreeterServiceImpl final : public Greeter::Service { - Status SayHello(ServerContext* context, const HelloRequest* request, - HelloReply* reply) override { - std::string prefix("Hello "); - reply->set_message(prefix + request->name()); - return Status::OK; - } - }; - - ``` - -- Build a server exporting the service - - ```cpp - GreeterServiceImpl service; - ServerBuilder builder; - builder.AddListeningPort("0.0.0.0:50051", grpc::InsecureServerCredentials()); - builder.RegisterService(&service); - std::unique_ptr server(builder.BuildAndStart()); - ``` - -For a working example, refer to [greeter_server.cc](greeter_server.cc). - -### Writing asynchronous client and server - -gRPC uses `CompletionQueue` API for asynchronous operations. The basic work flow -is -- bind a `CompletionQueue` to a rpc call -- do something like a read or write, present with a unique `void*` tag -- call `CompletionQueue::Next` to wait for operations to complete. If a tag - appears, it indicates that the corresponding operation is complete. - -#### Async client - -The channel and stub creation code is the same as the sync client. - -- Initiate the rpc and create a handle for the rpc. Bind the rpc to a - `CompletionQueue`. - - ```cpp - CompletionQueue cq; - auto rpc = stub->AsyncSayHello(&context, request, &cq); - ``` - -- Ask for reply and final status, with a unique tag - - ```cpp - Status status; - rpc->Finish(&reply, &status, (void*)1); - ``` - -- Wait for the completion queue to return the next tag. The reply and status are - ready once the tag passed into the corresponding `Finish()` call is returned. - - ```cpp - void* got_tag; - bool ok = false; - cq.Next(&got_tag, &ok); - if (ok && got_tag == (void*)1) { - // check reply and status - } - ``` - -For a working example, refer to [greeter_async_client.cc](greeter_async_client.cc). - -#### Async server - -The server implementation requests a rpc call with a tag and then wait for the -completion queue to return the tag. The basic flow is - -- Build a server exporting the async service - - ```cpp - helloworld::Greeter::AsyncService service; - ServerBuilder builder; - builder.AddListeningPort("0.0.0.0:50051", InsecureServerCredentials()); - builder.RegisterService(&service); - auto cq = builder.AddCompletionQueue(); - auto server = builder.BuildAndStart(); - ``` - -- Request one rpc - - ```cpp - ServerContext context; - HelloRequest request; - ServerAsyncResponseWriter responder; - service.RequestSayHello(&context, &request, &responder, &cq, &cq, (void*)1); - ``` - -- Wait for the completion queue to return the tag. The context, request and - responder are ready once the tag is retrieved. - - ```cpp - HelloReply reply; - Status status; - void* got_tag; - bool ok = false; - cq.Next(&got_tag, &ok); - if (ok && got_tag == (void*)1) { - // set reply and status - responder.Finish(reply, status, (void*)2); - } - ``` - -- Wait for the completion queue to return the tag. The rpc is finished when the - tag is back. - - ```cpp - void* got_tag; - bool ok = false; - cq.Next(&got_tag, &ok); - if (ok && got_tag == (void*)2) { - // clean up - } - ``` - -To handle multiple rpcs, the async server creates an object `CallData` to -maintain the state of each rpc and use the address of it as the unique tag. For -simplicity the server only uses one completion queue for all events, and runs a -main loop in `HandleRpcs` to query the queue. - -For a working example, refer to [greeter_async_server.cc](greeter_async_server.cc). - -#### Flags for the client - -```sh -./greeter_client --target="a target string used to create a GRPC client channel" -``` - -The Default value for --target is "localhost:50051". +[C++ Quick Start]: https://grpc.io/docs/quickstart/cpp From 44bc3b9bd6c030f9ed5983b682ae51e454bf1026 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Thu, 9 Apr 2020 22:45:08 -0700 Subject: [PATCH 49/73] Reviewer comments --- .../message_compress_filter.cc | 186 +++++++++--------- 1 file changed, 94 insertions(+), 92 deletions(-) diff --git a/src/core/ext/filters/http/message_compress/message_compress_filter.cc b/src/core/ext/filters/http/message_compress/message_compress_filter.cc index fcee54ebfe3..b962c99e1d5 100644 --- a/src/core/ext/filters/http/message_compress/message_compress_filter.cc +++ b/src/core/ext/filters/http/message_compress/message_compress_filter.cc @@ -21,6 +21,8 @@ #include #include +#include "absl/types/optional.h" + #include #include #include @@ -44,7 +46,7 @@ namespace { class ChannelData { public: - ChannelData(grpc_channel_element_args* args) { + explicit ChannelData(grpc_channel_element_args* args) { // Get the enabled and the default algorithms from channel args. enabled_compression_algorithms_bitset_ = grpc_channel_args_compression_algorithm_get_states(args->channel_args); @@ -101,8 +103,8 @@ class ChannelData { class CallData { public: CallData(grpc_call_element* elem, const grpc_call_element_args& args) - : elem_(elem), call_combiner_(args.call_combiner) { - ChannelData* channeld = static_cast(elem_->channel_data); + : call_combiner_(args.call_combiner) { + ChannelData* channeld = static_cast(elem->channel_data); // The call's message compression algorithm is set to channel's default // setting. It can be overridden later by initial metadata. if (GPR_LIKELY(GPR_BITGET(channeld->enabled_compression_algorithms_bitset(), @@ -112,7 +114,7 @@ class CallData { channeld->default_compression_algorithm()); } GRPC_CLOSURE_INIT(&start_send_message_batch_in_call_combiner_, - StartSendMessageBatch, this, grpc_schedule_on_exec_ctx); + StartSendMessageBatch, elem, grpc_schedule_on_exec_ctx); } ~CallData() { @@ -122,28 +124,29 @@ class CallData { GRPC_ERROR_UNREF(cancel_error_); } - static void CompressStartTransportStreamOpBatch( + void CompressStartTransportStreamOpBatch( grpc_call_element* elem, grpc_transport_stream_op_batch* batch); + private: bool SkipMessageCompression(); - void InitializeState(); + void InitializeState(grpc_call_element* elem); - grpc_error* ProcessSendInitialMetadata(grpc_metadata_batch* initial_metadata); + grpc_error* ProcessSendInitialMetadata(grpc_call_element* elem, + grpc_metadata_batch* initial_metadata); // Methods for processing a send_message batch - static void StartSendMessageBatch(void* arg, grpc_error* unused); - static void OnSendMessageNextDone(void* arg, grpc_error* error); + static void StartSendMessageBatch(void* elem_arg, grpc_error* unused); + static void OnSendMessageNextDone(void* elem_arg, grpc_error* error); grpc_error* PullSliceFromSendMessage(); - void ContinueReadingSendMessage(); - void FinishSendMessage(); - void SendMessageBatchContinue(); - static void FailSendMessageBatchInCallCombiner(void* arg, grpc_error* error); + void ContinueReadingSendMessage(grpc_call_element* elem); + void FinishSendMessage(grpc_call_element* elem); + void SendMessageBatchContinue(grpc_call_element* elem); + static void FailSendMessageBatchInCallCombiner(void* calld_arg, + grpc_error* error); - static void SendMessageOnComplete(void* arg, grpc_error* error); + static void SendMessageOnComplete(void* calld_arg, grpc_error* error); - private: - grpc_call_element* elem_ = nullptr; - grpc_core::CallCombiner* call_combiner_ = nullptr; + grpc_core::CallCombiner* call_combiner_; grpc_message_compression_algorithm message_compression_algorithm_ = GRPC_MESSAGE_COMPRESS_NONE; grpc_error* cancel_error_ = GRPC_ERROR_NONE; @@ -160,8 +163,7 @@ class CallData { grpc_linked_mdelem accept_encoding_storage_; grpc_linked_mdelem accept_stream_encoding_storage_; grpc_slice_buffer slices_; /**< Buffers up input slices to be compressed */ - grpc_core::ManualConstructor - replacement_stream_; + absl::optional replacement_stream_; grpc_closure* original_send_message_on_complete_ = nullptr; grpc_closure send_message_on_complete_; grpc_closure on_send_message_next_done_; @@ -217,19 +219,19 @@ grpc_compression_algorithm FindCompressionAlgorithm( return GRPC_COMPRESS_NONE; } -void CallData::InitializeState() { +void CallData::InitializeState(grpc_call_element* elem) { GPR_DEBUG_ASSERT(!state_initialized_); state_initialized_ = true; grpc_slice_buffer_init(&slices_); GRPC_CLOSURE_INIT(&send_message_on_complete_, SendMessageOnComplete, this, grpc_schedule_on_exec_ctx); - GRPC_CLOSURE_INIT(&on_send_message_next_done_, OnSendMessageNextDone, this, + GRPC_CLOSURE_INIT(&on_send_message_next_done_, OnSendMessageNextDone, elem, grpc_schedule_on_exec_ctx); } grpc_error* CallData::ProcessSendInitialMetadata( - grpc_metadata_batch* initial_metadata) { - ChannelData* channeld = static_cast(elem_->channel_data); + grpc_call_element* elem, grpc_metadata_batch* initial_metadata) { + ChannelData* channeld = static_cast(elem->channel_data); // Find the compression algorithm. grpc_compression_algorithm compression_algorithm = FindCompressionAlgorithm(initial_metadata, channeld); @@ -243,14 +245,14 @@ grpc_error* CallData::ProcessSendInitialMetadata( // Hint compression algorithm. grpc_error* error = GRPC_ERROR_NONE; if (message_compression_algorithm_ != GRPC_MESSAGE_COMPRESS_NONE) { - InitializeState(); + InitializeState(elem); error = grpc_metadata_batch_add_tail( initial_metadata, &message_compression_algorithm_storage_, grpc_message_compression_encoding_mdelem( message_compression_algorithm_), GRPC_BATCH_GRPC_ENCODING); } else if (stream_compression_algorithm != GRPC_STREAM_COMPRESS_NONE) { - InitializeState(); + InitializeState(elem); error = grpc_metadata_batch_add_tail( initial_metadata, &stream_compression_algorithm_storage_, grpc_stream_compression_encoding_mdelem(stream_compression_algorithm), @@ -276,25 +278,23 @@ grpc_error* CallData::ProcessSendInitialMetadata( return error; } -void CallData::SendMessageOnComplete(void* arg, grpc_error* error) { - CallData* calld = static_cast(arg); +void CallData::SendMessageOnComplete(void* calld_arg, grpc_error* error) { + CallData* calld = static_cast(calld_arg); grpc_slice_buffer_reset_and_unref_internal(&calld->slices_); grpc_core::Closure::Run(DEBUG_LOCATION, calld->original_send_message_on_complete_, GRPC_ERROR_REF(error)); } -void CallData::SendMessageBatchContinue() { +void CallData::SendMessageBatchContinue(grpc_call_element* elem) { // Note: The call to grpc_call_next_op() results in yielding the - // call combiner, so we need to clear calld->send_message_batch_ - // before we do that. - grpc_transport_stream_op_batch* send_message_batch = - this->send_message_batch_; + // call combiner, so we need to clear send_message_batch_ before we do that. + grpc_transport_stream_op_batch* send_message_batch = send_message_batch_; send_message_batch_ = nullptr; - grpc_call_next_op(elem_, send_message_batch); + grpc_call_next_op(elem, send_message_batch); } -void CallData::FinishSendMessage() { +void CallData::FinishSendMessage(grpc_call_element* elem) { GPR_DEBUG_ASSERT(message_compression_algorithm_ != GRPC_MESSAGE_COMPRESS_NONE); // Compress the data if appropriate. @@ -334,17 +334,17 @@ void CallData::FinishSendMessage() { grpc_slice_buffer_destroy_internal(&tmp); // Swap out the original byte stream with our new one and send the // batch down. - replacement_stream_.Init(&slices_, send_flags); + replacement_stream_.emplace(&slices_, send_flags); send_message_batch_->payload->send_message.send_message.reset( - replacement_stream_.get()); + &replacement_stream_.value()); original_send_message_on_complete_ = send_message_batch_->on_complete; send_message_batch_->on_complete = &send_message_on_complete_; - SendMessageBatchContinue(); + SendMessageBatchContinue(elem); } -void CallData::FailSendMessageBatchInCallCombiner(void* arg, +void CallData::FailSendMessageBatchInCallCombiner(void* calld_arg, grpc_error* error) { - CallData* calld = static_cast(arg); + CallData* calld = static_cast(calld_arg); if (calld->send_message_batch_ != nullptr) { grpc_transport_stream_op_batch_finish_with_failure( calld->send_message_batch_, GRPC_ERROR_REF(error), @@ -353,8 +353,7 @@ void CallData::FailSendMessageBatchInCallCombiner(void* arg, } } -// Pulls a slice from the send_message byte stream and adds it to -// calld->slices_. +// Pulls a slice from the send_message byte stream and adds it to slices_. grpc_error* CallData::PullSliceFromSendMessage() { grpc_slice incoming_slice; grpc_error* error = @@ -369,11 +368,11 @@ grpc_error* CallData::PullSliceFromSendMessage() { // Reads as many slices as possible from the send_message byte stream. // If all data has been read, invokes FinishSendMessage(). Otherwise, // an async call to ByteStream::Next() has been started, which will -// eventually result in calling on_send_message_next_done(). -void CallData::ContinueReadingSendMessage() { +// eventually result in calling OnSendMessageNextDone(). +void CallData::ContinueReadingSendMessage(grpc_call_element* elem) { if (slices_.length == send_message_batch_->payload->send_message.send_message->length()) { - FinishSendMessage(); + FinishSendMessage(elem); return; } while (send_message_batch_->payload->send_message.send_message->Next( @@ -387,15 +386,16 @@ void CallData::ContinueReadingSendMessage() { } if (slices_.length == send_message_batch_->payload->send_message.send_message->length()) { - FinishSendMessage(); + FinishSendMessage(elem); break; } } } // Async callback for ByteStream::Next(). -void CallData::OnSendMessageNextDone(void* arg, grpc_error* error) { - CallData* calld = static_cast(arg); +void CallData::OnSendMessageNextDone(void* elem_arg, grpc_error* error) { + grpc_call_element* elem = static_cast(elem_arg); + CallData* calld = static_cast(elem->call_data); if (error != GRPC_ERROR_NONE) { // Closure callback; does not take ownership of error. FailSendMessageBatchInCallCombiner(calld, error); @@ -410,109 +410,111 @@ void CallData::OnSendMessageNextDone(void* arg, grpc_error* error) { } if (calld->slices_.length == calld->send_message_batch_->payload->send_message .send_message->length()) { - calld->FinishSendMessage(); + calld->FinishSendMessage(elem); } else { - calld->ContinueReadingSendMessage(); + calld->ContinueReadingSendMessage(elem); } } -void CallData::StartSendMessageBatch(void* arg, grpc_error* /*unused*/) { - CallData* calld = static_cast(arg); +void CallData::StartSendMessageBatch(void* elem_arg, grpc_error* /*unused*/) { + grpc_call_element* elem = static_cast(elem_arg); + CallData* calld = static_cast(elem->call_data); if (calld->SkipMessageCompression()) { - calld->SendMessageBatchContinue(); + calld->SendMessageBatchContinue(elem); } else { - calld->ContinueReadingSendMessage(); + calld->ContinueReadingSendMessage(elem); } } void CallData::CompressStartTransportStreamOpBatch( grpc_call_element* elem, grpc_transport_stream_op_batch* batch) { GPR_TIMER_SCOPE("compress_start_transport_stream_op_batch", 0); - CallData* calld = static_cast(elem->call_data); // Handle cancel_stream. if (batch->cancel_stream) { - GRPC_ERROR_UNREF(calld->cancel_error_); - calld->cancel_error_ = - GRPC_ERROR_REF(batch->payload->cancel_stream.cancel_error); - if (calld->send_message_batch_ != nullptr) { - if (!calld->seen_initial_metadata_) { + GRPC_ERROR_UNREF(cancel_error_); + cancel_error_ = GRPC_ERROR_REF(batch->payload->cancel_stream.cancel_error); + if (send_message_batch_ != nullptr) { + if (!seen_initial_metadata_) { GRPC_CALL_COMBINER_START( - calld->call_combiner_, - GRPC_CLOSURE_CREATE(FailSendMessageBatchInCallCombiner, calld, + call_combiner_, + GRPC_CLOSURE_CREATE(FailSendMessageBatchInCallCombiner, this, grpc_schedule_on_exec_ctx), - GRPC_ERROR_REF(calld->cancel_error_), "failing send_message op"); + GRPC_ERROR_REF(cancel_error_), "failing send_message op"); } else { - calld->send_message_batch_->payload->send_message.send_message - ->Shutdown(GRPC_ERROR_REF(calld->cancel_error_)); + send_message_batch_->payload->send_message.send_message->Shutdown( + GRPC_ERROR_REF(cancel_error_)); } } - } else if (calld->cancel_error_ != GRPC_ERROR_NONE) { + } else if (cancel_error_ != GRPC_ERROR_NONE) { grpc_transport_stream_op_batch_finish_with_failure( - batch, GRPC_ERROR_REF(calld->cancel_error_), calld->call_combiner_); + batch, GRPC_ERROR_REF(cancel_error_), call_combiner_); return; } // Handle send_initial_metadata. if (batch->send_initial_metadata) { - GPR_ASSERT(!calld->seen_initial_metadata_); - grpc_error* error = calld->ProcessSendInitialMetadata( - batch->payload->send_initial_metadata.send_initial_metadata); + GPR_ASSERT(!seen_initial_metadata_); + grpc_error* error = ProcessSendInitialMetadata( + elem, batch->payload->send_initial_metadata.send_initial_metadata); if (error != GRPC_ERROR_NONE) { grpc_transport_stream_op_batch_finish_with_failure(batch, error, - calld->call_combiner_); + call_combiner_); return; } - calld->seen_initial_metadata_ = true; + seen_initial_metadata_ = true; // If we had previously received a batch containing a send_message op, // handle it now. Note that we need to re-enter the call combiner // for this, since we can't send two batches down while holding the // call combiner, since the connected_channel filter (at the bottom of // the call stack) will release the call combiner for each batch it sees. - if (calld->send_message_batch_ != nullptr) { + if (send_message_batch_ != nullptr) { GRPC_CALL_COMBINER_START( - calld->call_combiner_, - &calld->start_send_message_batch_in_call_combiner_, GRPC_ERROR_NONE, - "starting send_message after send_initial_metadata"); + call_combiner_, &start_send_message_batch_in_call_combiner_, + GRPC_ERROR_NONE, "starting send_message after send_initial_metadata"); } } // Handle send_message. if (batch->send_message) { - GPR_ASSERT(calld->send_message_batch_ == nullptr); - calld->send_message_batch_ = batch; + GPR_ASSERT(send_message_batch_ == nullptr); + send_message_batch_ = batch; // If we have not yet seen send_initial_metadata, then we have to - // wait. We save the batch in calld and then drop the call - // combiner, which we'll have to pick up again later when we get - // send_initial_metadata. - if (!calld->seen_initial_metadata_) { + // wait. We save the batch and then drop the call combiner, which we'll + // have to pick up again later when we get send_initial_metadata. + if (!seen_initial_metadata_) { GRPC_CALL_COMBINER_STOP( - calld->call_combiner_, - "send_message batch pending send_initial_metadata"); + call_combiner_, "send_message batch pending send_initial_metadata"); return; } - StartSendMessageBatch(calld, GRPC_ERROR_NONE); + StartSendMessageBatch(elem, GRPC_ERROR_NONE); } else { // Pass control down the stack. grpc_call_next_op(elem, batch); } } +void CompressStartTransportStreamOpBatch( + grpc_call_element* elem, grpc_transport_stream_op_batch* batch) { + CallData* calld = static_cast(elem->call_data); + calld->CompressStartTransportStreamOpBatch(elem, batch); +} + /* Constructor for call_data */ -static grpc_error* CompressInitCallElem(grpc_call_element* elem, - const grpc_call_element_args* args) { +grpc_error* CompressInitCallElem(grpc_call_element* elem, + const grpc_call_element_args* args) { new (elem->call_data) CallData(elem, *args); return GRPC_ERROR_NONE; } /* Destructor for call_data */ -static void CompressDestroyCallElem(grpc_call_element* elem, - const grpc_call_final_info* /*final_info*/, - grpc_closure* /*ignored*/) { +void CompressDestroyCallElem(grpc_call_element* elem, + const grpc_call_final_info* /*final_info*/, + grpc_closure* /*ignored*/) { CallData* calld = static_cast(elem->call_data); calld->~CallData(); } /* Constructor for ChannelData */ -static grpc_error* CompressInitChannelElem(grpc_channel_element* elem, - grpc_channel_element_args* args) { +grpc_error* CompressInitChannelElem(grpc_channel_element* elem, + grpc_channel_element_args* args) { new (elem->channel_data) ChannelData(args); return GRPC_ERROR_NONE; } @@ -526,7 +528,7 @@ void CompressDestroyChannelElem(grpc_channel_element* elem) { } // namespace const grpc_channel_filter grpc_message_compress_filter = { - CallData::CompressStartTransportStreamOpBatch, + CompressStartTransportStreamOpBatch, grpc_channel_next_op, sizeof(CallData), CompressInitCallElem, From 7a654ed1a5569c5bb3ee443ad24425a265e19669 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Fri, 10 Apr 2020 03:34:37 -0700 Subject: [PATCH 50/73] Remove const data member from ByteStream to make it copyable --- src/core/lib/transport/byte_stream.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/transport/byte_stream.h b/src/core/lib/transport/byte_stream.h index 6988ab843b2..ffa9b47c20d 100644 --- a/src/core/lib/transport/byte_stream.h +++ b/src/core/lib/transport/byte_stream.h @@ -72,7 +72,7 @@ class ByteStream : public Orphanable { : length_(length), flags_(flags) {} private: - const uint32_t length_; + uint32_t length_; uint32_t flags_; }; From d5d258b70f8a544f7386c81ea8a5b945369fddb0 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Fri, 10 Apr 2020 00:08:48 -0700 Subject: [PATCH 51/73] Do not set default port in url map patch --- tools/run_tests/run_xds_tests.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index 3dc66ab7424..b7e82310461 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -646,7 +646,7 @@ def create_url_map(gcp, name, backend_service, host_name): gcp.url_map = GcpResource(config['name'], result['targetLink']) -def patch_url_map_host_rule(gcp, name, backend_service, host_name): +def patch_url_map_host_rule_with_port(gcp, name, backend_service, host_name): config = { 'hostRules': [{ 'hosts': ['%s:%d' % (host_name, gcp.service_port)], @@ -1007,8 +1007,10 @@ try: if not gcp.service_port: raise Exception( 'Failed to find a valid ip:port for the forwarding rule') - patch_url_map_host_rule(gcp, url_map_name, backend_service, - service_host_name) + if gcp.service_port != _DEFAULT_SERVICE_PORT: + patch_url_map_host_rule_with_port(gcp, url_map_name, + backend_service, + service_host_name) startup_script = get_startup_script(args.path_to_server_binary, gcp.service_port) create_instance_template(gcp, template_name, args.network, From c34311d996f484fd9f3ba9de46966aef7295733c Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Fri, 10 Apr 2020 10:13:57 -0700 Subject: [PATCH 52/73] Wait longer for backend instance patch --- tools/run_tests/run_xds_tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index b7e82310461..3ee9286431d 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -797,7 +797,9 @@ def patch_backend_instances(gcp, result = gcp.alpha_compute.backendServices().patch( project=gcp.project, backendService=backend_service.name, body=config).execute() - wait_for_global_operation(gcp, result['name']) + wait_for_global_operation(gcp, + result['name'], + timeout_sec=_WAIT_FOR_BACKEND_SEC) def resize_instance_group(gcp, From 6355d71a9fd116f6fbf8c22e9caf79bc58c9d27d Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Fri, 10 Apr 2020 10:41:21 -0700 Subject: [PATCH 53/73] Make GCP alpha API use optional --- tools/run_tests/run_xds_tests.py | 119 +++++++++++++++++++++---------- 1 file changed, 82 insertions(+), 37 deletions(-) diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index 3ee9286431d..27d195efcb5 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -182,6 +182,10 @@ argp.add_argument('--log_client_output', help='Log captured client output', default=False, action='store_true') +argp.add_argument('--only_stable_gcp_apis', + help='Do not use alpha compute APIs', + default=False, + action='store_true') args = argp.parse_args() if args.verbose: @@ -577,16 +581,27 @@ def add_instance_group(gcp, zone, name, size): def create_health_check(gcp, name): - config = { - 'name': name, - 'type': 'GRPC', - 'grpcHealthCheck': { - 'portSpecification': 'USE_SERVING_PORT' + if gcp.alpha_compute: + config = { + 'name': name, + 'type': 'GRPC', + 'grpcHealthCheck': { + 'portSpecification': 'USE_SERVING_PORT' + } } - } + compute_to_use = gcp.alpha_compute + else: + config = { + 'name': name, + 'type': 'TCP', + 'tcpHealthCheck': { + 'portName': 'grpc' + } + } + compute_to_use = gcp.compute logger.debug('Sending GCP request with body=%s', config) - result = gcp.alpha_compute.healthChecks().insert(project=gcp.project, - body=config).execute() + result = compute_to_use.healthChecks().insert(project=gcp.project, + body=config).execute() wait_for_global_operation(gcp, result['name']) gcp.health_check = GcpResource(config['name'], result['targetLink']) @@ -610,15 +625,21 @@ def create_health_check_firewall_rule(gcp, name): def add_backend_service(gcp, name): + if gcp.alpha_compute: + protocol = 'GRPC' + compute_to_use = gcp.alpha_compute + else: + protocol = 'HTTP2' + compute_to_use = gcp.compute config = { 'name': name, 'loadBalancingScheme': 'INTERNAL_SELF_MANAGED', 'healthChecks': [gcp.health_check.url], 'portName': 'grpc', - 'protocol': 'GRPC' + 'protocol': protocol } logger.debug('Sending GCP request with body=%s', config) - result = gcp.alpha_compute.backendServices().insert(project=gcp.project, + result = compute_to_use.backendServices().insert(project=gcp.project, body=config).execute() wait_for_global_operation(gcp, result['name']) backend_service = GcpResource(config['name'], result['targetLink']) @@ -660,20 +681,33 @@ def patch_url_map_host_rule_with_port(gcp, name, backend_service, host_name): wait_for_global_operation(gcp, result['name']) -def create_target_grpc_proxy(gcp, name): - config = { - 'name': name, - 'url_map': gcp.url_map.url, - 'validate_for_proxyless': True, - } - logger.debug('Sending GCP request with body=%s', config) - result = gcp.alpha_compute.targetGrpcProxies().insert( - project=gcp.project, body=config).execute() +def create_target_proxy(gcp, name): + if gcp.alpha_compute: + config = { + 'name': name, + 'url_map': gcp.url_map.url, + 'validate_for_proxyless': True, + } + logger.debug('Sending GCP request with body=%s', config) + result = gcp.alpha_compute.targetGrpcProxies().insert( + project=gcp.project, body=config).execute() + else: + config = { + 'name': name, + 'url_map': gcp.url_map.url, + } + logger.debug('Sending GCP request with body=%s', config) + result = gcp.compute.targetHttpProxies().insert(project=gcp.project, + body=config).execute() wait_for_global_operation(gcp, result['name']) - gcp.target_grpc_proxy = GcpResource(config['name'], result['targetLink']) + gcp.target_proxy = GcpResource(config['name'], result['targetLink']) def create_global_forwarding_rule(gcp, name, potential_ports): + if gcp.alpha_compute: + compute_to_use = gcp.alpha_compute + else: + compute_to_use = gcp.compute for port in potential_ports: try: config = { @@ -682,10 +716,10 @@ def create_global_forwarding_rule(gcp, name, potential_ports): 'portRange': str(port), 'IPAddress': '0.0.0.0', 'network': args.network, - 'target': gcp.target_grpc_proxy.url, + 'target': gcp.target_proxy.url, } logger.debug('Sending GCP request with body=%s', config) - result = gcp.alpha_compute.globalForwardingRules().insert( + result = compute_to_use.globalForwardingRules().insert( project=gcp.project, body=config).execute() wait_for_global_operation(gcp, result['name']) gcp.global_forwarding_rule = GcpResource(config['name'], @@ -708,11 +742,16 @@ def delete_global_forwarding_rule(gcp): logger.info('Delete failed: %s', http_error) -def delete_target_grpc_proxy(gcp): +def delete_target_proxy(gcp): try: - result = gcp.alpha_compute.targetGrpcProxies().delete( - project=gcp.project, - targetGrpcProxy=gcp.target_grpc_proxy.name).execute() + if gcp.alpha_compute: + result = gcp.alpha_compute.targetGrpcProxies().delete( + project=gcp.project, + targetGrpcProxy=gcp.target_proxy.name).execute() + else: + result = gcp.alpha_compute.targetHttpProxies().delete( + project=gcp.project, + targetHttpProxy=gcp.target_proxy.name).execute() wait_for_global_operation(gcp, result['name']) except googleapiclient.errors.HttpError as http_error: logger.info('Delete failed: %s', http_error) @@ -786,6 +825,10 @@ def patch_backend_instances(gcp, backend_service, instance_groups, balancing_mode='UTILIZATION'): + if gcp.alpha_compute: + compute_to_use = gcp.alpha_compute + else: + compute_to_use = gcp.compute config = { 'backends': [{ 'group': instance_group.url, @@ -794,7 +837,7 @@ def patch_backend_instances(gcp, } for instance_group in instance_groups], } logger.debug('Sending GCP request with body=%s', config) - result = gcp.alpha_compute.backendServices().patch( + result = compute_to_use.backendServices().patch( project=gcp.project, backendService=backend_service.name, body=config).execute() wait_for_global_operation(gcp, @@ -922,8 +965,8 @@ def get_instance_names(gcp, instance_group): def clean_up(gcp): if gcp.global_forwarding_rule: delete_global_forwarding_rule(gcp) - if gcp.target_grpc_proxy: - delete_target_grpc_proxy(gcp) + if gcp.target_proxy: + delete_target_proxy(gcp) if gcp.url_map: delete_url_map(gcp) delete_backend_services(gcp) @@ -961,23 +1004,25 @@ class GcpState(object): self.health_check_firewall_rule = None self.backend_services = [] self.url_map = None - self.target_grpc_proxy = None + self.target_proxy = None self.global_forwarding_rule = None self.service_port = None self.instance_template = None self.instance_groups = [] - +alpha_compute = None if args.compute_discovery_document: with open(args.compute_discovery_document, 'r') as discovery_doc: compute = googleapiclient.discovery.build_from_document( discovery_doc.read()) - with open(args.alpha_compute_discovery_document, 'r') as discovery_doc: - alpha_compute = googleapiclient.discovery.build_from_document( - discovery_doc.read()) + if not args.only_stable_gcp_apis and args.alpha_compute_discovery_document: + with open(args.alpha_compute_discovery_document, 'r') as discovery_doc: + alpha_compute = googleapiclient.discovery.build_from_document( + discovery_doc.read()) else: compute = googleapiclient.discovery.build('compute', 'v1') - alpha_compute = googleapiclient.discovery.build('compute', 'alpha') + if not args.only_stable_gcp_apis: + alpha_compute = googleapiclient.discovery.build('compute', 'alpha') try: gcp = GcpState(compute, alpha_compute, args.project_id) @@ -987,7 +1032,7 @@ try: alternate_backend_service_name = _BASE_BACKEND_SERVICE_NAME + '-alternate' + args.gcp_suffix url_map_name = _BASE_URL_MAP_NAME + args.gcp_suffix service_host_name = _BASE_SERVICE_HOST + args.gcp_suffix - target_grpc_proxy_name = _BASE_TARGET_PROXY_NAME + args.gcp_suffix + target_proxy_name = _BASE_TARGET_PROXY_NAME + args.gcp_suffix forwarding_rule_name = _BASE_FORWARDING_RULE_NAME + args.gcp_suffix template_name = _BASE_TEMPLATE_NAME + args.gcp_suffix instance_group_name = _BASE_INSTANCE_GROUP_NAME + args.gcp_suffix @@ -1001,7 +1046,7 @@ try: alternate_backend_service = add_backend_service( gcp, alternate_backend_service_name) create_url_map(gcp, url_map_name, backend_service, service_host_name) - create_target_grpc_proxy(gcp, target_grpc_proxy_name) + create_target_proxy(gcp, target_proxy_name) potential_service_ports = list(args.service_port_range) random.shuffle(potential_service_ports) create_global_forwarding_rule(gcp, forwarding_rule_name, From ade157dd7afa0c7536767f188a3553c5e9966880 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Fri, 10 Apr 2020 11:01:54 -0700 Subject: [PATCH 54/73] yapf --- tools/run_tests/run_xds_tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index 27d195efcb5..845aa228171 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -640,7 +640,7 @@ def add_backend_service(gcp, name): } logger.debug('Sending GCP request with body=%s', config) result = compute_to_use.backendServices().insert(project=gcp.project, - body=config).execute() + body=config).execute() wait_for_global_operation(gcp, result['name']) backend_service = GcpResource(config['name'], result['targetLink']) gcp.backend_services.append(backend_service) @@ -1010,6 +1010,7 @@ class GcpState(object): self.instance_template = None self.instance_groups = [] + alpha_compute = None if args.compute_discovery_document: with open(args.compute_discovery_document, 'r') as discovery_doc: From 6aa983e819047a10eda73b235b5903f5ec42982c Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Fri, 10 Apr 2020 11:26:26 -0700 Subject: [PATCH 55/73] typo --- tools/run_tests/run_xds_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index 845aa228171..762743e0993 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -749,7 +749,7 @@ def delete_target_proxy(gcp): project=gcp.project, targetGrpcProxy=gcp.target_proxy.name).execute() else: - result = gcp.alpha_compute.targetHttpProxies().delete( + result = gcp.compute.targetHttpProxies().delete( project=gcp.project, targetHttpProxy=gcp.target_proxy.name).execute() wait_for_global_operation(gcp, result['name']) From edc2a6b988c8019b2e5debaaa8d4716c62fbe3d9 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Fri, 10 Apr 2020 12:16:37 -0700 Subject: [PATCH 56/73] Add option for logging keepalive pings --- doc/environment_variables.md | 1 + .../transport/chttp2/transport/chttp2_transport.cc | 8 ++++++-- .../transport/chttp2/transport/chttp2_transport.h | 1 + src/core/ext/transport/chttp2/transport/writing.cc | 13 +++++++++---- 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/doc/environment_variables.md b/doc/environment_variables.md index ab45f937bff..fe3a81657eb 100644 --- a/doc/environment_variables.md +++ b/doc/environment_variables.md @@ -66,6 +66,7 @@ some configuration as environment variables that can be set. - http2_stream_state - traces all http2 stream state mutations. - http1 - traces HTTP/1.x operations performed by gRPC - inproc - traces the in-process transport + - keepalive - traces keepalive pings - flowctl - traces http2 flow control - lrs_lb - traces lrs LB policy - op_failure - traces error information when failure is pushed onto a diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index 3f27466611f..790a66c1260 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -99,6 +99,7 @@ static int g_default_max_ping_strikes = DEFAULT_MAX_PING_STRIKES; #define MAX_CLIENT_STREAM_ID 0x7fffffffu grpc_core::TraceFlag grpc_http_trace(false, "http"); +grpc_core::TraceFlag grpc_keepalive_trace(false, "keepalive"); grpc_core::DebugOnlyTraceFlag grpc_trace_chttp2_refcount(false, "chttp2_refcount"); @@ -2771,6 +2772,7 @@ static void init_keepalive_ping(void* arg, grpc_error* error) { static void init_keepalive_ping_locked(void* arg, grpc_error* error) { grpc_chttp2_transport* t = static_cast(arg); + GPR_ASSERT(t->keepalive_state == GRPC_CHTTP2_KEEPALIVE_STATE_WAITING); if (t->destroying || t->closed_with_error != GRPC_ERROR_NONE) { t->keepalive_state = GRPC_CHTTP2_KEEPALIVE_STATE_DYING; @@ -2817,7 +2819,8 @@ static void start_keepalive_ping_locked(void* arg, grpc_error* error) { if (t->channelz_socket != nullptr) { t->channelz_socket->RecordKeepaliveSent(); } - if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace) || + GRPC_TRACE_FLAG_ENABLED(grpc_keepalive_trace)) { gpr_log(GPR_INFO, "%s: Start keepalive ping", t->peer_string); } GRPC_CHTTP2_REF_TRANSPORT(t, "keepalive watchdog"); @@ -2840,7 +2843,8 @@ static void finish_keepalive_ping_locked(void* arg, grpc_error* error) { grpc_chttp2_transport* t = static_cast(arg); if (t->keepalive_state == GRPC_CHTTP2_KEEPALIVE_STATE_PINGING) { if (error == GRPC_ERROR_NONE) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace) || + GRPC_TRACE_FLAG_ENABLED(grpc_keepalive_trace)) { gpr_log(GPR_INFO, "%s: Finish keepalive ping", t->peer_string); } if (!t->keepalive_ping_started) { diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.h b/src/core/ext/transport/chttp2/transport/chttp2_transport.h index 39574f93ec7..b04630bbe2b 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.h +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.h @@ -27,6 +27,7 @@ #include "src/core/lib/transport/transport.h" extern grpc_core::TraceFlag grpc_http_trace; +extern grpc_core::TraceFlag grpc_keepalive_trace; extern grpc_core::TraceFlag grpc_trace_http2_stream_state; extern grpc_core::DebugOnlyTraceFlag grpc_trace_chttp2_refcount; extern grpc_core::DebugOnlyTraceFlag grpc_trace_chttp2_hpack_parser; diff --git a/src/core/ext/transport/chttp2/transport/writing.cc b/src/core/ext/transport/chttp2/transport/writing.cc index c7613afcf6f..98d7f9f3edc 100644 --- a/src/core/ext/transport/chttp2/transport/writing.cc +++ b/src/core/ext/transport/chttp2/transport/writing.cc @@ -18,6 +18,7 @@ #include +#include "src/core/ext/transport/chttp2/transport/chttp2_transport.h" #include "src/core/ext/transport/chttp2/transport/context_list.h" #include "src/core/ext/transport/chttp2/transport/internal.h" @@ -54,7 +55,8 @@ static void maybe_initiate_ping(grpc_chttp2_transport* t) { if (!grpc_closure_list_empty(pq->lists[GRPC_CHTTP2_PCL_INFLIGHT])) { /* ping already in-flight: wait */ if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace) || - GRPC_TRACE_FLAG_ENABLED(grpc_bdp_estimator_trace)) { + GRPC_TRACE_FLAG_ENABLED(grpc_bdp_estimator_trace) || + GRPC_TRACE_FLAG_ENABLED(grpc_keepalive_trace)) { gpr_log(GPR_INFO, "%s: Ping delayed [%p]: already pinging", t->is_client ? "CLIENT" : "SERVER", t->peer_string); } @@ -64,7 +66,8 @@ static void maybe_initiate_ping(grpc_chttp2_transport* t) { t->ping_policy.max_pings_without_data != 0) { /* need to receive something of substance before sending a ping again */ if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace) || - GRPC_TRACE_FLAG_ENABLED(grpc_bdp_estimator_trace)) { + GRPC_TRACE_FLAG_ENABLED(grpc_bdp_estimator_trace) || + GRPC_TRACE_FLAG_ENABLED(grpc_keepalive_trace)) { gpr_log(GPR_INFO, "%s: Ping delayed [%p]: too many recent pings: %d/%d", t->is_client ? "CLIENT" : "SERVER", t->peer_string, t->ping_state.pings_before_data_required, @@ -85,7 +88,8 @@ static void maybe_initiate_ping(grpc_chttp2_transport* t) { if (next_allowed_ping > now) { /* not enough elapsed time between successive pings */ if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace) || - GRPC_TRACE_FLAG_ENABLED(grpc_bdp_estimator_trace)) { + GRPC_TRACE_FLAG_ENABLED(grpc_bdp_estimator_trace) || + GRPC_TRACE_FLAG_ENABLED(grpc_keepalive_trace)) { gpr_log(GPR_INFO, "%s: Ping delayed [%p]: not enough time elapsed since last ping. " " Last ping %f: Next ping %f: Now %f", @@ -116,7 +120,8 @@ static void maybe_initiate_ping(grpc_chttp2_transport* t) { GRPC_STATS_INC_HTTP2_PINGS_SENT(); t->ping_state.last_ping_sent_time = now; if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace) || - GRPC_TRACE_FLAG_ENABLED(grpc_bdp_estimator_trace)) { + GRPC_TRACE_FLAG_ENABLED(grpc_bdp_estimator_trace) || + GRPC_TRACE_FLAG_ENABLED(grpc_keepalive_trace)) { gpr_log(GPR_INFO, "%s: Ping sent [%s]: %d/%d", t->is_client ? "CLIENT" : "SERVER", t->peer_string, t->ping_state.pings_before_data_required, From c90dc0e0987cbce29db4b9ac39ecec75ec78dd23 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Fri, 10 Apr 2020 12:18:39 -0700 Subject: [PATCH 57/73] Remove blank line --- src/core/ext/transport/chttp2/transport/chttp2_transport.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index 790a66c1260..f1724764da3 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -2772,7 +2772,6 @@ static void init_keepalive_ping(void* arg, grpc_error* error) { static void init_keepalive_ping_locked(void* arg, grpc_error* error) { grpc_chttp2_transport* t = static_cast(arg); - GPR_ASSERT(t->keepalive_state == GRPC_CHTTP2_KEEPALIVE_STATE_WAITING); if (t->destroying || t->closed_with_error != GRPC_ERROR_NONE) { t->keepalive_state = GRPC_CHTTP2_KEEPALIVE_STATE_DYING; From 2d0ed47223e252a8ecf2d1fca9a0fdee8ac5711b Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Fri, 10 Apr 2020 12:28:25 -0700 Subject: [PATCH 58/73] Reviewer comments --- doc/environment_variables.md | 2 +- src/core/ext/transport/chttp2/transport/chttp2_transport.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/environment_variables.md b/doc/environment_variables.md index fe3a81657eb..ec1b4f86717 100644 --- a/doc/environment_variables.md +++ b/doc/environment_variables.md @@ -66,7 +66,7 @@ some configuration as environment variables that can be set. - http2_stream_state - traces all http2 stream state mutations. - http1 - traces HTTP/1.x operations performed by gRPC - inproc - traces the in-process transport - - keepalive - traces keepalive pings + - http_keepalive - traces gRPC keepalive pings - flowctl - traces http2 flow control - lrs_lb - traces lrs LB policy - op_failure - traces error information when failure is pushed onto a diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index f1724764da3..cc03a406903 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -99,7 +99,7 @@ static int g_default_max_ping_strikes = DEFAULT_MAX_PING_STRIKES; #define MAX_CLIENT_STREAM_ID 0x7fffffffu grpc_core::TraceFlag grpc_http_trace(false, "http"); -grpc_core::TraceFlag grpc_keepalive_trace(false, "keepalive"); +grpc_core::TraceFlag grpc_keepalive_trace(false, "http_keepalive"); grpc_core::DebugOnlyTraceFlag grpc_trace_chttp2_refcount(false, "chttp2_refcount"); From cdf7b2cc7a9404d993c43d44ca00b762b7cfa087 Mon Sep 17 00:00:00 2001 From: Yash Tibrewal Date: Fri, 10 Apr 2020 14:03:25 -0700 Subject: [PATCH 59/73] Use std::aligned_storage instead of absl::optional to maintain compatibility with Windows --- .../http/message_compress/message_compress_filter.cc | 11 ++++++++--- src/core/lib/transport/byte_stream.h | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/core/ext/filters/http/message_compress/message_compress_filter.cc b/src/core/ext/filters/http/message_compress/message_compress_filter.cc index b962c99e1d5..09c914fab07 100644 --- a/src/core/ext/filters/http/message_compress/message_compress_filter.cc +++ b/src/core/ext/filters/http/message_compress/message_compress_filter.cc @@ -163,7 +163,10 @@ class CallData { grpc_linked_mdelem accept_encoding_storage_; grpc_linked_mdelem accept_stream_encoding_storage_; grpc_slice_buffer slices_; /**< Buffers up input slices to be compressed */ - absl::optional replacement_stream_; + // Allocate space for the replacement stream + std::aligned_storage::type + replacement_stream_; grpc_closure* original_send_message_on_complete_ = nullptr; grpc_closure send_message_on_complete_; grpc_closure on_send_message_next_done_; @@ -334,9 +337,11 @@ void CallData::FinishSendMessage(grpc_call_element* elem) { grpc_slice_buffer_destroy_internal(&tmp); // Swap out the original byte stream with our new one and send the // batch down. - replacement_stream_.emplace(&slices_, send_flags); + new (&replacement_stream_) + grpc_core::SliceBufferByteStream(&slices_, send_flags); send_message_batch_->payload->send_message.send_message.reset( - &replacement_stream_.value()); + reinterpret_cast( + &replacement_stream_)); original_send_message_on_complete_ = send_message_batch_->on_complete; send_message_batch_->on_complete = &send_message_on_complete_; SendMessageBatchContinue(elem); diff --git a/src/core/lib/transport/byte_stream.h b/src/core/lib/transport/byte_stream.h index ffa9b47c20d..6988ab843b2 100644 --- a/src/core/lib/transport/byte_stream.h +++ b/src/core/lib/transport/byte_stream.h @@ -72,7 +72,7 @@ class ByteStream : public Orphanable { : length_(length), flags_(flags) {} private: - uint32_t length_; + const uint32_t length_; uint32_t flags_; }; From 4e6327493dad16acad922fade50c23404ebb4531 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Fri, 10 Apr 2020 18:57:37 -0700 Subject: [PATCH 60/73] Support SO_REUSEPORT on manylinux2010 --- src/core/lib/iomgr/socket_utils_common_posix.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/core/lib/iomgr/socket_utils_common_posix.cc b/src/core/lib/iomgr/socket_utils_common_posix.cc index a5048d890d3..f368cb76f62 100644 --- a/src/core/lib/iomgr/socket_utils_common_posix.cc +++ b/src/core/lib/iomgr/socket_utils_common_posix.cc @@ -210,7 +210,6 @@ static gpr_once g_probe_so_reuesport_once = GPR_ONCE_INIT; static int g_support_so_reuseport = false; void probe_so_reuseport_once(void) { -#ifndef GPR_MANYLINUX1 int s = socket(AF_INET, SOCK_STREAM, 0); if (s < 0) { /* This might be an ipv6-only environment in which case 'socket(AF_INET,..)' @@ -222,7 +221,6 @@ void probe_so_reuseport_once(void) { "check for SO_REUSEPORT", grpc_set_socket_reuse_port(s, 1)); close(s); } -#endif } bool grpc_is_socket_reuse_port_supported() { From 96024a9ad30c9863f7a94716ba79e6f087368067 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Fri, 10 Apr 2020 19:22:55 -0700 Subject: [PATCH 61/73] Update documentation --- examples/python/multiprocessing/README.md | 17 ++++++++--------- examples/python/multiprocessing/server.py | 6 ------ 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/examples/python/multiprocessing/README.md b/examples/python/multiprocessing/README.md index 709a815aca5..19cc00b6a4e 100644 --- a/examples/python/multiprocessing/README.md +++ b/examples/python/multiprocessing/README.md @@ -1,16 +1,19 @@ ## Multiprocessing with gRPC Python Multiprocessing allows application developers to sidestep the Python global -interpreter lock and achieve true concurrency on multicore systems. +interpreter lock and achieve true parallelism on multicore systems. Unfortunately, using multiprocessing and gRPC Python is not yet as simple as instantiating your server with a `futures.ProcessPoolExecutor`. The library is implemented as a C extension, maintaining much of the state that drives the system in native code. As such, upon calling -[`fork`](http://man7.org/linux/man-pages/man2/fork.2.html), much of the -state copied into the child process is invalid, leading to hangs and crashes. +[`fork`](http://man7.org/linux/man-pages/man2/fork.2.html), any threads in a +critical section may leave the state of the gRPC library invalid in the child +process. See this [excellent research +paper](https://www.microsoft.com/en-us/research/uploads/prod/2019/04/fork-hotos19.pdf) +for a thorough discussion of the topic. -However, calling `fork` without `exec` in your python process is supported +Ccalling `fork` without `exec` in your process *is* supported *before* any gRPC servers have been instantiated. Application developers can take advantage of this to parallelize their CPU-intensive operations. @@ -18,11 +21,7 @@ take advantage of this to parallelize their CPU-intensive operations. This example calculates the first 10,000 prime numbers as an RPC. We instantiate one server per subprocess, balancing requests between the servers using the -[`SO_REUSEPORT`](https://lwn.net/Articles/542629/) socket option. Note that this -option is not available in `manylinux1` distributions, which are, as of the time -of writing, the only gRPC Python wheels available on PyPI. To take advantage of this -feature, you'll need to build from source, either using bazel (as we do for -these examples) or via pip, using `pip install grpcio --no-binary grpcio`. +[`SO_REUSEPORT`](https://lwn.net/Articles/542629/) socket option. ```python _PROCESS_COUNT = multiprocessing.cpu_count() diff --git a/examples/python/multiprocessing/server.py b/examples/python/multiprocessing/server.py index ad788b8eb51..74a8860119f 100644 --- a/examples/python/multiprocessing/server.py +++ b/examples/python/multiprocessing/server.py @@ -67,12 +67,6 @@ def _run_server(bind_address): _LOGGER.info('Starting new server.') options = (('grpc.so_reuseport', 1),) - # WARNING: This example takes advantage of SO_REUSEPORT. Due to the - # limitations of manylinux1, none of our precompiled Linux wheels currently - # support this option. (https://github.com/grpc/grpc/issues/18210). To take - # advantage of this feature, install from source with - # `pip install grpcio --no-binary grpcio`. - server = grpc.server(futures.ThreadPoolExecutor( max_workers=_THREAD_CONCURRENCY,), options=options) From e97cd37e68c49504af71429bd0a3083f17aedd17 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Fri, 10 Apr 2020 19:33:07 -0700 Subject: [PATCH 62/73] Support running multiprocessing example without Bazel --- examples/python/multiprocessing/BUILD | 2 ++ examples/python/multiprocessing/README.md | 8 ++++++++ examples/python/multiprocessing/client.py | 4 ++-- examples/python/multiprocessing/server.py | 4 ++-- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/examples/python/multiprocessing/BUILD b/examples/python/multiprocessing/BUILD index ea9b6a3ec6f..1ee3b647ae4 100644 --- a/examples/python/multiprocessing/BUILD +++ b/examples/python/multiprocessing/BUILD @@ -44,6 +44,7 @@ py_binary( ":prime_proto_pb2_grpc", "//src/python/grpcio/grpc:grpcio", ], + imports = ["."], ) py_binary( @@ -60,6 +61,7 @@ py_binary( "//conditions:default": ["@futures//:futures"], "//:python3": [], }), + imports = ["."], ) py_test( diff --git a/examples/python/multiprocessing/README.md b/examples/python/multiprocessing/README.md index 19cc00b6a4e..a800190682f 100644 --- a/examples/python/multiprocessing/README.md +++ b/examples/python/multiprocessing/README.md @@ -64,3 +64,11 @@ For example, ``` bazel run //examples/python/multiprocessing:client -- [::]:33915 ``` + +Alternatively, generate code using the following and then run the client and server +directly: + +```python +cd examples/python/helloworld +python -m grpc_tools.protoc -I . prime.proto --python_out=. --grpc_python_out=. +``` diff --git a/examples/python/multiprocessing/client.py b/examples/python/multiprocessing/client.py index b9acc65fdc5..7676bd4ec88 100644 --- a/examples/python/multiprocessing/client.py +++ b/examples/python/multiprocessing/client.py @@ -26,8 +26,8 @@ import sys import grpc -from examples.python.multiprocessing import prime_pb2 -from examples.python.multiprocessing import prime_pb2_grpc +import prime_pb2 +import prime_pb2_grpc _PROCESS_COUNT = 8 _MAXIMUM_CANDIDATE = 10000 diff --git a/examples/python/multiprocessing/server.py b/examples/python/multiprocessing/server.py index 74a8860119f..a5ee00755e6 100644 --- a/examples/python/multiprocessing/server.py +++ b/examples/python/multiprocessing/server.py @@ -29,8 +29,8 @@ import sys import grpc -from examples.python.multiprocessing import prime_pb2 -from examples.python.multiprocessing import prime_pb2_grpc +import prime_pb2 +import prime_pb2_grpc _LOGGER = logging.getLogger(__name__) From c14fce7ab4e25c39639a306ca2ed54e0cf56f3fd Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Fri, 10 Apr 2020 19:35:01 -0700 Subject: [PATCH 63/73] Typo --- examples/python/multiprocessing/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/python/multiprocessing/README.md b/examples/python/multiprocessing/README.md index a800190682f..5dce50ad3bd 100644 --- a/examples/python/multiprocessing/README.md +++ b/examples/python/multiprocessing/README.md @@ -13,8 +13,8 @@ process. See this [excellent research paper](https://www.microsoft.com/en-us/research/uploads/prod/2019/04/fork-hotos19.pdf) for a thorough discussion of the topic. -Ccalling `fork` without `exec` in your process *is* supported -*before* any gRPC servers have been instantiated. Application developers can +Calling `fork` without `exec` in your process *is* supported +before any gRPC servers have been instantiated. Application developers can take advantage of this to parallelize their CPU-intensive operations. ## Calculating Prime Numbers with Multiple Processes From 0cdffa970c1ad18fb77cb97cec18a236cfbfbf9d Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Mon, 13 Apr 2020 10:25:56 -0700 Subject: [PATCH 64/73] Buildifier --- examples/python/multiprocessing/BUILD | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/python/multiprocessing/BUILD b/examples/python/multiprocessing/BUILD index 1ee3b647ae4..1d831e729b4 100644 --- a/examples/python/multiprocessing/BUILD +++ b/examples/python/multiprocessing/BUILD @@ -37,6 +37,7 @@ py_binary( name = "client", testonly = 1, srcs = ["client.py"], + imports = ["."], python_version = "PY3", srcs_version = "PY3", deps = [ @@ -44,13 +45,13 @@ py_binary( ":prime_proto_pb2_grpc", "//src/python/grpcio/grpc:grpcio", ], - imports = ["."], ) py_binary( name = "server", testonly = 1, srcs = ["server.py"], + imports = ["."], python_version = "PY3", srcs_version = "PY3", deps = [ @@ -61,7 +62,6 @@ py_binary( "//conditions:default": ["@futures//:futures"], "//:python3": [], }), - imports = ["."], ) py_test( From dc976d2a77166374c8acab6b1c90d9bf8c1b0ad1 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Mon, 13 Apr 2020 11:09:36 -0700 Subject: [PATCH 65/73] xds: Remove fallback code and support for old xds LB configs. --- include/grpc/impl/codegen/grpc_types.h | 4 - .../client_channel/lb_policy/xds/eds.cc | 382 +++--------------- test/cpp/end2end/xds_end2end_test.cc | 271 +------------ 3 files changed, 71 insertions(+), 586 deletions(-) diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index ff45450f3a3..c7dd23caa49 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -344,10 +344,6 @@ typedef struct { balancer before using fallback backend addresses from the resolver. If 0, enter fallback mode immediately. Default value is 10000. */ #define GRPC_ARG_GRPCLB_FALLBACK_TIMEOUT_MS "grpc.grpclb_fallback_timeout_ms" -/* Timeout in milliseconds to wait for the serverlist from the xDS load - balancer before using fallback backend addresses from the resolver. - If 0, enter fallback mode immediately. Default value is 10000. */ -#define GRPC_ARG_XDS_FALLBACK_TIMEOUT_MS "grpc.xds_fallback_timeout_ms" /* Timeout in milliseconds to wait for the child of a specific priority to complete its initial connection attempt before the priority LB policy fails over to the next priority. Default value is 10 seconds. */ diff --git a/src/core/ext/filters/client_channel/lb_policy/xds/eds.cc b/src/core/ext/filters/client_channel/lb_policy/xds/eds.cc index 099fb286223..8351d74d751 100644 --- a/src/core/ext/filters/client_channel/lb_policy/xds/eds.cc +++ b/src/core/ext/filters/client_channel/lb_policy/xds/eds.cc @@ -49,27 +49,22 @@ TraceFlag grpc_lb_eds_trace(false, "eds_lb"); namespace { -constexpr char kXds[] = "xds_experimental"; constexpr char kEds[] = "eds_experimental"; // Config for EDS LB policy. class EdsLbConfig : public LoadBalancingPolicy::Config { public: - EdsLbConfig(const char* name, std::string cluster_name, - std::string eds_service_name, + EdsLbConfig(std::string cluster_name, std::string eds_service_name, absl::optional lrs_load_reporting_server_name, - Json locality_picking_policy, Json endpoint_picking_policy, - RefCountedPtr fallback_policy) - : name_(name), - cluster_name_(std::move(cluster_name)), + Json locality_picking_policy, Json endpoint_picking_policy) + : cluster_name_(std::move(cluster_name)), eds_service_name_(std::move(eds_service_name)), lrs_load_reporting_server_name_( std::move(lrs_load_reporting_server_name)), locality_picking_policy_(std::move(locality_picking_policy)), - endpoint_picking_policy_(std::move(endpoint_picking_policy)), - fallback_policy_(std::move(fallback_policy)) {} + endpoint_picking_policy_(std::move(endpoint_picking_policy)) {} - const char* name() const override { return name_; } + const char* name() const override { return kEds; } const std::string& cluster_name() const { return cluster_name_; } const std::string& eds_service_name() const { return eds_service_name_; } @@ -82,26 +77,21 @@ class EdsLbConfig : public LoadBalancingPolicy::Config { const Json& endpoint_picking_policy() const { return endpoint_picking_policy_; } - RefCountedPtr fallback_policy() const { - return fallback_policy_; - } private: - const char* name_; std::string cluster_name_; std::string eds_service_name_; absl::optional lrs_load_reporting_server_name_; Json locality_picking_policy_; Json endpoint_picking_policy_; - RefCountedPtr fallback_policy_; }; // EDS LB policy. class EdsLb : public LoadBalancingPolicy { public: - EdsLb(const char* name, Args args); + explicit EdsLb(Args args); - const char* name() const override { return name_; } + const char* name() const override { return kEds; } void UpdateLocked(UpdateArgs args) override; void ResetBackoffLocked() override; @@ -153,24 +143,6 @@ class EdsLb : public LoadBalancingPolicy { RefCountedPtr eds_policy_; }; - class FallbackHelper : public ChannelControlHelper { - public: - explicit FallbackHelper(RefCountedPtr parent) - : parent_(std::move(parent)) {} - - ~FallbackHelper() { parent_.reset(DEBUG_LOCATION, "FallbackHelper"); } - - RefCountedPtr CreateSubchannel( - const grpc_channel_args& args) override; - void UpdateState(grpc_connectivity_state state, - std::unique_ptr picker) override; - void RequestReresolution() override; - void AddTraceEvent(TraceSeverity severity, StringView message) override; - - private: - RefCountedPtr parent_; - }; - ~EdsLb(); void ShutdownLocked() override; @@ -185,15 +157,6 @@ class EdsLb : public LoadBalancingPolicy { const grpc_channel_args* args_in); void MaybeUpdateDropPickerLocked(); - // Methods for dealing with fallback state. - void MaybeCancelFallbackAtStartupChecks(); - static void OnFallbackTimer(void* arg, grpc_error* error); - static void OnFallbackTimerLocked(void* arg, grpc_error* error); - void UpdateFallbackPolicyLocked(); - OrphanablePtr CreateFallbackPolicyLocked( - const grpc_channel_args* args); - void MaybeExitFallbackMode(); - // Caller must ensure that config_ is set before calling. const StringView GetEdsResourceName() const { if (xds_client_from_channel_ == nullptr) return server_name_; @@ -216,9 +179,6 @@ class EdsLb : public LoadBalancingPolicy { : xds_client_.get(); } - // Policy name (kXds or kEds). - const char* name_; - // Server name from target URI. std::string server_name_; @@ -251,26 +211,6 @@ class EdsLb : public LoadBalancingPolicy { // The latest state and picker returned from the child policy. grpc_connectivity_state child_state_; RefCountedPtr child_picker_; - - // Non-null iff we are in fallback mode. - OrphanablePtr fallback_policy_; - - // Whether the checks for fallback at startup are ALL pending. There are - // several cases where this can be reset: - // 1. The fallback timer fires, we enter fallback mode. - // 2. Before the fallback timer fires, the endpoint watcher reports an - // error, we enter fallback mode. - // 3. Before the fallback timer fires, if any child policy in the locality map - // becomes READY, we cancel the fallback timer. - bool fallback_at_startup_checks_pending_ = false; - // Timeout in milliseconds for before using fallback backend addresses. - // 0 means not using fallback. - const grpc_millis lb_fallback_timeout_ms_; - // The backend addresses from the resolver. - ServerAddressList fallback_backend_addresses_; - // Fallback timer. - grpc_timer lb_fallback_timer_; - grpc_closure lb_on_fallback_; }; // @@ -331,15 +271,6 @@ void EdsLb::Helper::UpdateState(grpc_connectivity_state state, eds_policy_->child_state_ = state; eds_policy_->child_picker_ = MakeRefCounted(std::move(picker)); - // If the new state is READY, cancel the fallback-at-startup checks. - if (state == GRPC_CHANNEL_READY) { - eds_policy_->MaybeCancelFallbackAtStartupChecks(); - eds_policy_->MaybeExitFallbackMode(); - } - // TODO(roth): If the child reports TRANSIENT_FAILURE and the - // fallback-at-startup checks are pending, we should probably go into - // fallback mode immediately (cancelling the fallback-at-startup timer - // if needed). // Wrap the picker in a DropPicker and pass it up. eds_policy_->MaybeUpdateDropPickerLocked(); } @@ -349,33 +280,6 @@ void EdsLb::Helper::AddTraceEvent(TraceSeverity severity, StringView message) { eds_policy_->channel_control_helper()->AddTraceEvent(severity, message); } -// -// EdsLb::FallbackHelper -// - -RefCountedPtr EdsLb::FallbackHelper::CreateSubchannel( - const grpc_channel_args& args) { - if (parent_->shutting_down_) return nullptr; - return parent_->channel_control_helper()->CreateSubchannel(args); -} - -void EdsLb::FallbackHelper::UpdateState( - grpc_connectivity_state state, std::unique_ptr picker) { - if (parent_->shutting_down_) return; - parent_->channel_control_helper()->UpdateState(state, std::move(picker)); -} - -void EdsLb::FallbackHelper::RequestReresolution() { - if (parent_->shutting_down_) return; - parent_->channel_control_helper()->RequestReresolution(); -} - -void EdsLb::FallbackHelper::AddTraceEvent(TraceSeverity severity, - StringView message) { - if (parent_->shutting_down_) return; - parent_->channel_control_helper()->AddTraceEvent(severity, message); -} - // // EdsLb::EndpointWatcher // @@ -392,9 +296,6 @@ class EdsLb::EndpointWatcher : public XdsClient::EndpointWatcherInterface { gpr_log(GPR_INFO, "[edslb %p] Received EDS update from xds client", eds_policy_.get()); } - // If the balancer tells us to drop all the calls, we should exit fallback - // mode immediately. - if (update.drop_config->drop_all()) eds_policy_->MaybeExitFallbackMode(); // Update the drop config. const bool drop_config_changed = eds_policy_->drop_config_ == nullptr || @@ -424,34 +325,18 @@ class EdsLb::EndpointWatcher : public XdsClient::EndpointWatcherInterface { } void OnError(grpc_error* error) override { - // If the fallback-at-startup checks are pending, go into fallback mode - // immediately. This short-circuits the timeout for the - // fallback-at-startup case. - if (eds_policy_->fallback_at_startup_checks_pending_) { - gpr_log(GPR_ERROR, - "[edslb %p] xds watcher reported error; entering fallback " - "mode: %s", - eds_policy_.get(), grpc_error_string(error)); - eds_policy_->fallback_at_startup_checks_pending_ = false; - grpc_timer_cancel(&eds_policy_->lb_fallback_timer_); - eds_policy_->UpdateFallbackPolicyLocked(); - // If the xds call failed, request re-resolution. - // TODO(roth): We check the error string contents here to - // differentiate between the xds call failing and the xds channel - // going into TRANSIENT_FAILURE. This is a pretty ugly hack, - // but it's okay for now, since we're not yet sure whether we will - // continue to support the current fallback functionality. If we - // decide to keep the fallback approach, then we should either - // find a cleaner way to expose the difference between these two - // cases or decide that we're okay re-resolving in both cases. - // Note that even if we do keep the current fallback functionality, - // this re-resolution will only be necessary if we are going to be - // using this LB policy with resolvers other than the xds resolver. - if (strstr(grpc_error_string(error), "xds call failed")) { - eds_policy_->channel_control_helper()->RequestReresolution(); - } + gpr_log(GPR_ERROR, "[edslb %p] xds watcher reported error: %s", + eds_policy_.get(), grpc_error_string(error)); + // Go into TRANSIENT_FAILURE if we have not yet created the child + // policy (i.e., we have not yet received data from xds). Otherwise, + // we keep running with the data we had previously. + if (eds_policy_->child_policy_ == nullptr) { + eds_policy_->channel_control_helper()->UpdateState( + GRPC_CHANNEL_TRANSIENT_FAILURE, + absl::make_unique(error)); + } else { + GRPC_ERROR_UNREF(error); } - GRPC_ERROR_UNREF(error); } private: @@ -462,13 +347,9 @@ class EdsLb::EndpointWatcher : public XdsClient::EndpointWatcherInterface { // EdsLb public methods // -EdsLb::EdsLb(const char* name, Args args) +EdsLb::EdsLb(Args args) : LoadBalancingPolicy(std::move(args)), - name_(name), - xds_client_from_channel_(XdsClient::GetFromChannelArgs(*args.args)), - lb_fallback_timeout_ms_(grpc_channel_args_find_integer( - args.args, GRPC_ARG_XDS_FALLBACK_TIMEOUT_MS, - {GRPC_EDS_DEFAULT_FALLBACK_TIMEOUT, 0, INT_MAX})) { + xds_client_from_channel_(XdsClient::GetFromChannelArgs(*args.args)) { if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { gpr_log(GPR_INFO, "[edslb %p] created -- xds client from channel: %p", this, xds_client_from_channel_.get()); @@ -499,7 +380,6 @@ void EdsLb::ShutdownLocked() { gpr_log(GPR_INFO, "[edslb %p] shutting down", this); } shutting_down_ = true; - MaybeCancelFallbackAtStartupChecks(); // Drop our ref to the child's picker, in case it's holding a ref to // the child. child_picker_.reset(); @@ -508,11 +388,6 @@ void EdsLb::ShutdownLocked() { interested_parties()); child_policy_.reset(); } - if (fallback_policy_ != nullptr) { - grpc_pollset_set_del_pollset_set(fallback_policy_->interested_parties(), - interested_parties()); - fallback_policy_.reset(); - } drop_stats_.reset(); // Cancel the endpoint watch here instead of in our dtor if we are using the // xds resolver, because the watcher holds a ref to us and we might not be @@ -540,15 +415,10 @@ void EdsLb::UpdateLocked(UpdateArgs args) { // Update config. auto old_config = std::move(config_); config_ = std::move(args.config); - // Update fallback address list. - fallback_backend_addresses_ = std::move(args.addresses); // Update args. grpc_channel_args_destroy(args_); args_ = args.args; args.args = nullptr; - // Update the existing fallback policy. The fallback policy config and/or the - // fallback addresses may be new. - if (fallback_policy_ != nullptr) UpdateFallbackPolicyLocked(); if (is_initial_update) { // Initialize XdsClient. if (xds_client_from_channel_ == nullptr) { @@ -556,7 +426,7 @@ void EdsLb::UpdateLocked(UpdateArgs args) { xds_client_ = MakeOrphanable( combiner(), interested_parties(), GetEdsResourceName(), nullptr /* service config watcher */, *args_, &error); - // TODO(roth): If we decide that we care about fallback mode, add + // TODO(roth): If we decide that we care about EDS-only mode, add // proper error handling here. GPR_ASSERT(error == GRPC_ERROR_NONE); if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { @@ -564,13 +434,6 @@ void EdsLb::UpdateLocked(UpdateArgs args) { xds_client_.get()); } } - // Start fallback-at-startup checks. - grpc_millis deadline = ExecCtx::Get()->Now() + lb_fallback_timeout_ms_; - Ref(DEBUG_LOCATION, "on_fallback_timer").release(); // Held by closure - GRPC_CLOSURE_INIT(&lb_on_fallback_, &EdsLb::OnFallbackTimer, this, - grpc_schedule_on_exec_ctx); - fallback_at_startup_checks_pending_ = true; - grpc_timer_init(&lb_fallback_timer_, deadline, &lb_on_fallback_); } // Update drop stats for load reporting if needed. if (is_initial_update || config_->lrs_load_reporting_server_name() != @@ -609,9 +472,6 @@ void EdsLb::ResetBackoffLocked() { if (child_policy_ != nullptr) { child_policy_->ResetBackoffLocked(); } - if (fallback_policy_ != nullptr) { - fallback_policy_->ResetBackoffLocked(); - } } // @@ -875,8 +735,6 @@ OrphanablePtr EdsLb::CreateChildPolicyLocked( } void EdsLb::MaybeUpdateDropPickerLocked() { - // If we are in fallback mode, don't override the picker. - if (fallback_policy_ != nullptr) return; // If we're dropping all calls, report READY, regardless of what (or // whether) the child has reported. if (drop_config_ != nullptr && drop_config_->drop_all()) { @@ -891,111 +749,24 @@ void EdsLb::MaybeUpdateDropPickerLocked() { } } -// -// fallback-related methods -// - -void EdsLb::MaybeCancelFallbackAtStartupChecks() { - if (!fallback_at_startup_checks_pending_) return; - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { - gpr_log(GPR_INFO, "[edslb %p] Cancelling fallback timer", this); - } - grpc_timer_cancel(&lb_fallback_timer_); - fallback_at_startup_checks_pending_ = false; -} - -void EdsLb::OnFallbackTimer(void* arg, grpc_error* error) { - EdsLb* edslb_policy = static_cast(arg); - edslb_policy->combiner()->Run( - GRPC_CLOSURE_INIT(&edslb_policy->lb_on_fallback_, - &EdsLb::OnFallbackTimerLocked, edslb_policy, nullptr), - GRPC_ERROR_REF(error)); -} - -void EdsLb::OnFallbackTimerLocked(void* arg, grpc_error* error) { - EdsLb* edslb_policy = static_cast(arg); - // If some fallback-at-startup check is done after the timer fires but before - // this callback actually runs, don't fall back. - if (edslb_policy->fallback_at_startup_checks_pending_ && - !edslb_policy->shutting_down_ && error == GRPC_ERROR_NONE) { - gpr_log(GPR_INFO, - "[edslb %p] Child policy not ready after fallback timeout; " - "entering fallback mode", - edslb_policy); - edslb_policy->fallback_at_startup_checks_pending_ = false; - edslb_policy->UpdateFallbackPolicyLocked(); - } - edslb_policy->Unref(DEBUG_LOCATION, "on_fallback_timer"); -} - -void EdsLb::UpdateFallbackPolicyLocked() { - if (shutting_down_) return; - // Create policy if needed. - if (fallback_policy_ == nullptr) { - fallback_policy_ = CreateFallbackPolicyLocked(args_); - } - // Construct update args. - UpdateArgs update_args; - update_args.addresses = fallback_backend_addresses_; - update_args.config = config_->fallback_policy(); - update_args.args = grpc_channel_args_copy(args_); - // Update the policy. - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { - gpr_log(GPR_INFO, "[edslb %p] Updating fallback child policy handler %p", - this, fallback_policy_.get()); - } - fallback_policy_->UpdateLocked(std::move(update_args)); -} - -OrphanablePtr EdsLb::CreateFallbackPolicyLocked( - const grpc_channel_args* args) { - LoadBalancingPolicy::Args lb_policy_args; - lb_policy_args.combiner = combiner(); - lb_policy_args.args = args; - lb_policy_args.channel_control_helper = - absl::make_unique(Ref(DEBUG_LOCATION, "FallbackHelper")); - OrphanablePtr lb_policy = - MakeOrphanable(std::move(lb_policy_args), - &grpc_lb_eds_trace); - if (GRPC_TRACE_FLAG_ENABLED(grpc_lb_eds_trace)) { - gpr_log(GPR_INFO, "[edslb %p] Created new fallback child policy handler %p", - this, lb_policy.get()); - } - // Add our interested_parties pollset_set to that of the newly created - // child policy. This will make the child policy progress upon activity on - // this policy, which in turn is tied to the application's call. - grpc_pollset_set_add_pollset_set(lb_policy->interested_parties(), - interested_parties()); - return lb_policy; -} - -void EdsLb::MaybeExitFallbackMode() { - if (fallback_policy_ == nullptr) return; - gpr_log(GPR_INFO, "[edslb %p] Exiting fallback mode", this); - fallback_policy_.reset(); -} - // // factory // class EdsLbFactory : public LoadBalancingPolicyFactory { public: - explicit EdsLbFactory(const char* name) : name_(name) {} - OrphanablePtr CreateLoadBalancingPolicy( LoadBalancingPolicy::Args args) const override { - return MakeOrphanable(std::move(args), &grpc_lb_eds_trace, - name_); + return MakeOrphanable(std::move(args), &grpc_lb_eds_trace); } - const char* name() const override { return name_; } + const char* name() const override { return kEds; } RefCountedPtr ParseLoadBalancingConfig( const Json& json, grpc_error** error) const override { GPR_DEBUG_ASSERT(error != nullptr && *error == GRPC_ERROR_NONE); if (json.type() == Json::Type::JSON_NULL) { - // xds was mentioned as a policy in the deprecated loadBalancingPolicy + // eds was mentioned as a policy in the deprecated loadBalancingPolicy // field or in the client API. *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( "field:loadBalancingPolicy error:eds policy requires configuration. " @@ -1016,21 +787,15 @@ class EdsLbFactory : public LoadBalancingPolicyFactory { } // Cluster name. std::string cluster_name; - if (name_ == kEds) { - it = json.object_value().find("clusterName"); - if (it == json.object_value().end()) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "field:clusterName error:required field missing")); - } else if (it->second.type() != Json::Type::STRING) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "field:clusterName error:type should be string")); - } else { - cluster_name = it->second.string_value(); - } + it = json.object_value().find("clusterName"); + if (it == json.object_value().end()) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:clusterName error:required field missing")); + } else if (it->second.type() != Json::Type::STRING) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "field:clusterName error:type should be string")); } else { - // For xds policy, this field does not exist in the config, so it - // will always be set to the same value as edsServiceName. - cluster_name = eds_service_name; + cluster_name = it->second.string_value(); } // LRS load reporting server name. absl::optional lrs_load_reporting_server_name; @@ -1043,20 +808,20 @@ class EdsLbFactory : public LoadBalancingPolicyFactory { lrs_load_reporting_server_name.emplace(it->second.string_value()); } } - // Locality-picking policy. Not supported for xds policy. - Json locality_picking_policy = Json::Array{ - Json::Object{ - {"weighted_target_experimental", - Json::Object{ - {"targets", Json::Object()}, - }}, - }, - }; - if (name_ == kEds) { - it = json.object_value().find("localityPickingPolicy"); - if (it != json.object_value().end()) { - locality_picking_policy = it->second; - } + // Locality-picking policy. + Json locality_picking_policy; + it = json.object_value().find("localityPickingPolicy"); + if (it == json.object_value().end()) { + locality_picking_policy = Json::Array{ + Json::Object{ + {"weighted_target_experimental", + Json::Object{ + {"targets", Json::Object()}, + }}, + }, + }; + } else { + locality_picking_policy = it->second; } grpc_error* parse_error = GRPC_ERROR_NONE; if (LoadBalancingPolicyRegistry::ParseLoadBalancingConfig( @@ -1067,10 +832,8 @@ class EdsLbFactory : public LoadBalancingPolicyFactory { GRPC_ERROR_UNREF(parse_error); } // Endpoint-picking policy. Called "childPolicy" for xds policy. - const char* field_name = - name_ == kEds ? "endpointPickingPolicy" : "childPolicy"; Json endpoint_picking_policy; - it = json.object_value().find(field_name); + it = json.object_value().find("endpointPickingPolicy"); if (it == json.object_value().end()) { endpoint_picking_policy = Json::Array{ Json::Object{ @@ -1085,36 +848,16 @@ class EdsLbFactory : public LoadBalancingPolicyFactory { endpoint_picking_policy, &parse_error) == nullptr) { GPR_DEBUG_ASSERT(parse_error != GRPC_ERROR_NONE); error_list.push_back(GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( - field_name, &parse_error, 1)); - GRPC_ERROR_UNREF(parse_error); - } - // Fallback policy. - Json fallback_policy_config; - it = json.object_value().find("fallbackPolicy"); - if (it == json.object_value().end()) { - fallback_policy_config = Json::Array{Json::Object{ - {"round_robin", Json::Object()}, - }}; - } else { - fallback_policy_config = it->second; - } - parse_error = GRPC_ERROR_NONE; - RefCountedPtr fallback_policy = - LoadBalancingPolicyRegistry::ParseLoadBalancingConfig( - fallback_policy_config, &parse_error); - if (fallback_policy == nullptr) { - GPR_DEBUG_ASSERT(parse_error != GRPC_ERROR_NONE); - error_list.push_back(GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( - "fallbackPolicy", &parse_error, 1)); + "endpointPickingPolicy", &parse_error, 1)); GRPC_ERROR_UNREF(parse_error); - error_list.push_back(parse_error); } + // Construct config. if (error_list.empty()) { return MakeRefCounted( - name_, std::move(cluster_name), std::move(eds_service_name), + std::move(cluster_name), std::move(eds_service_name), std::move(lrs_load_reporting_server_name), std::move(locality_picking_policy), - std::move(endpoint_picking_policy), std::move(fallback_policy)); + std::move(endpoint_picking_policy)); } else { *error = GRPC_ERROR_CREATE_FROM_VECTOR( "eds_experimental LB policy config", &error_list); @@ -1125,14 +868,14 @@ class EdsLbFactory : public LoadBalancingPolicyFactory { private: class EdsChildHandler : public ChildPolicyHandler { public: - EdsChildHandler(Args args, TraceFlag* tracer, const char* name) - : ChildPolicyHandler(std::move(args), tracer), name_(name) {} + EdsChildHandler(Args args, TraceFlag* tracer) + : ChildPolicyHandler(std::move(args), tracer) {} bool ConfigChangeRequiresNewPolicyInstance( LoadBalancingPolicy::Config* old_config, LoadBalancingPolicy::Config* new_config) const override { - GPR_ASSERT(old_config->name() == name_); - GPR_ASSERT(new_config->name() == name_); + GPR_ASSERT(old_config->name() == kEds); + GPR_ASSERT(new_config->name() == kEds); EdsLbConfig* old_eds_config = static_cast(old_config); EdsLbConfig* new_eds_config = static_cast(new_config); return old_eds_config->cluster_name() != new_eds_config->cluster_name() || @@ -1142,14 +885,9 @@ class EdsLbFactory : public LoadBalancingPolicyFactory { OrphanablePtr CreateLoadBalancingPolicy( const char* name, LoadBalancingPolicy::Args args) const override { - return MakeOrphanable(name_, std::move(args)); + return MakeOrphanable(std::move(args)); } - - private: - const char* name_; }; - - const char* name_; }; } // namespace @@ -1163,13 +901,7 @@ class EdsLbFactory : public LoadBalancingPolicyFactory { void grpc_lb_policy_eds_init() { grpc_core::LoadBalancingPolicyRegistry::Builder:: RegisterLoadBalancingPolicyFactory( - absl::make_unique(grpc_core::kEds)); - // TODO(roth): This is here just for backward compatibility with some - // old tests we have internally. Remove this once they are upgraded - // to use the new policy name and config. - grpc_core::LoadBalancingPolicyRegistry::Builder:: - RegisterLoadBalancingPolicyFactory( - absl::make_unique(grpc_core::kXds)); + absl::make_unique()); } void grpc_lb_policy_eds_shutdown() {} diff --git a/test/cpp/end2end/xds_end2end_test.cc b/test/cpp/end2end/xds_end2end_test.cc index 2e643a43670..f14d7c0a583 100644 --- a/test/cpp/end2end/xds_end2end_test.cc +++ b/test/cpp/end2end/xds_end2end_test.cc @@ -1144,14 +1144,10 @@ class XdsEnd2endTest : public ::testing::TestWithParam { void ShutdownBackend(size_t index) { backends_[index]->Shutdown(); } - void ResetStub(int fallback_timeout = 0, int failover_timeout = 0, + void ResetStub(int failover_timeout = 0, const grpc::string& expected_targets = "", int xds_resource_does_not_exist_timeout = 0) { ChannelArguments args; - // TODO(juanlishen): Add setter to ChannelArguments. - if (fallback_timeout > 0) { - args.SetInt(GRPC_ARG_XDS_FALLBACK_TIMEOUT_MS, fallback_timeout); - } if (failover_timeout > 0) { args.SetInt(GRPC_ARG_PRIORITY_FAILOVER_TIMEOUT_MS, failover_timeout); } @@ -1285,8 +1281,8 @@ class XdsEnd2endTest : public ::testing::TestWithParam { : kDefaultServiceConfigWithoutLoadReporting_; result.service_config = grpc_core::ServiceConfig::Create(service_config_json, &error); - ASSERT_NE(result.service_config.get(), nullptr); ASSERT_EQ(error, GRPC_ERROR_NONE) << grpc_error_string(error); + ASSERT_NE(result.service_config.get(), nullptr); grpc_arg arg = grpc_core::FakeResolverResponseGenerator::MakeChannelArg( lb_channel_response_generator == nullptr ? lb_channel_response_generator_.get() @@ -1519,7 +1515,8 @@ class XdsEnd2endTest : public ::testing::TestWithParam { "{\n" " \"loadBalancingConfig\":[\n" " { \"does_not_exist\":{} },\n" - " { \"xds_experimental\":{\n" + " { \"eds_experimental\":{\n" + " \"clusterName\": \"application_target_name\",\n" " \"lrsLoadReportingServerName\": \"\"\n" " } }\n" " ]\n" @@ -1528,7 +1525,8 @@ class XdsEnd2endTest : public ::testing::TestWithParam { "{\n" " \"loadBalancingConfig\":[\n" " { \"does_not_exist\":{} },\n" - " { \"xds_experimental\":{\n" + " { \"eds_experimental\":{\n" + " \"clusterName\": \"application_target_name\"\n" " } }\n" " ]\n" "}"; @@ -1563,7 +1561,7 @@ TEST_P(BasicTest, Vanilla) { } // Check LB policy name for the channel. EXPECT_EQ( - (GetParam().use_xds_resolver() ? "cds_experimental" : "xds_experimental"), + (GetParam().use_xds_resolver() ? "cds_experimental" : "eds_experimental"), channel_->GetLoadBalancingPolicyName()); } @@ -1941,7 +1939,7 @@ using SecureNamingTest = BasicTest; // Tests that secure naming check passes if target name is expected. TEST_P(SecureNamingTest, TargetNameIsExpected) { // TODO(juanlishen): Use separate fake creds for the balancer channel. - ResetStub(0, 0, kApplicationTargetName_ + ";lb"); + ResetStub(0, kApplicationTargetName_ + ";lb"); SetNextResolution({}); SetNextResolutionForLbChannel({balancers_[0]->port()}); const size_t kNumRpcsPerAddress = 100; @@ -1971,7 +1969,7 @@ TEST_P(SecureNamingTest, TargetNameIsUnexpected) { // the name from the balancer doesn't match expectations. ASSERT_DEATH_IF_SUPPORTED( { - ResetStub(0, 0, kApplicationTargetName_ + ";lb"); + ResetStub(0, kApplicationTargetName_ + ";lb"); SetNextResolution({}); SetNextResolutionForLbChannel({balancers_[0]->port()}); channel_->WaitForConnected(grpc_timeout_seconds_to_deadline(1)); @@ -2130,7 +2128,7 @@ TEST_P(LdsTest, RouteActionHasNoCluster) { // Tests that LDS client times out when no response received. TEST_P(LdsTest, Timeout) { - ResetStub(0, 0, "", 500); + ResetStub(0, "", 500); balancers_[0]->ads_service()->SetResourceIgnore(kLdsTypeUrl); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); @@ -2265,7 +2263,7 @@ TEST_P(RdsTest, RouteActionHasNoCluster) { // Tests that RDS client times out when no response received. TEST_P(RdsTest, Timeout) { - ResetStub(0, 0, "", 500); + ResetStub(0, "", 500); balancers_[0]->ads_service()->SetResourceIgnore(kRdsTypeUrl); balancers_[0]->ads_service()->SetLdsToUseDynamicRds(); SetNextResolution({}); @@ -2338,7 +2336,7 @@ TEST_P(CdsTest, WrongLrsServer) { // Tests that CDS client times out when no response received. TEST_P(CdsTest, Timeout) { - ResetStub(0, 0, "", 500); + ResetStub(0, "", 500); balancers_[0]->ads_service()->SetResourceIgnore(kCdsTypeUrl); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); @@ -2348,7 +2346,7 @@ TEST_P(CdsTest, Timeout) { using EdsTest = BasicTest; TEST_P(EdsTest, Timeout) { - ResetStub(0, 0, "", 500); + ResetStub(0, "", 500); balancers_[0]->ads_service()->SetResourceIgnore(kEdsTypeUrl); SetNextResolution({}); SetNextResolutionForLbChannelAllBalancers(); @@ -2608,7 +2606,7 @@ class FailoverTest : public BasicTest { public: void SetUp() override { BasicTest::SetUp(); - ResetStub(0, 100, ""); + ResetStub(100, ""); } }; @@ -3045,241 +3043,6 @@ TEST_P(DropTest, DropAll) { } } -using FallbackTest = BasicTest; - -// Tests that RPCs are handled by the fallback backends before the serverlist is -// received, but will be handled by the serverlist after it's received. -TEST_P(FallbackTest, Vanilla) { - const int kFallbackTimeoutMs = 200 * grpc_test_slowdown_factor(); - const int kServerlistDelayMs = 500 * grpc_test_slowdown_factor(); - const size_t kNumBackendsInResolution = backends_.size() / 2; - ResetStub(kFallbackTimeoutMs); - SetNextResolution(GetBackendPorts(0, kNumBackendsInResolution)); - SetNextResolutionForLbChannelAllBalancers(); - // Send non-empty serverlist only after kServerlistDelayMs. - AdsServiceImpl::EdsResourceArgs args({ - {"locality0", GetBackendPorts(kNumBackendsInResolution)}, - }); - std::thread delayed_resource_setter( - std::bind(&BasicTest::SetEdsResourceWithDelay, this, 0, - AdsServiceImpl::BuildEdsResource(args), kServerlistDelayMs, - kDefaultResourceName)); - // Wait until all the fallback backends are reachable. - WaitForAllBackends(0 /* start_index */, - kNumBackendsInResolution /* stop_index */); - gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH =========="); - CheckRpcSendOk(kNumBackendsInResolution); - gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH =========="); - // Fallback is used: each backend returned by the resolver should have - // gotten one request. - for (size_t i = 0; i < kNumBackendsInResolution; ++i) { - EXPECT_EQ(1U, backends_[i]->backend_service()->request_count()); - } - for (size_t i = kNumBackendsInResolution; i < backends_.size(); ++i) { - EXPECT_EQ(0U, backends_[i]->backend_service()->request_count()); - } - // Wait until the serverlist reception has been processed and all backends - // in the serverlist are reachable. - WaitForAllBackends(kNumBackendsInResolution /* start_index */); - gpr_log(GPR_INFO, "========= BEFORE SECOND BATCH =========="); - CheckRpcSendOk(backends_.size() - kNumBackendsInResolution); - gpr_log(GPR_INFO, "========= DONE WITH SECOND BATCH =========="); - // Serverlist is used: each backend returned by the balancer should - // have gotten one request. - for (size_t i = 0; i < kNumBackendsInResolution; ++i) { - EXPECT_EQ(0U, backends_[i]->backend_service()->request_count()); - } - for (size_t i = kNumBackendsInResolution; i < backends_.size(); ++i) { - EXPECT_EQ(1U, backends_[i]->backend_service()->request_count()); - } - delayed_resource_setter.join(); -} - -// Tests that RPCs are handled by the updated fallback backends before -// serverlist is received, -TEST_P(FallbackTest, Update) { - const int kFallbackTimeoutMs = 200 * grpc_test_slowdown_factor(); - const int kServerlistDelayMs = 500 * grpc_test_slowdown_factor(); - const size_t kNumBackendsInResolution = backends_.size() / 3; - const size_t kNumBackendsInResolutionUpdate = backends_.size() / 3; - ResetStub(kFallbackTimeoutMs); - SetNextResolution(GetBackendPorts(0, kNumBackendsInResolution)); - SetNextResolutionForLbChannelAllBalancers(); - // Send non-empty serverlist only after kServerlistDelayMs. - AdsServiceImpl::EdsResourceArgs args({ - {"locality0", GetBackendPorts(kNumBackendsInResolution + - kNumBackendsInResolutionUpdate)}, - }); - std::thread delayed_resource_setter( - std::bind(&BasicTest::SetEdsResourceWithDelay, this, 0, - AdsServiceImpl::BuildEdsResource(args), kServerlistDelayMs, - kDefaultResourceName)); - // Wait until all the fallback backends are reachable. - WaitForAllBackends(0 /* start_index */, - kNumBackendsInResolution /* stop_index */); - gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH =========="); - CheckRpcSendOk(kNumBackendsInResolution); - gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH =========="); - // Fallback is used: each backend returned by the resolver should have - // gotten one request. - for (size_t i = 0; i < kNumBackendsInResolution; ++i) { - EXPECT_EQ(1U, backends_[i]->backend_service()->request_count()); - } - for (size_t i = kNumBackendsInResolution; i < backends_.size(); ++i) { - EXPECT_EQ(0U, backends_[i]->backend_service()->request_count()); - } - SetNextResolution(GetBackendPorts( - kNumBackendsInResolution, - kNumBackendsInResolution + kNumBackendsInResolutionUpdate)); - // Wait until the resolution update has been processed and all the new - // fallback backends are reachable. - WaitForAllBackends(kNumBackendsInResolution /* start_index */, - kNumBackendsInResolution + - kNumBackendsInResolutionUpdate /* stop_index */); - gpr_log(GPR_INFO, "========= BEFORE SECOND BATCH =========="); - CheckRpcSendOk(kNumBackendsInResolutionUpdate); - gpr_log(GPR_INFO, "========= DONE WITH SECOND BATCH =========="); - // The resolution update is used: each backend in the resolution update should - // have gotten one request. - for (size_t i = 0; i < kNumBackendsInResolution; ++i) { - EXPECT_EQ(0U, backends_[i]->backend_service()->request_count()); - } - for (size_t i = kNumBackendsInResolution; - i < kNumBackendsInResolution + kNumBackendsInResolutionUpdate; ++i) { - EXPECT_EQ(1U, backends_[i]->backend_service()->request_count()); - } - for (size_t i = kNumBackendsInResolution + kNumBackendsInResolutionUpdate; - i < backends_.size(); ++i) { - EXPECT_EQ(0U, backends_[i]->backend_service()->request_count()); - } - // Wait until the serverlist reception has been processed and all backends - // in the serverlist are reachable. - WaitForAllBackends(kNumBackendsInResolution + - kNumBackendsInResolutionUpdate /* start_index */); - gpr_log(GPR_INFO, "========= BEFORE THIRD BATCH =========="); - CheckRpcSendOk(backends_.size() - kNumBackendsInResolution - - kNumBackendsInResolutionUpdate); - gpr_log(GPR_INFO, "========= DONE WITH THIRD BATCH =========="); - // Serverlist is used: each backend returned by the balancer should - // have gotten one request. - for (size_t i = 0; - i < kNumBackendsInResolution + kNumBackendsInResolutionUpdate; ++i) { - EXPECT_EQ(0U, backends_[i]->backend_service()->request_count()); - } - for (size_t i = kNumBackendsInResolution + kNumBackendsInResolutionUpdate; - i < backends_.size(); ++i) { - EXPECT_EQ(1U, backends_[i]->backend_service()->request_count()); - } - delayed_resource_setter.join(); -} - -// Tests that fallback will kick in immediately if the balancer channel fails. -TEST_P(FallbackTest, FallbackEarlyWhenBalancerChannelFails) { - const int kFallbackTimeoutMs = 10000 * grpc_test_slowdown_factor(); - ResetStub(kFallbackTimeoutMs); - // Return an unreachable balancer and one fallback backend. - SetNextResolution({backends_[0]->port()}); - SetNextResolutionForLbChannel({g_port_saver->GetPort()}); - // Send RPC with deadline less than the fallback timeout and make sure it - // succeeds. - CheckRpcSendOk(/* times */ 1, /* timeout_ms */ 1000, - /* wait_for_ready */ false); -} - -// Tests that fallback will kick in immediately if the balancer call fails. -TEST_P(FallbackTest, FallbackEarlyWhenBalancerCallFails) { - const int kFallbackTimeoutMs = 10000 * grpc_test_slowdown_factor(); - ResetStub(kFallbackTimeoutMs); - // Return one balancer and one fallback backend. - SetNextResolution({backends_[0]->port()}); - SetNextResolutionForLbChannelAllBalancers(); - // Balancer drops call without sending a serverlist. - balancers_[0]->ads_service()->NotifyDoneWithAdsCall(); - // Send RPC with deadline less than the fallback timeout and make sure it - // succeeds. - CheckRpcSendOk(/* times */ 1, /* timeout_ms */ 1000, - /* wait_for_ready */ false); -} - -// Tests that fallback mode is entered if balancer response is received but the -// backends can't be reached. -TEST_P(FallbackTest, FallbackIfResponseReceivedButChildNotReady) { - const int kFallbackTimeoutMs = 500 * grpc_test_slowdown_factor(); - ResetStub(kFallbackTimeoutMs); - SetNextResolution({backends_[0]->port()}); - SetNextResolutionForLbChannelAllBalancers(); - // Send a serverlist that only contains an unreachable backend before fallback - // timeout. - AdsServiceImpl::EdsResourceArgs args({ - {"locality0", {g_port_saver->GetPort()}}, - }); - balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); - // Because no child policy is ready before fallback timeout, we enter fallback - // mode. - WaitForBackend(0); -} - -// Tests that fallback mode is exited if the balancer tells the client to drop -// all the calls. -TEST_P(FallbackTest, FallbackModeIsExitedWhenBalancerSaysToDropAllCalls) { - // Return an unreachable balancer and one fallback backend. - SetNextResolution({backends_[0]->port()}); - SetNextResolutionForLbChannel({g_port_saver->GetPort()}); - // Enter fallback mode because the LB channel fails to connect. - WaitForBackend(0); - // Return a new balancer that sends a response to drop all calls. - AdsServiceImpl::EdsResourceArgs args({ - {"locality0", GetBackendPorts()}, - }); - args.drop_categories = {{kLbDropType, 1000000}}; - balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); - SetNextResolutionForLbChannelAllBalancers(); - // Send RPCs until failure. - gpr_timespec deadline = gpr_time_add( - gpr_now(GPR_CLOCK_REALTIME), gpr_time_from_millis(5000, GPR_TIMESPAN)); - do { - auto status = SendRpc(); - if (!status.ok()) break; - } while (gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), deadline) < 0); - CheckRpcSendFailure(); -} - -// Tests that fallback mode is exited if the child policy becomes ready. -TEST_P(FallbackTest, FallbackModeIsExitedAfterChildReady) { - // Return an unreachable balancer and one fallback backend. - SetNextResolution({backends_[0]->port()}); - SetNextResolutionForLbChannel({g_port_saver->GetPort()}); - // Enter fallback mode because the LB channel fails to connect. - WaitForBackend(0); - // Return a new balancer that sends a dead backend. - ShutdownBackend(1); - AdsServiceImpl::EdsResourceArgs args({ - {"locality0", {backends_[1]->port()}}, - }); - balancers_[0]->ads_service()->SetEdsResource( - AdsServiceImpl::BuildEdsResource(args), kDefaultResourceName); - SetNextResolutionForLbChannelAllBalancers(); - // The state (TRANSIENT_FAILURE) update from the child policy will be ignored - // because we are still in fallback mode. - gpr_timespec deadline = gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), - gpr_time_from_millis(500, GPR_TIMESPAN)); - // Send 0.5 second worth of RPCs. - do { - CheckRpcSendOk(); - } while (gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), deadline) < 0); - // After the backend is restarted, the child policy will eventually be READY, - // and we will exit fallback mode. - StartBackend(1); - WaitForBackend(1); - // We have exited fallback mode, so calls will go to the child policy - // exclusively. - CheckRpcSendOk(100); - EXPECT_EQ(0U, backends_[0]->backend_service()->request_count()); - EXPECT_EQ(100U, backends_[1]->backend_service()->request_count()); -} - class BalancerUpdateTest : public XdsEnd2endTest { public: BalancerUpdateTest() : XdsEnd2endTest(4, 3) {} @@ -3782,12 +3545,6 @@ INSTANTIATE_TEST_SUITE_P(XdsTest, DropTest, TestType(true, true)), &TestTypeName); -// Fallback does not work with xds resolver. -INSTANTIATE_TEST_SUITE_P(XdsTest, FallbackTest, - ::testing::Values(TestType(false, true), - TestType(false, false)), - &TestTypeName); - INSTANTIATE_TEST_SUITE_P(XdsTest, BalancerUpdateTest, ::testing::Values(TestType(false, true), TestType(false, false), From 317a55dd7ee99f667112a95e89d54a2c0133b743 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Mon, 13 Apr 2020 11:47:15 -0700 Subject: [PATCH 66/73] Use correct status code type --- test/cpp/end2end/xds_end2end_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/end2end/xds_end2end_test.cc b/test/cpp/end2end/xds_end2end_test.cc index 2e643a43670..1a597d57dc9 100644 --- a/test/cpp/end2end/xds_end2end_test.cc +++ b/test/cpp/end2end/xds_end2end_test.cc @@ -2450,7 +2450,7 @@ TEST_P(LocalityMapTest, NoLocalities) { AdsServiceImpl::BuildEdsResource({}), kDefaultResourceName); Status status = SendRpc(); EXPECT_FALSE(status.ok()); - EXPECT_EQ(status.error_code(), GRPC_STATUS_UNAVAILABLE); + EXPECT_EQ(status.error_code(), StatusCode::UNAVAILABLE); } // Tests that the locality map can work properly even when it contains a large From c95c2102c66d21372923a192b6f089eab3f8a2a3 Mon Sep 17 00:00:00 2001 From: Patrice Chalin Date: Mon, 13 Apr 2020 15:04:23 -0400 Subject: [PATCH 67/73] Delete cpptutorial.md --- examples/cpp/cpptutorial.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 examples/cpp/cpptutorial.md diff --git a/examples/cpp/cpptutorial.md b/examples/cpp/cpptutorial.md deleted file mode 100644 index 21411f9db98..00000000000 --- a/examples/cpp/cpptutorial.md +++ /dev/null @@ -1 +0,0 @@ -The content of this page has moved to [gRPC Basics - C++](https://grpc.io/docs/tutorials/basic/cpp). From d7a62868c8704e11fd13ad92bfb28e596c668568 Mon Sep 17 00:00:00 2001 From: "Mark D. Roth" Date: Tue, 14 Apr 2020 09:04:09 -0700 Subject: [PATCH 68/73] Mark client_channel_stress_test as manual. --- test/cpp/client/BUILD | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/test/cpp/client/BUILD b/test/cpp/client/BUILD index 18e9ce4973f..ce0f6d39f9f 100644 --- a/test/cpp/client/BUILD +++ b/test/cpp/client/BUILD @@ -35,18 +35,16 @@ grpc_cc_test( grpc_cc_test( name = "client_channel_stress_test", srcs = ["client_channel_stress_test.cc"], - flaky = True, # TODO(b/153136407) # TODO(jtattermusch): test fails frequently on Win RBE, but passes locally # reenable the tests once it works reliably on Win RBE. - # TODO(roth): Test disabled on msan and tsan due to variable - # duration problem triggered by https://github.com/grpc/grpc/pull/22481. - # Re-enable once the problem is diagnosed and fixed. Tracked - # internally in b/153136407. + # TODO(roth): Test marked as manual for now due to variable duration + # problem triggered by https://github.com/grpc/grpc/pull/22481. + # Once we figure out the problem, either re-enable or just decide to + # remove this test. Tracked internally in b/153136407. tags = [ + "manual", "no_test_android", # fails on android due to "Too many open files". "no_windows", - "nomsan", - "notsan", ], deps = [ "//:gpr", From e8d47d31de4c7a818b437a60013e80d67f0ea0ac Mon Sep 17 00:00:00 2001 From: ZhenLian Date: Wed, 8 Apr 2020 21:35:13 -0700 Subject: [PATCH 69/73] [3/n] Avoid using hardcoded test credentials --- test/core/end2end/fixtures/h2_oauth2.cc | 2 - test/core/end2end/fixtures/h2_ssl.cc | 3 +- .../end2end/fixtures/h2_ssl_cred_reload.cc | 51 +++++++------- test/core/end2end/fixtures/h2_ssl_proxy.cc | 62 +++++++++------- test/core/end2end/fixtures/h2_tls.cc | 70 ++++++++++++------- test/core/end2end/generate_tests.bzl | 4 -- 6 files changed, 107 insertions(+), 85 deletions(-) diff --git a/test/core/end2end/fixtures/h2_oauth2.cc b/test/core/end2end/fixtures/h2_oauth2.cc index 22f85a2feea..13a9e1c483b 100644 --- a/test/core/end2end/fixtures/h2_oauth2.cc +++ b/test/core/end2end/fixtures/h2_oauth2.cc @@ -31,8 +31,6 @@ #include "test/core/util/test_config.h" #define CA_CERT_PATH "src/core/tsi/test_creds/ca.pem" -#define CLIENT_CERT_PATH "src/core/tsi/test_creds/client.pem" -#define CLIENT_KEY_PATH "src/core/tsi/test_creds/client.key" #define SERVER_CERT_PATH "src/core/tsi/test_creds/server1.pem" #define SERVER_KEY_PATH "src/core/tsi/test_creds/server1.key" diff --git a/test/core/end2end/fixtures/h2_ssl.cc b/test/core/end2end/fixtures/h2_ssl.cc index 1e46486dd5e..9cb9aaf881a 100644 --- a/test/core/end2end/fixtures/h2_ssl.cc +++ b/test/core/end2end/fixtures/h2_ssl.cc @@ -31,9 +31,8 @@ #include "test/core/end2end/end2end_tests.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" + #define CA_CERT_PATH "src/core/tsi/test_creds/ca.pem" -#define CLIENT_CERT_PATH "src/core/tsi/test_creds/client.pem" -#define CLIENT_KEY_PATH "src/core/tsi/test_creds/client.key" #define SERVER_CERT_PATH "src/core/tsi/test_creds/server1.pem" #define SERVER_KEY_PATH "src/core/tsi/test_creds/server1.key" diff --git a/test/core/end2end/fixtures/h2_ssl_cred_reload.cc b/test/core/end2end/fixtures/h2_ssl_cred_reload.cc index 97600990230..589f2fd3d92 100644 --- a/test/core/end2end/fixtures/h2_ssl_cred_reload.cc +++ b/test/core/end2end/fixtures/h2_ssl_cred_reload.cc @@ -16,24 +16,26 @@ * */ -#include "test/core/end2end/end2end_tests.h" - -#include -#include - #include #include +#include +#include #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/gpr/string.h" #include "src/core/lib/gpr/tmpfile.h" #include "src/core/lib/gprpp/host_port.h" +#include "src/core/lib/iomgr/load_file.h" #include "src/core/lib/security/credentials/credentials.h" #include "src/core/lib/security/security_connector/ssl_utils_config.h" -#include "test/core/end2end/data/ssl_test_data.h" +#include "test/core/end2end/end2end_tests.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" +#define CA_CERT_PATH "src/core/tsi/test_creds/ca.pem" +#define SERVER_CERT_PATH "src/core/tsi/test_creds/server1.pem" +#define SERVER_KEY_PATH "src/core/tsi/test_creds/server1.key" + struct fullstack_secure_fixture_data { grpc_core::UniquePtr localaddr; bool server_credential_reloaded = false; @@ -48,10 +50,25 @@ ssl_server_certificate_config_callback( fullstack_secure_fixture_data* ffd = static_cast(user_data); if (!ffd->server_credential_reloaded) { - grpc_ssl_pem_key_cert_pair pem_key_cert_pair = {test_server1_key, - test_server1_cert}; - *config = grpc_ssl_server_certificate_config_create(test_root_cert, + grpc_slice ca_slice, cert_slice, key_slice; + GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", + grpc_load_file(CA_CERT_PATH, 1, &ca_slice))); + GPR_ASSERT(GRPC_LOG_IF_ERROR( + "load_file", grpc_load_file(SERVER_CERT_PATH, 1, &cert_slice))); + GPR_ASSERT(GRPC_LOG_IF_ERROR( + "load_file", grpc_load_file(SERVER_KEY_PATH, 1, &key_slice))); + const char* ca_cert = + reinterpret_cast GRPC_SLICE_START_PTR(ca_slice); + const char* server_cert = + reinterpret_cast GRPC_SLICE_START_PTR(cert_slice); + const char* server_key = + reinterpret_cast GRPC_SLICE_START_PTR(key_slice); + grpc_ssl_pem_key_cert_pair pem_key_cert_pair = {server_key, server_cert}; + *config = grpc_ssl_server_certificate_config_create(ca_cert, &pem_key_cert_pair, 1); + grpc_slice_unref(cert_slice); + grpc_slice_unref(key_slice); + grpc_slice_unref(ca_slice); ffd->server_credential_reloaded = true; return GRPC_SSL_CERTIFICATE_CONFIG_RELOAD_NEW; } else { @@ -175,20 +192,10 @@ static grpc_end2end_test_config configs[] = { int main(int argc, char** argv) { size_t i; - FILE* roots_file; - size_t roots_size = strlen(test_root_cert); - char* roots_filename; grpc::testing::TestEnvironment env(argc, argv); grpc_end2end_tests_pre_init(); - - /* Set the SSL roots env var. */ - roots_file = gpr_tmpfile("chttp2_simple_ssl_fullstack_test", &roots_filename); - GPR_ASSERT(roots_filename != nullptr); - GPR_ASSERT(roots_file != nullptr); - GPR_ASSERT(fwrite(test_root_cert, 1, roots_size, roots_file) == roots_size); - fclose(roots_file); - GPR_GLOBAL_CONFIG_SET(grpc_default_ssl_roots_file_path, roots_filename); + GPR_GLOBAL_CONFIG_SET(grpc_default_ssl_roots_file_path, CA_CERT_PATH); grpc_init(); @@ -198,9 +205,5 @@ int main(int argc, char** argv) { grpc_shutdown(); - /* Cleanup. */ - remove(roots_filename); - gpr_free(roots_filename); - return 0; } diff --git a/test/core/end2end/fixtures/h2_ssl_proxy.cc b/test/core/end2end/fixtures/h2_ssl_proxy.cc index aea19e54403..ecccda35838 100644 --- a/test/core/end2end/fixtures/h2_ssl_proxy.cc +++ b/test/core/end2end/fixtures/h2_ssl_proxy.cc @@ -16,24 +16,26 @@ * */ -#include "test/core/end2end/end2end_tests.h" - -#include -#include - #include #include +#include +#include #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/gpr/string.h" #include "src/core/lib/gpr/tmpfile.h" +#include "src/core/lib/iomgr/load_file.h" #include "src/core/lib/security/credentials/credentials.h" #include "src/core/lib/security/security_connector/ssl_utils_config.h" -#include "test/core/end2end/data/ssl_test_data.h" +#include "test/core/end2end/end2end_tests.h" #include "test/core/end2end/fixtures/proxy.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" +#define CA_CERT_PATH "src/core/tsi/test_creds/ca.pem" +#define SERVER_CERT_PATH "src/core/tsi/test_creds/server1.pem" +#define SERVER_KEY_PATH "src/core/tsi/test_creds/server1.key" + typedef struct fullstack_secure_fixture_data { grpc_end2end_proxy* proxy; } fullstack_secure_fixture_data; @@ -41,10 +43,20 @@ typedef struct fullstack_secure_fixture_data { static grpc_server* create_proxy_server(const char* port, grpc_channel_args* server_args) { grpc_server* s = grpc_server_create(server_args, nullptr); - grpc_ssl_pem_key_cert_pair pem_cert_key_pair = {test_server1_key, - test_server1_cert}; + grpc_slice cert_slice, key_slice; + GPR_ASSERT(GRPC_LOG_IF_ERROR( + "load_file", grpc_load_file(SERVER_CERT_PATH, 1, &cert_slice))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", + grpc_load_file(SERVER_KEY_PATH, 1, &key_slice))); + const char* server_cert = + reinterpret_cast GRPC_SLICE_START_PTR(cert_slice); + const char* server_key = + reinterpret_cast GRPC_SLICE_START_PTR(key_slice); + grpc_ssl_pem_key_cert_pair pem_key_cert_pair = {server_key, server_cert}; grpc_server_credentials* ssl_creds = grpc_ssl_server_credentials_create( - nullptr, &pem_cert_key_pair, 1, 0, nullptr); + nullptr, &pem_key_cert_pair, 1, 0, nullptr); + grpc_slice_unref(cert_slice); + grpc_slice_unref(key_slice); GPR_ASSERT(grpc_server_add_secure_http2_port(s, port, ssl_creds)); grpc_server_credentials_release(ssl_creds); return s; @@ -166,10 +178,20 @@ static int fail_server_auth_check(grpc_channel_args* server_args) { static void chttp2_init_server_simple_ssl_secure_fullstack( grpc_end2end_test_fixture* f, grpc_channel_args* server_args) { - grpc_ssl_pem_key_cert_pair pem_cert_key_pair = {test_server1_key, - test_server1_cert}; + grpc_slice cert_slice, key_slice; + GPR_ASSERT(GRPC_LOG_IF_ERROR( + "load_file", grpc_load_file(SERVER_CERT_PATH, 1, &cert_slice))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", + grpc_load_file(SERVER_KEY_PATH, 1, &key_slice))); + const char* server_cert = + reinterpret_cast GRPC_SLICE_START_PTR(cert_slice); + const char* server_key = + reinterpret_cast GRPC_SLICE_START_PTR(key_slice); + grpc_ssl_pem_key_cert_pair pem_key_cert_pair = {server_key, server_cert}; grpc_server_credentials* ssl_creds = grpc_ssl_server_credentials_create( - nullptr, &pem_cert_key_pair, 1, 0, nullptr); + nullptr, &pem_key_cert_pair, 1, 0, nullptr); + grpc_slice_unref(cert_slice); + grpc_slice_unref(key_slice); if (fail_server_auth_check(server_args)) { grpc_auth_metadata_processor processor = {process_auth_failure, nullptr, nullptr}; @@ -195,20 +217,10 @@ static grpc_end2end_test_config configs[] = { int main(int argc, char** argv) { size_t i; - FILE* roots_file; - size_t roots_size = strlen(test_root_cert); - char* roots_filename; grpc::testing::TestEnvironment env(argc, argv); grpc_end2end_tests_pre_init(); - - /* Set the SSL roots env var. */ - roots_file = gpr_tmpfile("chttp2_simple_ssl_fullstack_test", &roots_filename); - GPR_ASSERT(roots_filename != nullptr); - GPR_ASSERT(roots_file != nullptr); - GPR_ASSERT(fwrite(test_root_cert, 1, roots_size, roots_file) == roots_size); - fclose(roots_file); - GPR_GLOBAL_CONFIG_SET(grpc_default_ssl_roots_file_path, roots_filename); + GPR_GLOBAL_CONFIG_SET(grpc_default_ssl_roots_file_path, CA_CERT_PATH); grpc_init(); @@ -218,9 +230,5 @@ int main(int argc, char** argv) { grpc_shutdown(); - /* Cleanup. */ - remove(roots_filename); - gpr_free(roots_filename); - return 0; } diff --git a/test/core/end2end/fixtures/h2_tls.cc b/test/core/end2end/fixtures/h2_tls.cc index dfba0f12530..95cff3a2b00 100644 --- a/test/core/end2end/fixtures/h2_tls.cc +++ b/test/core/end2end/fixtures/h2_tls.cc @@ -30,14 +30,18 @@ #include "src/core/lib/gprpp/host_port.h" #include "src/core/lib/gprpp/inlined_vector.h" #include "src/core/lib/gprpp/thd.h" +#include "src/core/lib/iomgr/load_file.h" #include "src/core/lib/security/credentials/credentials.h" #include "src/core/lib/security/credentials/tls/grpc_tls_credentials_options.h" #include "src/core/lib/security/security_connector/ssl_utils_config.h" -#include "test/core/end2end/data/ssl_test_data.h" #include "test/core/end2end/end2end_tests.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" +#define CA_CERT_PATH "src/core/tsi/test_creds/ca.pem" +#define SERVER_CERT_PATH "src/core/tsi/test_creds/server1.pem" +#define SERVER_KEY_PATH "src/core/tsi/test_creds/server1.key" + typedef grpc_core::InlinedVector ThreadList; struct fullstack_secure_fixture_data { @@ -140,17 +144,30 @@ static int client_cred_reload_sync(void* /*config_user_data*/, arg->status = GRPC_SSL_CERTIFICATE_CONFIG_RELOAD_UNCHANGED; return 0; } - const grpc_ssl_pem_key_cert_pair pem_key_pair = { - test_server1_key, - test_server1_cert, - }; + grpc_slice ca_slice, cert_slice, key_slice; + GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", + grpc_load_file(CA_CERT_PATH, 1, &ca_slice))); + GPR_ASSERT(GRPC_LOG_IF_ERROR( + "load_file", grpc_load_file(SERVER_CERT_PATH, 1, &cert_slice))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", + grpc_load_file(SERVER_KEY_PATH, 1, &key_slice))); + const char* ca_cert = + reinterpret_cast GRPC_SLICE_START_PTR(ca_slice); + const char* server_cert = + reinterpret_cast GRPC_SLICE_START_PTR(cert_slice); + const char* server_key = + reinterpret_cast GRPC_SLICE_START_PTR(key_slice); + grpc_ssl_pem_key_cert_pair pem_key_cert_pair = {server_key, server_cert}; if (arg->key_materials_config->pem_key_cert_pair_list().empty()) { - const auto* pem_key_pair_ptr = &pem_key_pair; + const auto* pem_key_cert_pair_ptr = &pem_key_cert_pair; grpc_tls_key_materials_config_set_key_materials( - arg->key_materials_config, test_root_cert, &pem_key_pair_ptr, 1); + arg->key_materials_config, ca_cert, &pem_key_cert_pair_ptr, 1); } // new credential has been reloaded. arg->status = GRPC_SSL_CERTIFICATE_CONFIG_RELOAD_NEW; + grpc_slice_unref(cert_slice); + grpc_slice_unref(key_slice); + grpc_slice_unref(ca_slice); return 0; } @@ -163,21 +180,34 @@ static int server_cred_reload_sync(void* /*config_user_data*/, arg->status = GRPC_SSL_CERTIFICATE_CONFIG_RELOAD_UNCHANGED; return 0; } - const grpc_ssl_pem_key_cert_pair pem_key_pair = { - test_server1_key, - test_server1_cert, - }; + grpc_slice ca_slice, cert_slice, key_slice; + GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", + grpc_load_file(CA_CERT_PATH, 1, &ca_slice))); + GPR_ASSERT(GRPC_LOG_IF_ERROR( + "load_file", grpc_load_file(SERVER_CERT_PATH, 1, &cert_slice))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", + grpc_load_file(SERVER_KEY_PATH, 1, &key_slice))); + const char* ca_cert = + reinterpret_cast GRPC_SLICE_START_PTR(ca_slice); + const char* server_cert = + reinterpret_cast GRPC_SLICE_START_PTR(cert_slice); + const char* server_key = + reinterpret_cast GRPC_SLICE_START_PTR(key_slice); + grpc_ssl_pem_key_cert_pair pem_key_cert_pair = {server_key, server_cert}; GPR_ASSERT(arg != nullptr); GPR_ASSERT(arg->key_materials_config != nullptr); GPR_ASSERT(arg->key_materials_config->pem_key_cert_pair_list().data() != nullptr); if (arg->key_materials_config->pem_key_cert_pair_list().empty()) { - const auto* pem_key_pair_ptr = &pem_key_pair; + const auto* pem_key_cert_pair_ptr = &pem_key_cert_pair; grpc_tls_key_materials_config_set_key_materials( - arg->key_materials_config, test_root_cert, &pem_key_pair_ptr, 1); + arg->key_materials_config, ca_cert, &pem_key_cert_pair_ptr, 1); } // new credential has been reloaded. arg->status = GRPC_SSL_CERTIFICATE_CONFIG_RELOAD_NEW; + grpc_slice_unref(cert_slice); + grpc_slice_unref(key_slice); + grpc_slice_unref(ca_slice); return 0; } @@ -268,25 +298,13 @@ static grpc_end2end_test_config configs[] = { }; int main(int argc, char** argv) { - FILE* roots_file; - size_t roots_size = strlen(test_root_cert); - char* roots_filename; grpc::testing::TestEnvironment env(argc, argv); grpc_end2end_tests_pre_init(); - /* Set the SSL roots env var. */ - roots_file = gpr_tmpfile("chttp2_simple_ssl_fullstack_test", &roots_filename); - GPR_ASSERT(roots_filename != nullptr); - GPR_ASSERT(roots_file != nullptr); - GPR_ASSERT(fwrite(test_root_cert, 1, roots_size, roots_file) == roots_size); - fclose(roots_file); - GPR_GLOBAL_CONFIG_SET(grpc_default_ssl_roots_file_path, roots_filename); + GPR_GLOBAL_CONFIG_SET(grpc_default_ssl_roots_file_path, CA_CERT_PATH); grpc_init(); for (size_t ind = 0; ind < sizeof(configs) / sizeof(*configs); ind++) { grpc_end2end_tests(argc, argv, configs[ind]); } grpc_shutdown(); - /* Cleanup. */ - remove(roots_filename); - gpr_free(roots_filename); return 0; } diff --git a/test/core/end2end/generate_tests.bzl b/test/core/end2end/generate_tests.bzl index e7034222e52..851bb5ca0d0 100755 --- a/test/core/end2end/generate_tests.bzl +++ b/test/core/end2end/generate_tests.bzl @@ -431,8 +431,6 @@ def grpc_end2end_tests(): language = "C++", data = [ "//src/core/tsi/test_creds:ca.pem", - "//src/core/tsi/test_creds:client.key", - "//src/core/tsi/test_creds:client.pem", "//src/core/tsi/test_creds:server1.key", "//src/core/tsi/test_creds:server1.pem", ], @@ -508,8 +506,6 @@ def grpc_end2end_nosec_tests(): language = "C++", data = [ "//src/core/tsi/test_creds:ca.pem", - "//src/core/tsi/test_creds:client.key", - "//src/core/tsi/test_creds:client.pem", "//src/core/tsi/test_creds:server1.key", "//src/core/tsi/test_creds:server1.pem", ], From 6113d708120795565f4ccbab27b803f9630932c4 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Tue, 14 Apr 2020 10:55:37 -0700 Subject: [PATCH 70/73] Enable retries for failed GCP API calls The HttpRequest.execute() method used to make GCP calls has a `num_retries` parameter, as documented at https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.http.HttpRequest-class.html#execute --- tools/run_tests/run_xds_tests.py | 119 ++++++++++++++++++------------- 1 file changed, 71 insertions(+), 48 deletions(-) diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index 762743e0993..876825295db 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -198,6 +198,7 @@ _INSTANCE_GROUP_SIZE = args.instance_group_size _NUM_TEST_RPCS = 10 * args.qps _WAIT_FOR_STATS_SEC = 180 _WAIT_FOR_URL_MAP_PATCH_SEC = 300 +_GCP_API_RETRIES = 5 _BOOTSTRAP_TEMPLATE = """ {{ "node": {{ @@ -549,8 +550,8 @@ def create_instance_template(gcp, name, network, source_image, machine_type, } logger.debug('Sending GCP request with body=%s', config) - result = gcp.compute.instanceTemplates().insert(project=gcp.project, - body=config).execute() + result = gcp.compute.instanceTemplates().insert( + project=gcp.project, body=config).execute(num_retries=_GCP_API_RETRIES) wait_for_global_operation(gcp, result['name']) gcp.instance_template = GcpResource(config['name'], result['targetLink']) @@ -567,13 +568,14 @@ def add_instance_group(gcp, zone, name, size): } logger.debug('Sending GCP request with body=%s', config) - result = gcp.compute.instanceGroupManagers().insert(project=gcp.project, - zone=zone, - body=config).execute() + result = gcp.compute.instanceGroupManagers().insert( + project=gcp.project, zone=zone, + body=config).execute(num_retries=_GCP_API_RETRIES) wait_for_zone_operation(gcp, zone, result['name']) result = gcp.compute.instanceGroupManagers().get( project=gcp.project, zone=zone, - instanceGroupManager=config['name']).execute() + instanceGroupManager=config['name']).execute( + num_retries=_GCP_API_RETRIES) instance_group = InstanceGroup(config['name'], result['instanceGroup'], zone) gcp.instance_groups.append(instance_group) @@ -600,8 +602,8 @@ def create_health_check(gcp, name): } compute_to_use = gcp.compute logger.debug('Sending GCP request with body=%s', config) - result = compute_to_use.healthChecks().insert(project=gcp.project, - body=config).execute() + result = compute_to_use.healthChecks().insert( + project=gcp.project, body=config).execute(num_retries=_GCP_API_RETRIES) wait_for_global_operation(gcp, result['name']) gcp.health_check = GcpResource(config['name'], result['targetLink']) @@ -617,8 +619,8 @@ def create_health_check_firewall_rule(gcp, name): 'targetTags': ['allow-health-checks'], } logger.debug('Sending GCP request with body=%s', config) - result = gcp.compute.firewalls().insert(project=gcp.project, - body=config).execute() + result = gcp.compute.firewalls().insert( + project=gcp.project, body=config).execute(num_retries=_GCP_API_RETRIES) wait_for_global_operation(gcp, result['name']) gcp.health_check_firewall_rule = GcpResource(config['name'], result['targetLink']) @@ -639,8 +641,8 @@ def add_backend_service(gcp, name): 'protocol': protocol } logger.debug('Sending GCP request with body=%s', config) - result = compute_to_use.backendServices().insert(project=gcp.project, - body=config).execute() + result = compute_to_use.backendServices().insert( + project=gcp.project, body=config).execute(num_retries=_GCP_API_RETRIES) wait_for_global_operation(gcp, result['name']) backend_service = GcpResource(config['name'], result['targetLink']) gcp.backend_services.append(backend_service) @@ -661,8 +663,8 @@ def create_url_map(gcp, name, backend_service, host_name): }] } logger.debug('Sending GCP request with body=%s', config) - result = gcp.compute.urlMaps().insert(project=gcp.project, - body=config).execute() + result = gcp.compute.urlMaps().insert( + project=gcp.project, body=config).execute(num_retries=_GCP_API_RETRIES) wait_for_global_operation(gcp, result['name']) gcp.url_map = GcpResource(config['name'], result['targetLink']) @@ -675,9 +677,9 @@ def patch_url_map_host_rule_with_port(gcp, name, backend_service, host_name): }] } logger.debug('Sending GCP request with body=%s', config) - result = gcp.compute.urlMaps().patch(project=gcp.project, - urlMap=name, - body=config).execute() + result = gcp.compute.urlMaps().patch( + project=gcp.project, urlMap=name, + body=config).execute(num_retries=_GCP_API_RETRIES) wait_for_global_operation(gcp, result['name']) @@ -690,15 +692,17 @@ def create_target_proxy(gcp, name): } logger.debug('Sending GCP request with body=%s', config) result = gcp.alpha_compute.targetGrpcProxies().insert( - project=gcp.project, body=config).execute() + project=gcp.project, + body=config).execute(num_retries=_GCP_API_RETRIES) else: config = { 'name': name, 'url_map': gcp.url_map.url, } logger.debug('Sending GCP request with body=%s', config) - result = gcp.compute.targetHttpProxies().insert(project=gcp.project, - body=config).execute() + result = gcp.compute.targetHttpProxies().insert( + project=gcp.project, + body=config).execute(num_retries=_GCP_API_RETRIES) wait_for_global_operation(gcp, result['name']) gcp.target_proxy = GcpResource(config['name'], result['targetLink']) @@ -720,7 +724,8 @@ def create_global_forwarding_rule(gcp, name, potential_ports): } logger.debug('Sending GCP request with body=%s', config) result = compute_to_use.globalForwardingRules().insert( - project=gcp.project, body=config).execute() + project=gcp.project, + body=config).execute(num_retries=_GCP_API_RETRIES) wait_for_global_operation(gcp, result['name']) gcp.global_forwarding_rule = GcpResource(config['name'], result['targetLink']) @@ -736,7 +741,8 @@ def delete_global_forwarding_rule(gcp): try: result = gcp.compute.globalForwardingRules().delete( project=gcp.project, - forwardingRule=gcp.global_forwarding_rule.name).execute() + forwardingRule=gcp.global_forwarding_rule.name).execute( + num_retries=_GCP_API_RETRIES) wait_for_global_operation(gcp, result['name']) except googleapiclient.errors.HttpError as http_error: logger.info('Delete failed: %s', http_error) @@ -747,11 +753,13 @@ def delete_target_proxy(gcp): if gcp.alpha_compute: result = gcp.alpha_compute.targetGrpcProxies().delete( project=gcp.project, - targetGrpcProxy=gcp.target_proxy.name).execute() + targetGrpcProxy=gcp.target_proxy.name).execute( + num_retries=_GCP_API_RETRIES) else: result = gcp.compute.targetHttpProxies().delete( project=gcp.project, - targetHttpProxy=gcp.target_proxy.name).execute() + targetHttpProxy=gcp.target_proxy.name).execute( + num_retries=_GCP_API_RETRIES) wait_for_global_operation(gcp, result['name']) except googleapiclient.errors.HttpError as http_error: logger.info('Delete failed: %s', http_error) @@ -760,7 +768,8 @@ def delete_target_proxy(gcp): def delete_url_map(gcp): try: result = gcp.compute.urlMaps().delete( - project=gcp.project, urlMap=gcp.url_map.name).execute() + project=gcp.project, + urlMap=gcp.url_map.name).execute(num_retries=_GCP_API_RETRIES) wait_for_global_operation(gcp, result['name']) except googleapiclient.errors.HttpError as http_error: logger.info('Delete failed: %s', http_error) @@ -771,7 +780,8 @@ def delete_backend_services(gcp): try: result = gcp.compute.backendServices().delete( project=gcp.project, - backendService=backend_service.name).execute() + backendService=backend_service.name).execute( + num_retries=_GCP_API_RETRIES) wait_for_global_operation(gcp, result['name']) except googleapiclient.errors.HttpError as http_error: logger.info('Delete failed: %s', http_error) @@ -781,7 +791,8 @@ def delete_firewall(gcp): try: result = gcp.compute.firewalls().delete( project=gcp.project, - firewall=gcp.health_check_firewall_rule.name).execute() + firewall=gcp.health_check_firewall_rule.name).execute( + num_retries=_GCP_API_RETRIES) wait_for_global_operation(gcp, result['name']) except googleapiclient.errors.HttpError as http_error: logger.info('Delete failed: %s', http_error) @@ -790,7 +801,8 @@ def delete_firewall(gcp): def delete_health_check(gcp): try: result = gcp.compute.healthChecks().delete( - project=gcp.project, healthCheck=gcp.health_check.name).execute() + project=gcp.project, healthCheck=gcp.health_check.name).execute( + num_retries=_GCP_API_RETRIES) wait_for_global_operation(gcp, result['name']) except googleapiclient.errors.HttpError as http_error: logger.info('Delete failed: %s', http_error) @@ -802,7 +814,8 @@ def delete_instance_groups(gcp): result = gcp.compute.instanceGroupManagers().delete( project=gcp.project, zone=instance_group.zone, - instanceGroupManager=instance_group.name).execute() + instanceGroupManager=instance_group.name).execute( + num_retries=_GCP_API_RETRIES) wait_for_zone_operation(gcp, instance_group.zone, result['name'], @@ -815,7 +828,8 @@ def delete_instance_template(gcp): try: result = gcp.compute.instanceTemplates().delete( project=gcp.project, - instanceTemplate=gcp.instance_template.name).execute() + instanceTemplate=gcp.instance_template.name).execute( + num_retries=_GCP_API_RETRIES) wait_for_global_operation(gcp, result['name']) except googleapiclient.errors.HttpError as http_error: logger.info('Delete failed: %s', http_error) @@ -839,7 +853,7 @@ def patch_backend_instances(gcp, logger.debug('Sending GCP request with body=%s', config) result = compute_to_use.backendServices().patch( project=gcp.project, backendService=backend_service.name, - body=config).execute() + body=config).execute(num_retries=_GCP_API_RETRIES) wait_for_global_operation(gcp, result['name'], timeout_sec=_WAIT_FOR_BACKEND_SEC) @@ -853,7 +867,7 @@ def resize_instance_group(gcp, project=gcp.project, zone=instance_group.zone, instanceGroupManager=instance_group.name, - size=new_size).execute() + size=new_size).execute(num_retries=_GCP_API_RETRIES) wait_for_zone_operation(gcp, instance_group.zone, result['name'], @@ -878,9 +892,9 @@ def patch_url_map_backend_service(gcp, backend_service): }] } logger.debug('Sending GCP request with body=%s', config) - result = gcp.compute.urlMaps().patch(project=gcp.project, - urlMap=gcp.url_map.name, - body=config).execute() + result = gcp.compute.urlMaps().patch( + project=gcp.project, urlMap=gcp.url_map.name, + body=config).execute(num_retries=_GCP_API_RETRIES) wait_for_global_operation(gcp, result['name']) @@ -890,7 +904,8 @@ def wait_for_global_operation(gcp, start_time = time.time() while time.time() - start_time <= timeout_sec: result = gcp.compute.globalOperations().get( - project=gcp.project, operation=operation).execute() + project=gcp.project, + operation=operation).execute(num_retries=_GCP_API_RETRIES) if result['status'] == 'DONE': if 'error' in result: raise Exception(result['error']) @@ -907,7 +922,8 @@ def wait_for_zone_operation(gcp, start_time = time.time() while time.time() - start_time <= timeout_sec: result = gcp.compute.zoneOperations().get( - project=gcp.project, zone=zone, operation=operation).execute() + project=gcp.project, zone=zone, + operation=operation).execute(num_retries=_GCP_API_RETRIES) if result['status'] == 'DONE': if 'error' in result: raise Exception(result['error']) @@ -927,7 +943,7 @@ def wait_for_healthy_backends(gcp, result = gcp.compute.backendServices().getHealth( project=gcp.project, backendService=backend_service.name, - body=config).execute() + body=config).execute(num_retries=_GCP_API_RETRIES) if 'healthStatus' in result: healthy = True for instance in result['healthStatus']: @@ -949,7 +965,7 @@ def get_instance_names(gcp, instance_group): instanceGroup=instance_group.name, body={ 'instanceState': 'ALL' - }).execute() + }).execute(num_retries=_GCP_API_RETRIES) if 'items' not in result: return [] for item in result['items']: @@ -1081,19 +1097,22 @@ try: if not gcp.instance_template: result = compute.instanceTemplates().get( project=args.project_id, - instanceTemplate=template_name).execute() + instanceTemplate=template_name).execute( + num_retries=_GCP_API_RETRIES) gcp.instance_template = GcpResource(template_name, result['selfLink']) if not gcp.backend_services: result = compute.backendServices().get( project=args.project_id, - backendService=backend_service_name).execute() + backendService=backend_service_name).execute( + num_retries=_GCP_API_RETRIES) backend_service = GcpResource(backend_service_name, result['selfLink']) gcp.backend_services.append(backend_service) result = compute.backendServices().get( project=args.project_id, - backendService=alternate_backend_service_name).execute() + backendService=alternate_backend_service_name).execute( + num_retries=_GCP_API_RETRIES) alternate_backend_service = GcpResource( alternate_backend_service_name, result['selfLink']) gcp.backend_services.append(alternate_backend_service) @@ -1101,14 +1120,16 @@ try: result = compute.instanceGroups().get( project=args.project_id, zone=args.zone, - instanceGroup=instance_group_name).execute() + instanceGroup=instance_group_name).execute( + num_retries=_GCP_API_RETRIES) instance_group = InstanceGroup(instance_group_name, result['selfLink'], args.zone) gcp.instance_groups.append(instance_group) result = compute.instanceGroups().get( project=args.project_id, zone=args.zone, - instanceGroup=same_zone_instance_group_name).execute() + instanceGroup=same_zone_instance_group_name).execute( + num_retries=_GCP_API_RETRIES) same_zone_instance_group = InstanceGroup( same_zone_instance_group_name, result['selfLink'], args.zone) @@ -1118,7 +1139,7 @@ try: project=args.project_id, zone=args.secondary_zone, instanceGroup=secondary_zone_instance_group_name - ).execute() + ).execute(num_retries=_GCP_API_RETRIES) secondary_zone_instance_group = InstanceGroup( secondary_zone_instance_group_name, result['selfLink'], args.secondary_zone) @@ -1126,12 +1147,14 @@ try: if not gcp.health_check: result = compute.healthChecks().get( project=args.project_id, - healthCheck=health_check_name).execute() + healthCheck=health_check_name).execute( + num_retries=_GCP_API_RETRIES) gcp.health_check = GcpResource(health_check_name, result['selfLink']) if not gcp.url_map: - result = compute.urlMaps().get(project=args.project_id, - urlMap=url_map_name).execute() + result = compute.urlMaps().get( + project=args.project_id, + urlMap=url_map_name).execute(num_retries=_GCP_API_RETRIES) gcp.url_map = GcpResource(url_map_name, result['selfLink']) if not gcp.service_port: gcp.service_port = args.service_port_range[0] From d1cb8a32aa58ada95b0de3c8a7e2d7c69c9f1673 Mon Sep 17 00:00:00 2001 From: Eric Gribkoff Date: Tue, 14 Apr 2020 11:05:56 -0700 Subject: [PATCH 71/73] increase time.sleep between polling operations --- tools/run_tests/run_xds_tests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/run_tests/run_xds_tests.py b/tools/run_tests/run_xds_tests.py index 876825295db..c6247c15fb8 100755 --- a/tools/run_tests/run_xds_tests.py +++ b/tools/run_tests/run_xds_tests.py @@ -879,7 +879,7 @@ def resize_instance_group(gcp, break if time.time() - start_time > timeout_sec: raise Exception('Failed to resize primary instance group') - time.sleep(1) + time.sleep(2) def patch_url_map_backend_service(gcp, backend_service): @@ -910,7 +910,7 @@ def wait_for_global_operation(gcp, if 'error' in result: raise Exception(result['error']) return - time.sleep(1) + time.sleep(2) raise Exception('Operation %s did not complete within %d', operation, timeout_sec) @@ -928,7 +928,7 @@ def wait_for_zone_operation(gcp, if 'error' in result: raise Exception(result['error']) return - time.sleep(1) + time.sleep(2) raise Exception('Operation %s did not complete within %d', operation, timeout_sec) @@ -952,7 +952,7 @@ def wait_for_healthy_backends(gcp, break if healthy: return - time.sleep(1) + time.sleep(2) raise Exception('Not all backends became healthy within %d seconds: %s' % (timeout_sec, result)) From 096c2761821ef1f6e0277ca1872fe62c791b96a3 Mon Sep 17 00:00:00 2001 From: ZhenLian Date: Tue, 14 Apr 2020 00:06:16 -0700 Subject: [PATCH 72/73] [4/n] Avoid using hardcoded test credentials --- test/core/security/BUILD | 15 +++++ .../grpc_tls_credentials_options_test.cc | 58 ++++++++++++++----- test/core/security/ssl_server_fuzzer.cc | 33 ++++++----- .../security/tls_security_connector_test.cc | 33 ++++++++--- test/core/surface/BUILD | 5 ++ .../surface/sequential_connectivity_test.cc | 28 +++++++-- test/core/util/grpc_fuzzer.bzl | 4 +- 7 files changed, 136 insertions(+), 40 deletions(-) diff --git a/test/core/security/BUILD b/test/core/security/BUILD index d4ffb2b3a32..ad698aa68ad 100644 --- a/test/core/security/BUILD +++ b/test/core/security/BUILD @@ -37,6 +37,11 @@ grpc_fuzzer( name = "ssl_server_fuzzer", srcs = ["ssl_server_fuzzer.cc"], corpus = "corpus/ssl_server_corpus", + data = [ + "//src/core/tsi/test_creds:ca.pem", + "//src/core/tsi/test_creds:server1.key", + "//src/core/tsi/test_creds:server1.pem", + ], language = "C++", tags = ["no_windows"], deps = [ @@ -248,6 +253,11 @@ grpc_cc_test( grpc_cc_test( name = "tls_security_connector_test", srcs = ["tls_security_connector_test.cc"], + data = [ + "//src/core/tsi/test_creds:ca.pem", + "//src/core/tsi/test_creds:server1.key", + "//src/core/tsi/test_creds:server1.pem", + ], external_deps = [ "gtest", ], @@ -266,6 +276,11 @@ grpc_cc_test( grpc_cc_test( name = "grpc_tls_credentials_options_test", srcs = ["grpc_tls_credentials_options_test.cc"], + data = [ + "//src/core/tsi/test_creds:ca.pem", + "//src/core/tsi/test_creds:server1.key", + "//src/core/tsi/test_creds:server1.pem", + ], external_deps = ["gtest"], language = "C++", deps = [ diff --git a/test/core/security/grpc_tls_credentials_options_test.cc b/test/core/security/grpc_tls_credentials_options_test.cc index 142aabf5858..ff8bf1dac04 100644 --- a/test/core/security/grpc_tls_credentials_options_test.cc +++ b/test/core/security/grpc_tls_credentials_options_test.cc @@ -17,7 +17,6 @@ */ #include "src/core/lib/security/credentials/tls/grpc_tls_credentials_options.h" -#include "test/core/end2end/data/ssl_test_data.h" #include #include @@ -25,28 +24,61 @@ #include #include +#include "src/core/lib/iomgr/load_file.h" + +#define CA_CERT_PATH "src/core/tsi/test_creds/ca.pem" +#define SERVER_CERT_PATH "src/core/tsi/test_creds/server1.pem" +#define SERVER_KEY_PATH "src/core/tsi/test_creds/server1.key" + namespace testing { static void SetKeyMaterials(grpc_tls_key_materials_config* config) { - const grpc_ssl_pem_key_cert_pair pem_key_pair = { - test_server1_key, - test_server1_cert, - }; - const auto* pem_key_pair_ptr = &pem_key_pair; - grpc_tls_key_materials_config_set_key_materials(config, test_root_cert, - &pem_key_pair_ptr, 1); + grpc_slice ca_slice, cert_slice, key_slice; + GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", + grpc_load_file(CA_CERT_PATH, 1, &ca_slice))); + GPR_ASSERT(GRPC_LOG_IF_ERROR( + "load_file", grpc_load_file(SERVER_CERT_PATH, 1, &cert_slice))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", + grpc_load_file(SERVER_KEY_PATH, 1, &key_slice))); + const char* ca_cert = + reinterpret_cast GRPC_SLICE_START_PTR(ca_slice); + const char* server_cert = + reinterpret_cast GRPC_SLICE_START_PTR(cert_slice); + const char* server_key = + reinterpret_cast GRPC_SLICE_START_PTR(key_slice); + grpc_ssl_pem_key_cert_pair pem_key_cert_pair = {server_key, server_cert}; + const auto* pem_key_cert_pair_ptr = &pem_key_cert_pair; + grpc_tls_key_materials_config_set_key_materials(config, ca_cert, + &pem_key_cert_pair_ptr, 1); + grpc_slice_unref(cert_slice); + grpc_slice_unref(key_slice); + grpc_slice_unref(ca_slice); } TEST(GrpcTlsCredentialsOptionsTest, SetKeyMaterials) { grpc_tls_key_materials_config* config = grpc_tls_key_materials_config_create(); SetKeyMaterials(config); - EXPECT_STREQ(config->pem_root_certs(), test_root_cert); + grpc_slice ca_slice, cert_slice, key_slice; + GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", + grpc_load_file(CA_CERT_PATH, 1, &ca_slice))); + GPR_ASSERT(GRPC_LOG_IF_ERROR( + "load_file", grpc_load_file(SERVER_CERT_PATH, 1, &cert_slice))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", + grpc_load_file(SERVER_KEY_PATH, 1, &key_slice))); + const char* ca_cert = + reinterpret_cast GRPC_SLICE_START_PTR(ca_slice); + const char* server_cert = + reinterpret_cast GRPC_SLICE_START_PTR(cert_slice); + const char* server_key = + reinterpret_cast GRPC_SLICE_START_PTR(key_slice); + EXPECT_STREQ(config->pem_root_certs(), ca_cert); EXPECT_EQ(config->pem_key_cert_pair_list().size(), 1); - EXPECT_STREQ(config->pem_key_cert_pair_list()[0].private_key(), - test_server1_key); - EXPECT_STREQ(config->pem_key_cert_pair_list()[0].cert_chain(), - test_server1_cert); + EXPECT_STREQ(config->pem_key_cert_pair_list()[0].private_key(), server_key); + EXPECT_STREQ(config->pem_key_cert_pair_list()[0].cert_chain(), server_cert); + grpc_slice_unref(cert_slice); + grpc_slice_unref(key_slice); + grpc_slice_unref(ca_slice); delete config; } diff --git a/test/core/security/ssl_server_fuzzer.cc b/test/core/security/ssl_server_fuzzer.cc index 18f1dd7814b..538e43964c1 100644 --- a/test/core/security/ssl_server_fuzzer.cc +++ b/test/core/security/ssl_server_fuzzer.cc @@ -23,9 +23,12 @@ #include "src/core/lib/iomgr/load_file.h" #include "src/core/lib/security/credentials/credentials.h" #include "src/core/lib/security/security_connector/security_connector.h" -#include "test/core/end2end/data/ssl_test_data.h" #include "test/core/util/mock_endpoint.h" +#define CA_CERT_PATH "src/core/tsi/test_creds/ca.pem" +#define SERVER_CERT_PATH "src/core/tsi/test_creds/server1.pem" +#define SERVER_KEY_PATH "src/core/tsi/test_creds/server1.key" + bool squelch = true; // ssl has an array of global gpr_mu's that are never released. // Turning this on will fail the leak check. @@ -66,18 +69,25 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { mock_endpoint, grpc_slice_from_copied_buffer((const char*)data, size)); // Load key pair and establish server SSL credentials. - grpc_ssl_pem_key_cert_pair pem_key_cert_pair; grpc_slice ca_slice, cert_slice, key_slice; - ca_slice = grpc_slice_from_static_string(test_root_cert); - cert_slice = grpc_slice_from_static_string(test_server1_cert); - key_slice = grpc_slice_from_static_string(test_server1_key); - const char* ca_cert = (const char*)GRPC_SLICE_START_PTR(ca_slice); - pem_key_cert_pair.private_key = - (const char*)GRPC_SLICE_START_PTR(key_slice); - pem_key_cert_pair.cert_chain = - (const char*)GRPC_SLICE_START_PTR(cert_slice); + GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", + grpc_load_file(CA_CERT_PATH, 1, &ca_slice))); + GPR_ASSERT(GRPC_LOG_IF_ERROR( + "load_file", grpc_load_file(SERVER_CERT_PATH, 1, &cert_slice))); + GPR_ASSERT(GRPC_LOG_IF_ERROR( + "load_file", grpc_load_file(SERVER_KEY_PATH, 1, &key_slice))); + const char* ca_cert = + reinterpret_cast GRPC_SLICE_START_PTR(ca_slice); + const char* server_cert = + reinterpret_cast GRPC_SLICE_START_PTR(cert_slice); + const char* server_key = + reinterpret_cast GRPC_SLICE_START_PTR(key_slice); + grpc_ssl_pem_key_cert_pair pem_key_cert_pair = {server_key, server_cert}; grpc_server_credentials* creds = grpc_ssl_server_credentials_create( ca_cert, &pem_key_cert_pair, 1, 0, nullptr); + grpc_slice_unref(cert_slice); + grpc_slice_unref(key_slice); + grpc_slice_unref(ca_slice); // Create security connector grpc_core::RefCountedPtr sc = @@ -109,9 +119,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { sc.reset(DEBUG_LOCATION, "test"); grpc_server_credentials_release(creds); - grpc_slice_unref(cert_slice); - grpc_slice_unref(key_slice); - grpc_slice_unref(ca_slice); grpc_core::ExecCtx::Get()->Flush(); } diff --git a/test/core/security/tls_security_connector_test.cc b/test/core/security/tls_security_connector_test.cc index 3f81f025a80..ef2f94a80b8 100644 --- a/test/core/security/tls_security_connector_test.cc +++ b/test/core/security/tls_security_connector_test.cc @@ -26,22 +26,39 @@ #include #include +#include "src/core/lib/iomgr/load_file.h" #include "src/core/tsi/transport_security.h" -#include "test/core/end2end/data/ssl_test_data.h" #include "test/core/util/test_config.h" +#define CA_CERT_PATH "src/core/tsi/test_creds/ca.pem" +#define SERVER_CERT_PATH "src/core/tsi/test_creds/server1.pem" +#define SERVER_KEY_PATH "src/core/tsi/test_creds/server1.key" + namespace { enum CredReloadResult { FAIL, SUCCESS, UNCHANGED, ASYNC }; void SetKeyMaterials(grpc_tls_key_materials_config* config) { - const grpc_ssl_pem_key_cert_pair pem_key_pair = { - test_server1_key, - test_server1_cert, - }; - const auto* pem_key_pair_ptr = &pem_key_pair; - grpc_tls_key_materials_config_set_key_materials(config, test_root_cert, - &pem_key_pair_ptr, 1); + grpc_slice ca_slice, cert_slice, key_slice; + GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", + grpc_load_file(CA_CERT_PATH, 1, &ca_slice))); + GPR_ASSERT(GRPC_LOG_IF_ERROR( + "load_file", grpc_load_file(SERVER_CERT_PATH, 1, &cert_slice))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", + grpc_load_file(SERVER_KEY_PATH, 1, &key_slice))); + const char* ca_cert = + reinterpret_cast GRPC_SLICE_START_PTR(ca_slice); + const char* server_cert = + reinterpret_cast GRPC_SLICE_START_PTR(cert_slice); + const char* server_key = + reinterpret_cast GRPC_SLICE_START_PTR(key_slice); + grpc_ssl_pem_key_cert_pair pem_key_cert_pair = {server_key, server_cert}; + const auto* pem_key_cert_pair_ptr = &pem_key_cert_pair; + grpc_tls_key_materials_config_set_key_materials(config, ca_cert, + &pem_key_cert_pair_ptr, 1); + grpc_slice_unref(cert_slice); + grpc_slice_unref(key_slice); + grpc_slice_unref(ca_slice); } int CredReloadSuccess(void* /*config_user_data*/, diff --git a/test/core/surface/BUILD b/test/core/surface/BUILD index 37a371dc4e8..b7ea065f020 100644 --- a/test/core/surface/BUILD +++ b/test/core/surface/BUILD @@ -136,6 +136,11 @@ grpc_cc_test( grpc_cc_test( name = "sequential_connectivity_test", srcs = ["sequential_connectivity_test.cc"], + data = [ + "//src/core/tsi/test_creds:ca.pem", + "//src/core/tsi/test_creds:server1.key", + "//src/core/tsi/test_creds:server1.pem", + ], flaky = True, # TODO(b/151696318) language = "C++", deps = [ diff --git a/test/core/surface/sequential_connectivity_test.cc b/test/core/surface/sequential_connectivity_test.cc index c4a8667331c..8f7a46bfe38 100644 --- a/test/core/surface/sequential_connectivity_test.cc +++ b/test/core/surface/sequential_connectivity_test.cc @@ -25,10 +25,14 @@ #include "src/core/lib/gprpp/host_port.h" #include "src/core/lib/gprpp/thd.h" #include "src/core/lib/iomgr/exec_ctx.h" -#include "test/core/end2end/data/ssl_test_data.h" +#include "src/core/lib/iomgr/load_file.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" +#define CA_CERT_PATH "src/core/tsi/test_creds/ca.pem" +#define SERVER_CERT_PATH "src/core/tsi/test_creds/server1.pem" +#define SERVER_KEY_PATH "src/core/tsi/test_creds/server1.key" + typedef struct test_fixture { const char* name; void (*add_server_port)(grpc_server* server, const char* addr); @@ -139,17 +143,33 @@ static const test_fixture insecure_test = { }; static void secure_test_add_port(grpc_server* server, const char* addr) { - grpc_ssl_pem_key_cert_pair pem_cert_key_pair = {test_server1_key, - test_server1_cert}; + grpc_slice cert_slice, key_slice; + GPR_ASSERT(GRPC_LOG_IF_ERROR( + "load_file", grpc_load_file(SERVER_CERT_PATH, 1, &cert_slice))); + GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", + grpc_load_file(SERVER_KEY_PATH, 1, &key_slice))); + const char* server_cert = + reinterpret_cast GRPC_SLICE_START_PTR(cert_slice); + const char* server_key = + reinterpret_cast GRPC_SLICE_START_PTR(key_slice); + grpc_ssl_pem_key_cert_pair pem_key_cert_pair = {server_key, server_cert}; grpc_server_credentials* ssl_creds = grpc_ssl_server_credentials_create( - nullptr, &pem_cert_key_pair, 1, 0, nullptr); + nullptr, &pem_key_cert_pair, 1, 0, nullptr); + grpc_slice_unref(cert_slice); + grpc_slice_unref(key_slice); grpc_server_add_secure_http2_port(server, addr, ssl_creds); grpc_server_credentials_release(ssl_creds); } static grpc_channel* secure_test_create_channel(const char* addr) { + grpc_slice ca_slice; + GPR_ASSERT(GRPC_LOG_IF_ERROR("load_file", + grpc_load_file(CA_CERT_PATH, 1, &ca_slice))); + const char* test_root_cert = + reinterpret_cast GRPC_SLICE_START_PTR(ca_slice); grpc_channel_credentials* ssl_creds = grpc_ssl_credentials_create(test_root_cert, nullptr, nullptr, nullptr); + grpc_slice_unref(ca_slice); grpc_arg ssl_name_override = { GRPC_ARG_STRING, const_cast(GRPC_SSL_TARGET_NAME_OVERRIDE_ARG), diff --git a/test/core/util/grpc_fuzzer.bzl b/test/core/util/grpc_fuzzer.bzl index e0a9b78df2f..99594b29e1d 100644 --- a/test/core/util/grpc_fuzzer.bzl +++ b/test/core/util/grpc_fuzzer.bzl @@ -14,12 +14,12 @@ load("//bazel:grpc_build_system.bzl", "grpc_cc_test") -def grpc_fuzzer(name, corpus, srcs = [], deps = [], size = "large", **kwargs): +def grpc_fuzzer(name, corpus, srcs = [], deps = [], data = [], size = "large", **kwargs): grpc_cc_test( name = name, srcs = srcs, deps = deps + ["//test/core/util:fuzzer_corpus_test"], - data = native.glob([corpus + "/**"]), + data = data + native.glob([corpus + "/**"]), external_deps = [ "gtest", ], From 7d82170ec027e16a6015541cdb812660060213a3 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 14 Apr 2020 12:56:51 -0700 Subject: [PATCH 73/73] Remove unneeded assignment --- src/core/lib/surface/server.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/lib/surface/server.cc b/src/core/lib/surface/server.cc index e6c81d9d460..08b15ac001a 100644 --- a/src/core/lib/surface/server.cc +++ b/src/core/lib/surface/server.cc @@ -577,7 +577,6 @@ static void publish_new_rpc(void* arg, grpc_error* error) { rm->pending_tail->pending_next = calld; rm->pending_tail = calld; } - calld->pending_next = nullptr; gpr_mu_unlock(&server->mu_call); }