diff --git a/Makefile b/Makefile index 662cdd98174..06a9467e70c 100644 --- a/Makefile +++ b/Makefile @@ -1669,8 +1669,6 @@ PUBLIC_HEADERS_C += \ include/grpc/support/sync_posix.h \ include/grpc/support/sync_win32.h \ include/grpc/support/thd.h \ - include/grpc/support/thd_posix.h \ - include/grpc/support/thd_win32.h \ include/grpc/support/time.h \ include/grpc/support/time_posix.h \ include/grpc/support/time_win32.h \ diff --git a/build.json b/build.json index f075df3ad1c..9248d47b293 100644 --- a/build.json +++ b/build.json @@ -224,8 +224,6 @@ "include/grpc/support/sync_posix.h", "include/grpc/support/sync_win32.h", "include/grpc/support/thd.h", - "include/grpc/support/thd_posix.h", - "include/grpc/support/thd_win32.h", "include/grpc/support/time.h", "include/grpc/support/time_posix.h", "include/grpc/support/time_win32.h", diff --git a/include/grpc/support/thd.h b/include/grpc/support/thd.h index 91e0c9f16f5..92d40b44752 100644 --- a/include/grpc/support/thd.h +++ b/include/grpc/support/thd.h @@ -44,18 +44,12 @@ #include -#if defined(GPR_POSIX_SYNC) -#include -#elif defined(GPR_WIN32) -#include -#else -#error could not determine platform for thd -#endif - #ifdef __cplusplus extern "C" { #endif +typedef gpr_uint64 gpr_thd_id; + /* Thread creation options. */ typedef struct { int flags; /* Flags below can be set here. Default value 0. */ @@ -72,6 +66,9 @@ int gpr_thd_new(gpr_thd_id *t, void (*thd_body)(void *arg), void *arg, /* Return a gpr_thd_options struct with all fields set to defaults. */ gpr_thd_options gpr_thd_options_default(void); +/* Returns the identifier of the current thread. */ +gpr_thd_id gpr_thd_currentid(void); + #ifdef __cplusplus } #endif diff --git a/include/grpc/support/thd_posix.h b/include/grpc/support/thd_posix.h deleted file mode 100644 index b688e45bc54..00000000000 --- a/include/grpc/support/thd_posix.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * - * Copyright 2014, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#ifndef __GRPC_SUPPORT_THD_POSIX_H__ -#define __GRPC_SUPPORT_THD_POSIX_H__ -/* Posix variant of gpr_thd_platform.h. */ - -#include - -typedef pthread_t gpr_thd_id; - -#endif /* __GRPC_SUPPORT_THD_POSIX_H__ */ diff --git a/include/grpc/support/thd_win32.h b/include/grpc/support/thd_win32.h deleted file mode 100644 index b4ab3c7271f..00000000000 --- a/include/grpc/support/thd_win32.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * - * Copyright 2014, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#ifndef __GRPC_SUPPORT_THD_WIN32_H__ -#define __GRPC_SUPPORT_THD_WIN32_H__ - -/* Win32 variant of gpr_thd_platform.h */ - -#include -#include - -typedef int gpr_thd_id; - -#endif /* __GRPC_SUPPORT_THD_WIN32_H__ */ diff --git a/src/core/support/thd_posix.c b/src/core/support/thd_posix.c index bac1d9c2203..74ca9424bbc 100644 --- a/src/core/support/thd_posix.c +++ b/src/core/support/thd_posix.c @@ -62,17 +62,19 @@ int gpr_thd_new(gpr_thd_id *t, void (*thd_body)(void *arg), void *arg, const gpr_thd_options *options) { int thread_started; pthread_attr_t attr; + pthread_t p; struct thd_arg *a = gpr_malloc(sizeof(*a)); a->body = thd_body; a->arg = arg; GPR_ASSERT(pthread_attr_init(&attr) == 0); GPR_ASSERT(pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) == 0); - thread_started = (pthread_create(t, &attr, &thread_body, a) == 0); + thread_started = (pthread_create(&p, &attr, &thread_body, a) == 0); GPR_ASSERT(pthread_attr_destroy(&attr) == 0); if (!thread_started) { gpr_free(a); } + *t = (gpr_thd_id)p; return thread_started; } @@ -82,4 +84,8 @@ gpr_thd_options gpr_thd_options_default(void) { return options; } +gpr_thd_id gpr_thd_currentid(void) { + return (gpr_thd_id)pthread_self(); +} + #endif /* GPR_POSIX_SYNC */ diff --git a/src/core/support/thd_win32.c b/src/core/support/thd_win32.c index 1762f87f3cf..2ee14170484 100644 --- a/src/core/support/thd_win32.c +++ b/src/core/support/thd_win32.c @@ -58,16 +58,18 @@ static DWORD WINAPI thread_body(void *v) { int gpr_thd_new(gpr_thd_id *t, void (*thd_body)(void *arg), void *arg, const gpr_thd_options *options) { HANDLE handle; + DWORD thread_id; struct thd_arg *a = gpr_malloc(sizeof(*a)); a->body = thd_body; a->arg = arg; *t = 0; - handle = CreateThread(NULL, 64 * 1024, thread_body, a, 0, NULL); + handle = CreateThread(NULL, 64 * 1024, thread_body, a, 0, &thread_id); if (handle == NULL) { gpr_free(a); } else { CloseHandle(handle); /* threads are "detached" */ } + *t = (gpr_thd_id)thread_id; return handle != NULL; } @@ -77,4 +79,8 @@ gpr_thd_options gpr_thd_options_default(void) { return options; } +gpr_thd_id gpr_thd_currentid(void) { + return (gpr_thd_id)GetCurrentThreadId(); +} + #endif /* GPR_WIN32 */ diff --git a/src/core/tsi/ssl_transport_security.c b/src/core/tsi/ssl_transport_security.c index 0f8cbccb625..e23421fc157 100644 --- a/src/core/tsi/ssl_transport_security.c +++ b/src/core/tsi/ssl_transport_security.c @@ -37,6 +37,7 @@ #include #include +#include #include #include "src/core/tsi/transport_security.h" @@ -103,11 +104,32 @@ typedef struct { /* --- Library Initialization. ---*/ static gpr_once init_openssl_once = GPR_ONCE_INIT; +static gpr_mu *openssl_mutexes = NULL; + +static void openssl_locking_cb(int mode, int type, const char* file, int line) { + if (mode & CRYPTO_LOCK) { + gpr_mu_lock(&openssl_mutexes[type]); + } else { + gpr_mu_unlock(&openssl_mutexes[type]); + } +} + +static unsigned long openssl_thread_id_cb(void) { + return (unsigned long)gpr_thd_currentid(); +} static void init_openssl(void) { + int i; SSL_library_init(); SSL_load_error_strings(); OpenSSL_add_all_algorithms(); + openssl_mutexes = malloc(CRYPTO_num_locks() * sizeof(gpr_mu)); + GPR_ASSERT(openssl_mutexes != NULL); + for (i = 0; i < CRYPTO_num_locks(); i++) { + gpr_mu_init(&openssl_mutexes[i]); + } + CRYPTO_set_locking_callback(openssl_locking_cb); + CRYPTO_set_id_callback(openssl_thread_id_cb); } /* --- Ssl utils. ---*/ diff --git a/src/cpp/server/async_server_context.cc b/src/cpp/server/async_server_context.cc index 2dd3cd1e8e0..886e794137f 100644 --- a/src/cpp/server/async_server_context.cc +++ b/src/cpp/server/async_server_context.cc @@ -54,7 +54,7 @@ AsyncServerContext::~AsyncServerContext() { grpc_call_destroy(call_); } void AsyncServerContext::Accept(grpc_completion_queue *cq) { GPR_ASSERT(grpc_call_server_accept_old(call_, cq, this) == GRPC_CALL_OK); - GPR_ASSERT(grpc_call_server_end_initial_metadata_old(call_, 0) == + GPR_ASSERT(grpc_call_server_end_initial_metadata_old(call_, GRPC_WRITE_BUFFER_HINT) == GRPC_CALL_OK); } diff --git a/src/cpp/server/server_rpc_handler.cc b/src/cpp/server/server_rpc_handler.cc index 061ac1c2f39..bf02de8b805 100644 --- a/src/cpp/server/server_rpc_handler.cc +++ b/src/cpp/server/server_rpc_handler.cc @@ -77,7 +77,7 @@ void ServerRpcHandler::StartRpc() { if (status.IsOk()) { // Send the response if we get an ok status. - async_server_context_->StartWrite(*response, 0); + async_server_context_->StartWrite(*response, GRPC_WRITE_BUFFER_HINT); type = WaitForNextEvent(); if (type != CompletionQueue::SERVER_WRITE_OK) { status = Status(StatusCode::INTERNAL, "Error writing response."); diff --git a/src/php/ext/grpc/call.c b/src/php/ext/grpc/call.c index bd3490b3627..3bc9ce2bead 100644 --- a/src/php/ext/grpc/call.c +++ b/src/php/ext/grpc/call.c @@ -85,73 +85,25 @@ zval *grpc_call_create_metadata_array(int count, grpc_metadata *elements) { memcpy(str_val, elem->value, elem->value_length); if (zend_hash_find(array_hash, str_key, key_len, (void **)data) == SUCCESS) { - switch (Z_TYPE_P(*data)) { - case IS_STRING: - MAKE_STD_ZVAL(inner_array); - array_init(inner_array); - add_next_index_zval(inner_array, *data); - add_assoc_zval(array, str_key, inner_array); - break; - case IS_ARRAY: - inner_array = *data; - break; - default: - zend_throw_exception(zend_exception_get_default(), - "Metadata hash somehow contains wrong types.", - 1 TSRMLS_CC); + if (Z_TYPE_P(*data) != IS_ARRAY) { + zend_throw_exception(zend_exception_get_default(), + "Metadata hash somehow contains wrong types.", + 1 TSRMLS_CC); efree(str_key); efree(str_val); return NULL; } - add_next_index_stringl(inner_array, str_val, elem->value_length, false); + add_next_index_stringl(*data, str_val, elem->value_length, false); } else { - add_assoc_stringl(array, str_key, str_val, elem->value_length, false); + MAKE_STD_ZVAL(inner_array); + array_init(inner_array); + add_next_index_stringl(inner_array, str_val, elem->value_length, false); + add_assoc_zval(array, str_key, inner_array); } } return array; } -int php_grpc_call_add_metadata_array_walk(void *elem TSRMLS_DC, int num_args, - va_list args, - zend_hash_key *hash_key) { - grpc_call_error error_code; - zval **data = (zval **)elem; - grpc_metadata metadata; - grpc_call *call = va_arg(args, grpc_call *); - gpr_uint32 flags = va_arg(args, gpr_uint32); - const char *key; - HashTable *inner_hash; - /* We assume that either two args were passed, and we are in the recursive - case (and the second argument is the key), or one arg was passed and - hash_key is the string key. */ - if (num_args > 2) { - key = va_arg(args, const char *); - } else { - /* TODO(mlumish): If possible, check that hash_key is a string */ - key = hash_key->arKey; - } - switch (Z_TYPE_P(*data)) { - case IS_STRING: - metadata.key = (char *)key; - metadata.value = Z_STRVAL_P(*data); - metadata.value_length = Z_STRLEN_P(*data); - error_code = grpc_call_add_metadata_old(call, &metadata, 0u); - MAYBE_THROW_CALL_ERROR(add_metadata, error_code); - break; - case IS_ARRAY: - inner_hash = Z_ARRVAL_P(*data); - zend_hash_apply_with_arguments(inner_hash TSRMLS_CC, - php_grpc_call_add_metadata_array_walk, 3, - call, flags, key); - break; - default: - zend_throw_exception(zend_exception_get_default(), - "Metadata hash somehow contains wrong types.", - 1 TSRMLS_CC); - } - return ZEND_HASH_APPLY_KEEP; -} - /** * Constructs a new instance of the Call class. * @param Channel $channel The channel to associate the call with. Must not be @@ -204,8 +156,18 @@ PHP_METHOD(Call, __construct) { PHP_METHOD(Call, add_metadata) { wrapped_grpc_call *call = (wrapped_grpc_call *)zend_object_store_get_object(getThis() TSRMLS_CC); + grpc_metadata metadata; + grpc_call_error error_code; zval *array; + zval **inner_array; + zval **value; HashTable *array_hash; + HashPosition array_pointer; + HashTable *inner_array_hash; + HashPosition inner_array_pointer; + char *key; + uint key_len; + ulong index; long flags = 0; /* "a|l" == 1 array, 1 optional long */ if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "a|l", &array, &flags) == @@ -216,9 +178,41 @@ PHP_METHOD(Call, add_metadata) { return; } array_hash = Z_ARRVAL_P(array); - zend_hash_apply_with_arguments(array_hash TSRMLS_CC, - php_grpc_call_add_metadata_array_walk, 2, - call->wrapped, (gpr_uint32)flags); + for (zend_hash_internal_pointer_reset_ex(array_hash, &array_pointer); + zend_hash_get_current_data_ex(array_hash, (void**)&inner_array, + &array_pointer) == SUCCESS; + zend_hash_move_forward_ex(array_hash, &array_pointer)) { + if (zend_hash_get_current_key_ex(array_hash, &key, &key_len, &index, 0, + &array_pointer) != HASH_KEY_IS_STRING) { + zend_throw_exception(spl_ce_InvalidArgumentException, + "metadata keys must be strings", 1 TSRMLS_CC); + return; + } + if (Z_TYPE_P(*inner_array) != IS_ARRAY) { + zend_throw_exception(spl_ce_InvalidArgumentException, + "metadata values must be arrays", + 1 TSRMLS_CC); + return; + } + inner_array_hash = Z_ARRVAL_P(*inner_array); + for (zend_hash_internal_pointer_reset_ex(inner_array_hash, + &inner_array_pointer); + zend_hash_get_current_data_ex(inner_array_hash, (void**)&value, + &inner_array_pointer) == SUCCESS; + zend_hash_move_forward_ex(inner_array_hash, &inner_array_pointer)) { + if (Z_TYPE_P(*value) != IS_STRING) { + zend_throw_exception(spl_ce_InvalidArgumentException, + "metadata values must be arrays of strings", + 1 TSRMLS_CC); + return; + } + metadata.key = key; + metadata.value = Z_STRVAL_P(*value); + metadata.value_length = Z_STRLEN_P(*value); + error_code = grpc_call_add_metadata_old(call->wrapped, &metadata, 0u); + MAYBE_THROW_CALL_ERROR(add_metadata, error_code); + } + } } /** diff --git a/src/php/ext/grpc/call.h b/src/php/ext/grpc/call.h index 232c5d7cf29..ba1f1e797fb 100644 --- a/src/php/ext/grpc/call.h +++ b/src/php/ext/grpc/call.h @@ -19,6 +19,7 @@ zend_throw_exception(spl_ce_LogicException, \ #func_name " was called incorrectly", \ (long)error_code TSRMLS_CC); \ + return; \ } \ } while (0) diff --git a/src/php/ext/grpc/server.c b/src/php/ext/grpc/server.c index bc4fcf07c9c..47ea38db0c9 100644 --- a/src/php/ext/grpc/server.c +++ b/src/php/ext/grpc/server.c @@ -146,7 +146,7 @@ PHP_METHOD(Server, add_http2_port) { "add_http2_port expects a string", 1 TSRMLS_CC); return; } - RETURN_BOOL(grpc_server_add_http2_port(server->wrapped, addr)); + RETURN_LONG(grpc_server_add_http2_port(server->wrapped, addr)); } PHP_METHOD(Server, add_secure_http2_port) { @@ -161,7 +161,7 @@ PHP_METHOD(Server, add_secure_http2_port) { "add_http2_port expects a string", 1 TSRMLS_CC); return; } - RETURN_BOOL(grpc_server_add_secure_http2_port(server->wrapped, addr)); + RETURN_LONG(grpc_server_add_secure_http2_port(server->wrapped, addr)); } /** diff --git a/src/php/tests/interop/interop_client.php b/src/php/tests/interop/interop_client.php index d1f994a84b6..5266e9a9fac 100755 --- a/src/php/tests/interop/interop_client.php +++ b/src/php/tests/interop/interop_client.php @@ -149,6 +149,25 @@ function pingPong($stub) { 'Call did not complete successfully'); } +function cancelAfterFirstResponse($stub) { + $call = $stub->FullDuplexCall(); + $request = new grpc\testing\StreamingOutputCallRequest(); + $request->setResponseType(grpc\testing\PayloadType::COMPRESSABLE); + $response_parameters = new grpc\testing\ResponseParameters(); + $response_parameters->setSize(31415); + $request->addResponseParameters($response_parameters); + $payload = new grpc\testing\Payload(); + $payload->setBody(str_repeat("\0", 27182)); + $request->setPayload($payload); + + $call->write($request); + $response = $call->read(); + + $call->cancel(); + hardAssert($call->getStatus()->code === Grpc\STATUS_CANCELLED, + 'Call status was not CANCELLED'); +} + $args = getopt('', array('server_host:', 'server_port:', 'test_case:')); if (!array_key_exists('server_host', $args) || !array_key_exists('server_port', $args) || @@ -187,4 +206,6 @@ switch($args['test_case']) { case 'ping_pong': pingPong($stub); break; + case 'cancel_after_first_response': + cancelAfterFirstResponse($stub); } \ No newline at end of file diff --git a/src/php/tests/unit_tests/CallTest.php b/src/php/tests/unit_tests/CallTest.php index 795831cb65c..8f709b7e9cc 100755 --- a/src/php/tests/unit_tests/CallTest.php +++ b/src/php/tests/unit_tests/CallTest.php @@ -1,16 +1,17 @@ add_http2_port('localhost:9001'); + self::$port = self::$server->add_http2_port('0.0.0.0:0'); } public function setUp() { $this->cq = new Grpc\CompletionQueue(); - $this->channel = new Grpc\Channel('localhost:9001', []); + $this->channel = new Grpc\Channel('localhost:' . self::$port, []); $this->call = new Grpc\Call($this->channel, '/foo', Grpc\Timeval::inf_future()); @@ -46,7 +47,7 @@ class CallTest extends PHPUnit_Framework_TestCase{ } public function testAddSingleMetadata() { - $this->call->add_metadata(['key' => 'value'], 0); + $this->call->add_metadata(['key' => ['value']], 0); /* Dummy assert: Checks that the previous call completed without error */ $this->assertTrue(true); } @@ -59,7 +60,7 @@ class CallTest extends PHPUnit_Framework_TestCase{ public function testAddSingleAndMultiValueMetadata() { $this->call->add_metadata( - ['key1' => 'value1', + ['key1' => ['value1'], 'key2' => ['value2', 'value3']], 0); /* Dummy assert: Checks that the previous call completed without error */ $this->assertTrue(true); diff --git a/src/php/tests/unit_tests/EndToEndTest.php b/src/php/tests/unit_tests/EndToEndTest.php index a2d8029b047..05104c0e121 100755 --- a/src/php/tests/unit_tests/EndToEndTest.php +++ b/src/php/tests/unit_tests/EndToEndTest.php @@ -1,13 +1,11 @@ client_queue = new Grpc\CompletionQueue(); $this->server_queue = new Grpc\CompletionQueue(); $this->server = new Grpc\Server($this->server_queue, []); - $address = '127.0.0.1:' . getNewPort(); - $this->server->add_http2_port($address); - $this->channel = new Grpc\Channel($address, []); + $port = $this->server->add_http2_port('0.0.0.0:0'); + $this->channel = new Grpc\Channel('localhost:' . $port, []); } public function tearDown() { diff --git a/src/php/tests/unit_tests/SecureEndToEndTest.php b/src/php/tests/unit_tests/SecureEndToEndTest.php index 7ba4984bd86..5e95b11b44e 100755 --- a/src/php/tests/unit_tests/SecureEndToEndTest.php +++ b/src/php/tests/unit_tests/SecureEndToEndTest.php @@ -11,10 +11,9 @@ class SecureEndToEndTest extends PHPUnit_Framework_TestCase{ file_get_contents(dirname(__FILE__) . '/../data/server1.pem')); $this->server = new Grpc\Server($this->server_queue, ['credentials' => $server_credentials]); - $address = '127.0.0.1:' . getNewPort(); - $this->server->add_secure_http2_port($address); + $port = $this->server->add_secure_http2_port('0.0.0.0:0'); $this->channel = new Grpc\Channel( - $address, + 'localhost:' . $port, [ 'grpc.ssl_target_name_override' => 'foo.test.google.com', 'credentials' => $credentials diff --git a/src/php/tests/util/port_picker.php b/src/php/tests/util/port_picker.php deleted file mode 100755 index d869d8b0a49..00000000000 --- a/src/php/tests/util/port_picker.php +++ /dev/null @@ -1,6 +0,0 @@ -&2 return 1 } - - [[ -n $1 ]] && { # server_type - case $1 in - cxx) grpc_port=8010 ;; - go) grpc_port=8020 ;; - java) grpc_port=8030 ;; - node) grpc_port=8040 ;; - python) grpc_port=8050 ;; - ruby) grpc_port=8060 ;; - *) echo "bad server_type: $1" 1>&2; return 1 ;; - esac - docker_label="grpc/$1" - docker_name="grpc_interop_$1" - shift - } || { - echo "$FUNCNAME: missing arg: server_type" 1>&2 - return 1 - } } -# Launches a server on a docker instance. + +# Shows servers on a docker instance. # # call-seq; -# grpc_launch_server +# grpc_show_servers +# E.g +# grpc_show_server grpc-docker-server # -# Runs the server_type on a GCE instance running docker with server_name -grpc_launch_server() { +# Shows the grpc servers on the GCE instance +grpc_show_servers() { # declare vars local so that they don't pollute the shell environment # where they this func is used. local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone - # set by grpc_launch_server_args - local docker_label docker_name host grpc_port + # set by _grpc_show_servers + local host # set the project zone and check that all necessary args are provided - _grpc_set_project_and_zone -f grpc_launch_server_args "$@" || return 1 + _grpc_set_project_and_zone -f _grpc_show_servers_args "$@" || return 1 gce_has_instance $grpc_project $host || return 1; - cmd="sudo docker kill $docker_name > /dev/null 2>&1; " - cmd+="sudo docker rm $docker_name > /dev/null 2>&1; " - cmd+="sudo docker run -d --name $docker_name" - cmd+=" -p $grpc_port:$grpc_port $docker_label" - local project_opt="--project $grpc_project" - local zone_opt="--zone $grpc_zone" + local cmd="sudo docker ps | grep grpc_" local ssh_cmd="bash -l -c \"$cmd\"" echo "will run:" echo " $ssh_cmd" echo "on $host" - [[ $dry_run == 1 ]] && return 0 # don't run the command on a dry run + [[ $dry_run == 1 ]] && continue # don't run the command on a dry run gcloud compute $project_opt ssh $zone_opt $host --command "$cmd" } +_grpc_launch_servers_args() { + [[ -n $1 ]] && { # host + host=$1 + shift + } || { + echo "$FUNCNAME: missing arg: host" 1>&2 + return 1 + } + [[ -n $1 ]] && { + servers="$@" + } || { + servers="cxx java go node ruby" + echo "$FUNCNAME: no servers specified, will launch defaults '$servers'" + } +} + +# Launches servers on a docker instance. +# +# call-seq; +# grpc_launch_servers [server1 server2 ...] +# E.g +# grpc_launch_server grpc-docker-server ruby node +# +# Restarts all the specified servers on the GCE instance +# If no servers are specified, it launches all known servers +grpc_launch_servers() { + # declare vars local so that they don't pollute the shell environment + # where they this func is used. + local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone + # set by _grpc_launch_servers_args + local host servers + + # set the project zone and check that all necessary args are provided + _grpc_set_project_and_zone -f _grpc_launch_servers_args "$@" || return 1 + gce_has_instance $grpc_project $host || return 1; + + # launch each of the servers in turn + for server in $servers + do + local grpc_port + case $server in + cxx) grpc_port=8010 ;; + go) grpc_port=8020 ;; + java) grpc_port=8030 ;; + node) grpc_port=8040 ;; + python) grpc_port=8050 ;; + ruby) grpc_port=8060 ;; + *) echo "bad server_type: $1" 1>&2; return 1 ;; + esac + local docker_label="grpc/$server" + local docker_name="grpc_interop_$server" + + cmd="sudo docker kill $docker_name > /dev/null 2>&1; " + cmd+="sudo docker rm $docker_name > /dev/null 2>&1; " + cmd+="sudo docker run -d --name $docker_name" + cmd+=" -p $grpc_port:$grpc_port $docker_label" + local project_opt="--project $grpc_project" + local zone_opt="--zone $grpc_zone" + local ssh_cmd="bash -l -c \"$cmd\"" + echo "will run:" + echo " $ssh_cmd" + echo "on $host" + [[ $dry_run == 1 ]] && return 0 # don't run the command on a dry run + gcloud compute $project_opt ssh $zone_opt $host --command "$cmd" + done +} + # Runs a test command on a docker instance. # # call-seq: @@ -822,7 +869,6 @@ grpc_interop_gen_ruby_cmd() { echo $the_cmd } - # constructs the full dockerized java interop test cmd. # # call-seq: @@ -832,12 +878,43 @@ grpc_cloud_prod_gen_ruby_cmd() { local cmd_prefix="sudo docker run grpc/ruby bin/bash -l -c" local test_script="/var/local/git/grpc/src/ruby/bin/interop/interop_client.rb" local test_script+=" --use_tls" - local gfe_flags=" --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com" + local gfe_flags=$(_grpc_prod_gfe_flags) local env_prefix="SSL_CERT_FILE=/cacerts/roots.pem" local the_cmd="$cmd_prefix '$env_prefix ruby $test_script $gfe_flags $@'" echo $the_cmd } +# constructs the full dockerized ruby service_account auth interop test cmd. +# +# call-seq: +# flags= .... # generic flags to include the command +# cmd=$($grpc_gen_test_cmd $flags) +grpc_cloud_prod_auth_service_account_creds_gen_ruby_cmd() { + local cmd_prefix="sudo docker run grpc/ruby bin/bash -l -c"; + local test_script="/var/local/git/grpc/src/ruby/bin/interop/interop_client.rb" + local test_script+=" --use_tls" + local gfe_flags=$(_grpc_prod_gfe_flags) + local added_gfe_flags=$(_grpc_svc_acc_test_flags) + local env_prefix="SSL_CERT_FILE=/cacerts/roots.pem" + local the_cmd="$cmd_prefix '$env_prefix ruby $test_script $gfe_flags $added_gfe_flag $@'" + echo $the_cmd +} + +# constructs the full dockerized ruby gce auth interop test cmd. +# +# call-seq: +# flags= .... # generic flags to include the command +# cmd=$($grpc_gen_test_cmd $flags) +grpc_cloud_prod_auth_compute_engine_creds_gen_ruby_cmd() { + local cmd_prefix="sudo docker run grpc/ruby bin/bash -l -c"; + local test_script="/var/local/git/grpc/src/ruby/bin/interop/interop_client.rb" + local test_script+=" --use_tls" + local gfe_flags=$(_grpc_prod_gfe_flags) + local added_gfe_flags=$(_grpc_gce_test_flags) + local env_prefix="SSL_CERT_FILE=/cacerts/roots.pem" + local the_cmd="$cmd_prefix '$env_prefix ruby $test_script $gfe_flags $added_gfe_flag $@'" + echo $the_cmd +} # constructs the full dockerized Go interop test cmd. # @@ -888,7 +965,7 @@ grpc_cloud_prod_gen_java_cmd() { local cmd_prefix="sudo docker run grpc/java"; local test_script="/var/local/git/grpc-java/run-test-client.sh"; local test_script+=" --use_tls=true" - local gfe_flags=" --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com" + local gfe_flags=$(_grpc_prod_gfe_flags) local the_cmd="$cmd_prefix $test_script $gfe_flags $@"; echo $the_cmd } @@ -909,19 +986,11 @@ grpc_interop_gen_php_cmd() { echo $the_cmd } -# constructs the full dockerized cpp interop test cmd. -# +# constructs the full dockerized node interop test cmd. # # call-seq: # flags= .... # generic flags to include the command # cmd=$($grpc_gen_test_cmd $flags) -grpc_interop_gen_cxx_cmd() { - local cmd_prefix="sudo docker run grpc/cxx"; - local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl"; - local the_cmd="$cmd_prefix $test_script $@"; - echo $the_cmd -} - grpc_interop_gen_node_cmd() { local cmd_prefix="sudo docker run grpc/node"; local test_script="/usr/bin/nodejs /var/local/git/grpc/src/node/interop/interop_client.js --use_tls=true"; @@ -931,46 +1000,70 @@ grpc_interop_gen_node_cmd() { # constructs the full dockerized cpp interop test cmd. # +# call-seq: +# flags= .... # generic flags to include the command +# cmd=$($grpc_gen_test_cmd $flags) +grpc_interop_gen_cxx_cmd() { + local cmd_prefix="sudo docker run grpc/cxx"; + local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl"; + local the_cmd="$cmd_prefix $test_script $@"; + echo $the_cmd +} + +# constructs the full dockerized cpp gce=>prod interop test cmd. # # call-seq: # flags= .... # generic flags to include the command # cmd=$($grpc_gen_test_cmd $flags) grpc_cloud_prod_gen_cxx_cmd() { local cmd_prefix="sudo docker run grpc/cxx"; - local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl"; - local gfe_flags=" --use_prod_roots --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com" + local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl --use_prod_roots"; + local gfe_flags=$(_grpc_prod_gfe_flags) local the_cmd="$cmd_prefix $test_script $gfe_flags $@"; echo $the_cmd } -# constructs the full dockerized cpp interop test cmd. -# +# constructs the full dockerized cpp service_account auth interop test cmd. # # call-seq: # flags= .... # generic flags to include the command # cmd=$($grpc_gen_test_cmd $flags) grpc_cloud_prod_auth_service_account_creds_gen_cxx_cmd() { local cmd_prefix="sudo docker run grpc/cxx"; - local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl"; - local gfe_flags=" --use_prod_roots --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com" - local added_gfe_flags=" --service_account_key_file=/service_account/stubbyCloudTestingTest-7dd63462c60c.json --oauth_scope=https://www.googleapis.com/auth/xapi.zoo" + local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl --use_prod_roots"; + local gfe_flags=$(_grpc_prod_gfe_flags) + local added_gfe_flags=$(_grpc_svc_acc_test_flags) local the_cmd="$cmd_prefix $test_script $gfe_flags $added_gfe_flags $@"; echo $the_cmd } -# constructs the full dockerized cpp interop test cmd. -# +# constructs the full dockerized cpp gce auth interop test cmd. # # call-seq: # flags= .... # generic flags to include the command # cmd=$($grpc_gen_test_cmd $flags) grpc_cloud_prod_auth_compute_engine_creds_gen_cxx_cmd() { local cmd_prefix="sudo docker run grpc/cxx"; - local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl"; - local gfe_flags=" --use_prod_roots --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com" - local added_gfe_flags=" --default_service_account=155450119199-r5aaqa2vqoa9g5mv2m6s3m1l293rlmel@developer.gserviceaccount.com --oauth_scope=https://www.googleapis.com/auth/xapi.zoo" + local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl --use_prod_roots"; + local gfe_flags=$(_grpc_prod_gfe_flags) + local added_gfe_flags=$(_grpc_gce_test_flags) local the_cmd="$cmd_prefix $test_script $gfe_flags $added_gfe_flags $@"; echo $the_cmd } -# TODO(grpc-team): add grpc_interop_gen_xxx_cmd for python|nodejs +# outputs the flags passed to gfe tests +_grpc_prod_gfe_flags() { + echo " --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com" +} + +# outputs the flags passed to the service account auth tests +_grpc_svc_acc_test_flags() { + echo " --service_account_key_file=/service_account/stubbyCloudTestingTest-7dd63462c60c.json --oauth_scope=https://www.googleapis.com/auth/xapi.zoo" +} + +# outputs the flags passed to the gcloud auth tests +_grpc_gce_test_flags() { + echo " --default_service_account=155450119199-r5aaqa2vqoa9g5mv2m6s3m1l293rlmel@developer.gserviceaccount.com --oauth_scope=https://www.googleapis.com/auth/xapi.zoo" +} + +# TODO(grpc-team): add grpc_interop_gen_xxx_cmd for python diff --git a/tools/gce_setup/new_grpc_docker_builder_on_startup.sh b/tools/gce_setup/new_grpc_docker_builder_on_startup.sh index 4b20a8be7d7..cfd05415a0e 100755 --- a/tools/gce_setup/new_grpc_docker_builder_on_startup.sh +++ b/tools/gce_setup/new_grpc_docker_builder_on_startup.sh @@ -41,7 +41,82 @@ _source_gs_script() { source $script_path } +# Args: +# $1: numerator +# $2: denominator +# $3: threshold (optional; defaults to $THRESHOLD) +# +# Returns: +# 1 if (numerator / denominator > threshold) +# 0 otherwise +_gce_disk_cmp_ratio() { + local DEFAULT_THRESHOLD="1.1" + local numer="${1}" + local denom="${2}" + local threshold="${3:-${DEFAULT_THRESHOLD}}" + + if `which python > /dev/null 2>&1`; then + python -c "print(1 if (1. * ${numer} / ${denom} > ${threshold}) else 0)" + else + echo "Can't find python; calculation not done." 1>&2 + return 1 + fi +} + +# Repartitions the disk or resizes the file system, depending on the current +# state of the partition table. +# +# Automates the process described in +# - https://cloud.google.com/compute/docs/disks/persistent-disks#repartitionrootpd +_gce_disk_maybe_resize_then_reboot() { + # Determine the size in blocks, of the whole disk and the first partition. + local dev_sda="$(fdisk -s /dev/sda)" + local dev_sda1="$(fdisk -s /dev/sda1)" + local dev_sda1_start="$(sudo fdisk -l /dev/sda | grep /dev/sda1 | sed -e 's/ \+/ /g' | cut -d' ' -f 3)" + + # Use fdisk to + # - first see if the partion 1 is using as much of the disk as it should + # - then to resize the partition if it's not + # + # fdisk(1) flags: + # -c: disable DOS compatibility mode + # -u: change display mode to sectors (from cylinders) + # + # fdisk(1) commands: + # d: delete partition (automatically selects the first one) + # n: new partition + # p: primary + # 1: partition number + # $dev_sda1_start: specify the value for the start sector, the default may be incorrect + # <1 blank lines>: accept the defaults for end sectors + # w: write partition table + if [ $(_gce_disk_cmp_ratio "${dev_sda}" "${dev_sda1}") -eq 1 ]; then + echo "$FUNCNAME: Updating the partition table to use full ${dev_sda} instead ${dev_sda1}" + cat < - - diff --git a/vsprojects/vs2013/gpr.vcxproj.filters b/vsprojects/vs2013/gpr.vcxproj.filters index 013ed4b3c99..e385efcfb24 100644 --- a/vsprojects/vs2013/gpr.vcxproj.filters +++ b/vsprojects/vs2013/gpr.vcxproj.filters @@ -135,12 +135,6 @@ include\grpc\support - - include\grpc\support - - - include\grpc\support - include\grpc\support