Merge github.com:google/grpc into async-api-new

pull/381/head
Craig Tiller 10 years ago
commit d77c92bd98
  1. 2
      Makefile
  2. 2
      build.json
  3. 13
      include/grpc/support/thd.h
  4. 42
      include/grpc/support/thd_posix.h
  5. 44
      include/grpc/support/thd_win32.h
  6. 8
      src/core/support/thd_posix.c
  7. 8
      src/core/support/thd_win32.c
  8. 22
      src/core/tsi/ssl_transport_security.c
  9. 2
      src/cpp/server/async_server_context.cc
  10. 2
      src/cpp/server/server_rpc_handler.cc
  11. 114
      src/php/ext/grpc/call.c
  12. 1
      src/php/ext/grpc/call.h
  13. 4
      src/php/ext/grpc/server.c
  14. 21
      src/php/tests/interop/interop_client.php
  15. 9
      src/php/tests/unit_tests/CallTest.php
  16. 6
      src/php/tests/unit_tests/EndToEndTest.php
  17. 5
      src/php/tests/unit_tests/SecureEndToEndTest.php
  18. 6
      src/php/tests/util/port_picker.php
  19. 12
      tools/dockerfile/grpc_ruby/Dockerfile
  20. 209
      tools/gce_setup/grpc_docker.sh
  21. 75
      tools/gce_setup/new_grpc_docker_builder_on_startup.sh
  22. 5
      tools/gce_setup/shared_startup_funcs.sh
  23. 2
      vsprojects/vs2013/gpr.vcxproj
  24. 6
      vsprojects/vs2013/gpr.vcxproj.filters

@ -1669,8 +1669,6 @@ PUBLIC_HEADERS_C += \
include/grpc/support/sync_posix.h \
include/grpc/support/sync_win32.h \
include/grpc/support/thd.h \
include/grpc/support/thd_posix.h \
include/grpc/support/thd_win32.h \
include/grpc/support/time.h \
include/grpc/support/time_posix.h \
include/grpc/support/time_win32.h \

@ -224,8 +224,6 @@
"include/grpc/support/sync_posix.h",
"include/grpc/support/sync_win32.h",
"include/grpc/support/thd.h",
"include/grpc/support/thd_posix.h",
"include/grpc/support/thd_win32.h",
"include/grpc/support/time.h",
"include/grpc/support/time_posix.h",
"include/grpc/support/time_win32.h",

@ -44,18 +44,12 @@
#include <grpc/support/port_platform.h>
#if defined(GPR_POSIX_SYNC)
#include <grpc/support/thd_posix.h>
#elif defined(GPR_WIN32)
#include <grpc/support/thd_win32.h>
#else
#error could not determine platform for thd
#endif
#ifdef __cplusplus
extern "C" {
#endif
typedef gpr_uint64 gpr_thd_id;
/* Thread creation options. */
typedef struct {
int flags; /* Flags below can be set here. Default value 0. */
@ -72,6 +66,9 @@ int gpr_thd_new(gpr_thd_id *t, void (*thd_body)(void *arg), void *arg,
/* Return a gpr_thd_options struct with all fields set to defaults. */
gpr_thd_options gpr_thd_options_default(void);
/* Returns the identifier of the current thread. */
gpr_thd_id gpr_thd_currentid(void);
#ifdef __cplusplus
}
#endif

@ -1,42 +0,0 @@
/*
*
* Copyright 2014, Google Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Google Inc. nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
#ifndef __GRPC_SUPPORT_THD_POSIX_H__
#define __GRPC_SUPPORT_THD_POSIX_H__
/* Posix variant of gpr_thd_platform.h. */
#include <pthread.h>
typedef pthread_t gpr_thd_id;
#endif /* __GRPC_SUPPORT_THD_POSIX_H__ */

@ -1,44 +0,0 @@
/*
*
* Copyright 2014, Google Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Google Inc. nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
#ifndef __GRPC_SUPPORT_THD_WIN32_H__
#define __GRPC_SUPPORT_THD_WIN32_H__
/* Win32 variant of gpr_thd_platform.h */
#include <windows.h>
#include <grpc/support/atm.h>
typedef int gpr_thd_id;
#endif /* __GRPC_SUPPORT_THD_WIN32_H__ */

@ -62,17 +62,19 @@ int gpr_thd_new(gpr_thd_id *t, void (*thd_body)(void *arg), void *arg,
const gpr_thd_options *options) {
int thread_started;
pthread_attr_t attr;
pthread_t p;
struct thd_arg *a = gpr_malloc(sizeof(*a));
a->body = thd_body;
a->arg = arg;
GPR_ASSERT(pthread_attr_init(&attr) == 0);
GPR_ASSERT(pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) == 0);
thread_started = (pthread_create(t, &attr, &thread_body, a) == 0);
thread_started = (pthread_create(&p, &attr, &thread_body, a) == 0);
GPR_ASSERT(pthread_attr_destroy(&attr) == 0);
if (!thread_started) {
gpr_free(a);
}
*t = (gpr_thd_id)p;
return thread_started;
}
@ -82,4 +84,8 @@ gpr_thd_options gpr_thd_options_default(void) {
return options;
}
gpr_thd_id gpr_thd_currentid(void) {
return (gpr_thd_id)pthread_self();
}
#endif /* GPR_POSIX_SYNC */

@ -58,16 +58,18 @@ static DWORD WINAPI thread_body(void *v) {
int gpr_thd_new(gpr_thd_id *t, void (*thd_body)(void *arg), void *arg,
const gpr_thd_options *options) {
HANDLE handle;
DWORD thread_id;
struct thd_arg *a = gpr_malloc(sizeof(*a));
a->body = thd_body;
a->arg = arg;
*t = 0;
handle = CreateThread(NULL, 64 * 1024, thread_body, a, 0, NULL);
handle = CreateThread(NULL, 64 * 1024, thread_body, a, 0, &thread_id);
if (handle == NULL) {
gpr_free(a);
} else {
CloseHandle(handle); /* threads are "detached" */
}
*t = (gpr_thd_id)thread_id;
return handle != NULL;
}
@ -77,4 +79,8 @@ gpr_thd_options gpr_thd_options_default(void) {
return options;
}
gpr_thd_id gpr_thd_currentid(void) {
return (gpr_thd_id)GetCurrentThreadId();
}
#endif /* GPR_WIN32 */

@ -37,6 +37,7 @@
#include <grpc/support/log.h>
#include <grpc/support/sync.h>
#include <grpc/support/thd.h>
#include <grpc/support/useful.h>
#include "src/core/tsi/transport_security.h"
@ -103,11 +104,32 @@ typedef struct {
/* --- Library Initialization. ---*/
static gpr_once init_openssl_once = GPR_ONCE_INIT;
static gpr_mu *openssl_mutexes = NULL;
static void openssl_locking_cb(int mode, int type, const char* file, int line) {
if (mode & CRYPTO_LOCK) {
gpr_mu_lock(&openssl_mutexes[type]);
} else {
gpr_mu_unlock(&openssl_mutexes[type]);
}
}
static unsigned long openssl_thread_id_cb(void) {
return (unsigned long)gpr_thd_currentid();
}
static void init_openssl(void) {
int i;
SSL_library_init();
SSL_load_error_strings();
OpenSSL_add_all_algorithms();
openssl_mutexes = malloc(CRYPTO_num_locks() * sizeof(gpr_mu));
GPR_ASSERT(openssl_mutexes != NULL);
for (i = 0; i < CRYPTO_num_locks(); i++) {
gpr_mu_init(&openssl_mutexes[i]);
}
CRYPTO_set_locking_callback(openssl_locking_cb);
CRYPTO_set_id_callback(openssl_thread_id_cb);
}
/* --- Ssl utils. ---*/

@ -54,7 +54,7 @@ AsyncServerContext::~AsyncServerContext() { grpc_call_destroy(call_); }
void AsyncServerContext::Accept(grpc_completion_queue *cq) {
GPR_ASSERT(grpc_call_server_accept_old(call_, cq, this) == GRPC_CALL_OK);
GPR_ASSERT(grpc_call_server_end_initial_metadata_old(call_, 0) ==
GPR_ASSERT(grpc_call_server_end_initial_metadata_old(call_, GRPC_WRITE_BUFFER_HINT) ==
GRPC_CALL_OK);
}

@ -77,7 +77,7 @@ void ServerRpcHandler::StartRpc() {
if (status.IsOk()) {
// Send the response if we get an ok status.
async_server_context_->StartWrite(*response, 0);
async_server_context_->StartWrite(*response, GRPC_WRITE_BUFFER_HINT);
type = WaitForNextEvent();
if (type != CompletionQueue::SERVER_WRITE_OK) {
status = Status(StatusCode::INTERNAL, "Error writing response.");

@ -85,73 +85,25 @@ zval *grpc_call_create_metadata_array(int count, grpc_metadata *elements) {
memcpy(str_val, elem->value, elem->value_length);
if (zend_hash_find(array_hash, str_key, key_len, (void **)data) ==
SUCCESS) {
switch (Z_TYPE_P(*data)) {
case IS_STRING:
MAKE_STD_ZVAL(inner_array);
array_init(inner_array);
add_next_index_zval(inner_array, *data);
add_assoc_zval(array, str_key, inner_array);
break;
case IS_ARRAY:
inner_array = *data;
break;
default:
zend_throw_exception(zend_exception_get_default(),
"Metadata hash somehow contains wrong types.",
1 TSRMLS_CC);
if (Z_TYPE_P(*data) != IS_ARRAY) {
zend_throw_exception(zend_exception_get_default(),
"Metadata hash somehow contains wrong types.",
1 TSRMLS_CC);
efree(str_key);
efree(str_val);
return NULL;
}
add_next_index_stringl(inner_array, str_val, elem->value_length, false);
add_next_index_stringl(*data, str_val, elem->value_length, false);
} else {
add_assoc_stringl(array, str_key, str_val, elem->value_length, false);
MAKE_STD_ZVAL(inner_array);
array_init(inner_array);
add_next_index_stringl(inner_array, str_val, elem->value_length, false);
add_assoc_zval(array, str_key, inner_array);
}
}
return array;
}
int php_grpc_call_add_metadata_array_walk(void *elem TSRMLS_DC, int num_args,
va_list args,
zend_hash_key *hash_key) {
grpc_call_error error_code;
zval **data = (zval **)elem;
grpc_metadata metadata;
grpc_call *call = va_arg(args, grpc_call *);
gpr_uint32 flags = va_arg(args, gpr_uint32);
const char *key;
HashTable *inner_hash;
/* We assume that either two args were passed, and we are in the recursive
case (and the second argument is the key), or one arg was passed and
hash_key is the string key. */
if (num_args > 2) {
key = va_arg(args, const char *);
} else {
/* TODO(mlumish): If possible, check that hash_key is a string */
key = hash_key->arKey;
}
switch (Z_TYPE_P(*data)) {
case IS_STRING:
metadata.key = (char *)key;
metadata.value = Z_STRVAL_P(*data);
metadata.value_length = Z_STRLEN_P(*data);
error_code = grpc_call_add_metadata_old(call, &metadata, 0u);
MAYBE_THROW_CALL_ERROR(add_metadata, error_code);
break;
case IS_ARRAY:
inner_hash = Z_ARRVAL_P(*data);
zend_hash_apply_with_arguments(inner_hash TSRMLS_CC,
php_grpc_call_add_metadata_array_walk, 3,
call, flags, key);
break;
default:
zend_throw_exception(zend_exception_get_default(),
"Metadata hash somehow contains wrong types.",
1 TSRMLS_CC);
}
return ZEND_HASH_APPLY_KEEP;
}
/**
* Constructs a new instance of the Call class.
* @param Channel $channel The channel to associate the call with. Must not be
@ -204,8 +156,18 @@ PHP_METHOD(Call, __construct) {
PHP_METHOD(Call, add_metadata) {
wrapped_grpc_call *call =
(wrapped_grpc_call *)zend_object_store_get_object(getThis() TSRMLS_CC);
grpc_metadata metadata;
grpc_call_error error_code;
zval *array;
zval **inner_array;
zval **value;
HashTable *array_hash;
HashPosition array_pointer;
HashTable *inner_array_hash;
HashPosition inner_array_pointer;
char *key;
uint key_len;
ulong index;
long flags = 0;
/* "a|l" == 1 array, 1 optional long */
if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "a|l", &array, &flags) ==
@ -216,9 +178,41 @@ PHP_METHOD(Call, add_metadata) {
return;
}
array_hash = Z_ARRVAL_P(array);
zend_hash_apply_with_arguments(array_hash TSRMLS_CC,
php_grpc_call_add_metadata_array_walk, 2,
call->wrapped, (gpr_uint32)flags);
for (zend_hash_internal_pointer_reset_ex(array_hash, &array_pointer);
zend_hash_get_current_data_ex(array_hash, (void**)&inner_array,
&array_pointer) == SUCCESS;
zend_hash_move_forward_ex(array_hash, &array_pointer)) {
if (zend_hash_get_current_key_ex(array_hash, &key, &key_len, &index, 0,
&array_pointer) != HASH_KEY_IS_STRING) {
zend_throw_exception(spl_ce_InvalidArgumentException,
"metadata keys must be strings", 1 TSRMLS_CC);
return;
}
if (Z_TYPE_P(*inner_array) != IS_ARRAY) {
zend_throw_exception(spl_ce_InvalidArgumentException,
"metadata values must be arrays",
1 TSRMLS_CC);
return;
}
inner_array_hash = Z_ARRVAL_P(*inner_array);
for (zend_hash_internal_pointer_reset_ex(inner_array_hash,
&inner_array_pointer);
zend_hash_get_current_data_ex(inner_array_hash, (void**)&value,
&inner_array_pointer) == SUCCESS;
zend_hash_move_forward_ex(inner_array_hash, &inner_array_pointer)) {
if (Z_TYPE_P(*value) != IS_STRING) {
zend_throw_exception(spl_ce_InvalidArgumentException,
"metadata values must be arrays of strings",
1 TSRMLS_CC);
return;
}
metadata.key = key;
metadata.value = Z_STRVAL_P(*value);
metadata.value_length = Z_STRLEN_P(*value);
error_code = grpc_call_add_metadata_old(call->wrapped, &metadata, 0u);
MAYBE_THROW_CALL_ERROR(add_metadata, error_code);
}
}
}
/**

@ -19,6 +19,7 @@
zend_throw_exception(spl_ce_LogicException, \
#func_name " was called incorrectly", \
(long)error_code TSRMLS_CC); \
return; \
} \
} while (0)

@ -146,7 +146,7 @@ PHP_METHOD(Server, add_http2_port) {
"add_http2_port expects a string", 1 TSRMLS_CC);
return;
}
RETURN_BOOL(grpc_server_add_http2_port(server->wrapped, addr));
RETURN_LONG(grpc_server_add_http2_port(server->wrapped, addr));
}
PHP_METHOD(Server, add_secure_http2_port) {
@ -161,7 +161,7 @@ PHP_METHOD(Server, add_secure_http2_port) {
"add_http2_port expects a string", 1 TSRMLS_CC);
return;
}
RETURN_BOOL(grpc_server_add_secure_http2_port(server->wrapped, addr));
RETURN_LONG(grpc_server_add_secure_http2_port(server->wrapped, addr));
}
/**

@ -149,6 +149,25 @@ function pingPong($stub) {
'Call did not complete successfully');
}
function cancelAfterFirstResponse($stub) {
$call = $stub->FullDuplexCall();
$request = new grpc\testing\StreamingOutputCallRequest();
$request->setResponseType(grpc\testing\PayloadType::COMPRESSABLE);
$response_parameters = new grpc\testing\ResponseParameters();
$response_parameters->setSize(31415);
$request->addResponseParameters($response_parameters);
$payload = new grpc\testing\Payload();
$payload->setBody(str_repeat("\0", 27182));
$request->setPayload($payload);
$call->write($request);
$response = $call->read();
$call->cancel();
hardAssert($call->getStatus()->code === Grpc\STATUS_CANCELLED,
'Call status was not CANCELLED');
}
$args = getopt('', array('server_host:', 'server_port:', 'test_case:'));
if (!array_key_exists('server_host', $args) ||
!array_key_exists('server_port', $args) ||
@ -187,4 +206,6 @@ switch($args['test_case']) {
case 'ping_pong':
pingPong($stub);
break;
case 'cancel_after_first_response':
cancelAfterFirstResponse($stub);
}

@ -1,16 +1,17 @@
<?php
class CallTest extends PHPUnit_Framework_TestCase{
static $server;
static $port;
public static function setUpBeforeClass() {
$cq = new Grpc\CompletionQueue();
self::$server = new Grpc\Server($cq, []);
self::$server->add_http2_port('localhost:9001');
self::$port = self::$server->add_http2_port('0.0.0.0:0');
}
public function setUp() {
$this->cq = new Grpc\CompletionQueue();
$this->channel = new Grpc\Channel('localhost:9001', []);
$this->channel = new Grpc\Channel('localhost:' . self::$port, []);
$this->call = new Grpc\Call($this->channel,
'/foo',
Grpc\Timeval::inf_future());
@ -46,7 +47,7 @@ class CallTest extends PHPUnit_Framework_TestCase{
}
public function testAddSingleMetadata() {
$this->call->add_metadata(['key' => 'value'], 0);
$this->call->add_metadata(['key' => ['value']], 0);
/* Dummy assert: Checks that the previous call completed without error */
$this->assertTrue(true);
}
@ -59,7 +60,7 @@ class CallTest extends PHPUnit_Framework_TestCase{
public function testAddSingleAndMultiValueMetadata() {
$this->call->add_metadata(
['key1' => 'value1',
['key1' => ['value1'],
'key2' => ['value2', 'value3']], 0);
/* Dummy assert: Checks that the previous call completed without error */
$this->assertTrue(true);

@ -1,13 +1,11 @@
<?php
require __DIR__ . '/../util/port_picker.php';
class EndToEndTest extends PHPUnit_Framework_TestCase{
public function setUp() {
$this->client_queue = new Grpc\CompletionQueue();
$this->server_queue = new Grpc\CompletionQueue();
$this->server = new Grpc\Server($this->server_queue, []);
$address = '127.0.0.1:' . getNewPort();
$this->server->add_http2_port($address);
$this->channel = new Grpc\Channel($address, []);
$port = $this->server->add_http2_port('0.0.0.0:0');
$this->channel = new Grpc\Channel('localhost:' . $port, []);
}
public function tearDown() {

@ -11,10 +11,9 @@ class SecureEndToEndTest extends PHPUnit_Framework_TestCase{
file_get_contents(dirname(__FILE__) . '/../data/server1.pem'));
$this->server = new Grpc\Server($this->server_queue,
['credentials' => $server_credentials]);
$address = '127.0.0.1:' . getNewPort();
$this->server->add_secure_http2_port($address);
$port = $this->server->add_secure_http2_port('0.0.0.0:0');
$this->channel = new Grpc\Channel(
$address,
'localhost:' . $port,
[
'grpc.ssl_target_name_override' => 'foo.test.google.com',
'credentials' => $credentials

@ -1,6 +0,0 @@
<?php
function getNewPort() {
static $port = 10000;
$port += 1;
return $port;
}

@ -6,17 +6,19 @@ RUN cd /var/local/git/grpc \
&& git pull --recurse-submodules \
&& git submodule update --init --recursive
# TODO: remove this, once make install is fixed
RUN touch /var/local/git/grpc/include/grpc/support/string.h
# Build the C core.
RUN make install_c -C /var/local/git/grpc
# Build ruby gRPC and run its tests
RUN /bin/bash -l -c 'cd /var/local/git/grpc/src/ruby && bundle && rake'
# Add a cacerts directory containing the Google root pem file, allowing the ruby client to access the production test instance
# Add a cacerts directory containing the Google root pem file, allowing the
# ruby client to access the production test instance
ADD cacerts cacerts
# Specify the default command such that the interop server runs on its known testing port
# Add a service_account directory containing the auth creds file
ADD service_account service_account
# Specify the default command such that the interop server runs on its known
# testing port
CMD ["/bin/bash", "-l", "-c", "ruby /var/local/git/grpc/src/ruby/bin/interop/interop_server.rb --use_tls --port 8060"]

@ -590,7 +590,7 @@ grpc_sync_images() {
done
}
grpc_launch_server_args() {
_grpc_show_servers_args() {
[[ -n $1 ]] && { # host
host=$1
shift
@ -598,57 +598,104 @@ grpc_launch_server_args() {
echo "$FUNCNAME: missing arg: host" 1>&2
return 1
}
[[ -n $1 ]] && { # server_type
case $1 in
cxx) grpc_port=8010 ;;
go) grpc_port=8020 ;;
java) grpc_port=8030 ;;
node) grpc_port=8040 ;;
python) grpc_port=8050 ;;
ruby) grpc_port=8060 ;;
*) echo "bad server_type: $1" 1>&2; return 1 ;;
esac
docker_label="grpc/$1"
docker_name="grpc_interop_$1"
shift
} || {
echo "$FUNCNAME: missing arg: server_type" 1>&2
return 1
}
}
# Launches a server on a docker instance.
# Shows servers on a docker instance.
#
# call-seq;
# grpc_launch_server <server_name> <server_type>
# grpc_show_servers <server_name>
# E.g
# grpc_show_server grpc-docker-server
#
# Runs the server_type on a GCE instance running docker with server_name
grpc_launch_server() {
# Shows the grpc servers on the GCE instance <server_name>
grpc_show_servers() {
# declare vars local so that they don't pollute the shell environment
# where they this func is used.
local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone
# set by grpc_launch_server_args
local docker_label docker_name host grpc_port
# set by _grpc_show_servers
local host
# set the project zone and check that all necessary args are provided
_grpc_set_project_and_zone -f grpc_launch_server_args "$@" || return 1
_grpc_set_project_and_zone -f _grpc_show_servers_args "$@" || return 1
gce_has_instance $grpc_project $host || return 1;
cmd="sudo docker kill $docker_name > /dev/null 2>&1; "
cmd+="sudo docker rm $docker_name > /dev/null 2>&1; "
cmd+="sudo docker run -d --name $docker_name"
cmd+=" -p $grpc_port:$grpc_port $docker_label"
local project_opt="--project $grpc_project"
local zone_opt="--zone $grpc_zone"
local cmd="sudo docker ps | grep grpc_"
local ssh_cmd="bash -l -c \"$cmd\""
echo "will run:"
echo " $ssh_cmd"
echo "on $host"
[[ $dry_run == 1 ]] && return 0 # don't run the command on a dry run
[[ $dry_run == 1 ]] && continue # don't run the command on a dry run
gcloud compute $project_opt ssh $zone_opt $host --command "$cmd"
}
_grpc_launch_servers_args() {
[[ -n $1 ]] && { # host
host=$1
shift
} || {
echo "$FUNCNAME: missing arg: host" 1>&2
return 1
}
[[ -n $1 ]] && {
servers="$@"
} || {
servers="cxx java go node ruby"
echo "$FUNCNAME: no servers specified, will launch defaults '$servers'"
}
}
# Launches servers on a docker instance.
#
# call-seq;
# grpc_launch_servers <server_name> [server1 server2 ...]
# E.g
# grpc_launch_server grpc-docker-server ruby node
#
# Restarts all the specified servers on the GCE instance <server_name>
# If no servers are specified, it launches all known servers
grpc_launch_servers() {
# declare vars local so that they don't pollute the shell environment
# where they this func is used.
local grpc_zone grpc_project dry_run # set by _grpc_set_project_and_zone
# set by _grpc_launch_servers_args
local host servers
# set the project zone and check that all necessary args are provided
_grpc_set_project_and_zone -f _grpc_launch_servers_args "$@" || return 1
gce_has_instance $grpc_project $host || return 1;
# launch each of the servers in turn
for server in $servers
do
local grpc_port
case $server in
cxx) grpc_port=8010 ;;
go) grpc_port=8020 ;;
java) grpc_port=8030 ;;
node) grpc_port=8040 ;;
python) grpc_port=8050 ;;
ruby) grpc_port=8060 ;;
*) echo "bad server_type: $1" 1>&2; return 1 ;;
esac
local docker_label="grpc/$server"
local docker_name="grpc_interop_$server"
cmd="sudo docker kill $docker_name > /dev/null 2>&1; "
cmd+="sudo docker rm $docker_name > /dev/null 2>&1; "
cmd+="sudo docker run -d --name $docker_name"
cmd+=" -p $grpc_port:$grpc_port $docker_label"
local project_opt="--project $grpc_project"
local zone_opt="--zone $grpc_zone"
local ssh_cmd="bash -l -c \"$cmd\""
echo "will run:"
echo " $ssh_cmd"
echo "on $host"
[[ $dry_run == 1 ]] && return 0 # don't run the command on a dry run
gcloud compute $project_opt ssh $zone_opt $host --command "$cmd"
done
}
# Runs a test command on a docker instance.
#
# call-seq:
@ -822,7 +869,6 @@ grpc_interop_gen_ruby_cmd() {
echo $the_cmd
}
# constructs the full dockerized java interop test cmd.
#
# call-seq:
@ -832,12 +878,43 @@ grpc_cloud_prod_gen_ruby_cmd() {
local cmd_prefix="sudo docker run grpc/ruby bin/bash -l -c"
local test_script="/var/local/git/grpc/src/ruby/bin/interop/interop_client.rb"
local test_script+=" --use_tls"
local gfe_flags=" --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com"
local gfe_flags=$(_grpc_prod_gfe_flags)
local env_prefix="SSL_CERT_FILE=/cacerts/roots.pem"
local the_cmd="$cmd_prefix '$env_prefix ruby $test_script $gfe_flags $@'"
echo $the_cmd
}
# constructs the full dockerized ruby service_account auth interop test cmd.
#
# call-seq:
# flags= .... # generic flags to include the command
# cmd=$($grpc_gen_test_cmd $flags)
grpc_cloud_prod_auth_service_account_creds_gen_ruby_cmd() {
local cmd_prefix="sudo docker run grpc/ruby bin/bash -l -c";
local test_script="/var/local/git/grpc/src/ruby/bin/interop/interop_client.rb"
local test_script+=" --use_tls"
local gfe_flags=$(_grpc_prod_gfe_flags)
local added_gfe_flags=$(_grpc_svc_acc_test_flags)
local env_prefix="SSL_CERT_FILE=/cacerts/roots.pem"
local the_cmd="$cmd_prefix '$env_prefix ruby $test_script $gfe_flags $added_gfe_flag $@'"
echo $the_cmd
}
# constructs the full dockerized ruby gce auth interop test cmd.
#
# call-seq:
# flags= .... # generic flags to include the command
# cmd=$($grpc_gen_test_cmd $flags)
grpc_cloud_prod_auth_compute_engine_creds_gen_ruby_cmd() {
local cmd_prefix="sudo docker run grpc/ruby bin/bash -l -c";
local test_script="/var/local/git/grpc/src/ruby/bin/interop/interop_client.rb"
local test_script+=" --use_tls"
local gfe_flags=$(_grpc_prod_gfe_flags)
local added_gfe_flags=$(_grpc_gce_test_flags)
local env_prefix="SSL_CERT_FILE=/cacerts/roots.pem"
local the_cmd="$cmd_prefix '$env_prefix ruby $test_script $gfe_flags $added_gfe_flag $@'"
echo $the_cmd
}
# constructs the full dockerized Go interop test cmd.
#
@ -888,7 +965,7 @@ grpc_cloud_prod_gen_java_cmd() {
local cmd_prefix="sudo docker run grpc/java";
local test_script="/var/local/git/grpc-java/run-test-client.sh";
local test_script+=" --use_tls=true"
local gfe_flags=" --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com"
local gfe_flags=$(_grpc_prod_gfe_flags)
local the_cmd="$cmd_prefix $test_script $gfe_flags $@";
echo $the_cmd
}
@ -909,19 +986,11 @@ grpc_interop_gen_php_cmd() {
echo $the_cmd
}
# constructs the full dockerized cpp interop test cmd.
#
# constructs the full dockerized node interop test cmd.
#
# call-seq:
# flags= .... # generic flags to include the command
# cmd=$($grpc_gen_test_cmd $flags)
grpc_interop_gen_cxx_cmd() {
local cmd_prefix="sudo docker run grpc/cxx";
local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl";
local the_cmd="$cmd_prefix $test_script $@";
echo $the_cmd
}
grpc_interop_gen_node_cmd() {
local cmd_prefix="sudo docker run grpc/node";
local test_script="/usr/bin/nodejs /var/local/git/grpc/src/node/interop/interop_client.js --use_tls=true";
@ -931,46 +1000,70 @@ grpc_interop_gen_node_cmd() {
# constructs the full dockerized cpp interop test cmd.
#
# call-seq:
# flags= .... # generic flags to include the command
# cmd=$($grpc_gen_test_cmd $flags)
grpc_interop_gen_cxx_cmd() {
local cmd_prefix="sudo docker run grpc/cxx";
local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl";
local the_cmd="$cmd_prefix $test_script $@";
echo $the_cmd
}
# constructs the full dockerized cpp gce=>prod interop test cmd.
#
# call-seq:
# flags= .... # generic flags to include the command
# cmd=$($grpc_gen_test_cmd $flags)
grpc_cloud_prod_gen_cxx_cmd() {
local cmd_prefix="sudo docker run grpc/cxx";
local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl";
local gfe_flags=" --use_prod_roots --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com"
local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl --use_prod_roots";
local gfe_flags=$(_grpc_prod_gfe_flags)
local the_cmd="$cmd_prefix $test_script $gfe_flags $@";
echo $the_cmd
}
# constructs the full dockerized cpp interop test cmd.
#
# constructs the full dockerized cpp service_account auth interop test cmd.
#
# call-seq:
# flags= .... # generic flags to include the command
# cmd=$($grpc_gen_test_cmd $flags)
grpc_cloud_prod_auth_service_account_creds_gen_cxx_cmd() {
local cmd_prefix="sudo docker run grpc/cxx";
local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl";
local gfe_flags=" --use_prod_roots --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com"
local added_gfe_flags=" --service_account_key_file=/service_account/stubbyCloudTestingTest-7dd63462c60c.json --oauth_scope=https://www.googleapis.com/auth/xapi.zoo"
local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl --use_prod_roots";
local gfe_flags=$(_grpc_prod_gfe_flags)
local added_gfe_flags=$(_grpc_svc_acc_test_flags)
local the_cmd="$cmd_prefix $test_script $gfe_flags $added_gfe_flags $@";
echo $the_cmd
}
# constructs the full dockerized cpp interop test cmd.
#
# constructs the full dockerized cpp gce auth interop test cmd.
#
# call-seq:
# flags= .... # generic flags to include the command
# cmd=$($grpc_gen_test_cmd $flags)
grpc_cloud_prod_auth_compute_engine_creds_gen_cxx_cmd() {
local cmd_prefix="sudo docker run grpc/cxx";
local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl";
local gfe_flags=" --use_prod_roots --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com"
local added_gfe_flags=" --default_service_account=155450119199-r5aaqa2vqoa9g5mv2m6s3m1l293rlmel@developer.gserviceaccount.com --oauth_scope=https://www.googleapis.com/auth/xapi.zoo"
local test_script="/var/local/git/grpc/bins/opt/interop_client --enable_ssl --use_prod_roots";
local gfe_flags=$(_grpc_prod_gfe_flags)
local added_gfe_flags=$(_grpc_gce_test_flags)
local the_cmd="$cmd_prefix $test_script $gfe_flags $added_gfe_flags $@";
echo $the_cmd
}
# TODO(grpc-team): add grpc_interop_gen_xxx_cmd for python|nodejs
# outputs the flags passed to gfe tests
_grpc_prod_gfe_flags() {
echo " --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com"
}
# outputs the flags passed to the service account auth tests
_grpc_svc_acc_test_flags() {
echo " --service_account_key_file=/service_account/stubbyCloudTestingTest-7dd63462c60c.json --oauth_scope=https://www.googleapis.com/auth/xapi.zoo"
}
# outputs the flags passed to the gcloud auth tests
_grpc_gce_test_flags() {
echo " --default_service_account=155450119199-r5aaqa2vqoa9g5mv2m6s3m1l293rlmel@developer.gserviceaccount.com --oauth_scope=https://www.googleapis.com/auth/xapi.zoo"
}
# TODO(grpc-team): add grpc_interop_gen_xxx_cmd for python

@ -41,7 +41,82 @@ _source_gs_script() {
source $script_path
}
# Args:
# $1: numerator
# $2: denominator
# $3: threshold (optional; defaults to $THRESHOLD)
#
# Returns:
# 1 if (numerator / denominator > threshold)
# 0 otherwise
_gce_disk_cmp_ratio() {
local DEFAULT_THRESHOLD="1.1"
local numer="${1}"
local denom="${2}"
local threshold="${3:-${DEFAULT_THRESHOLD}}"
if `which python > /dev/null 2>&1`; then
python -c "print(1 if (1. * ${numer} / ${denom} > ${threshold}) else 0)"
else
echo "Can't find python; calculation not done." 1>&2
return 1
fi
}
# Repartitions the disk or resizes the file system, depending on the current
# state of the partition table.
#
# Automates the process described in
# - https://cloud.google.com/compute/docs/disks/persistent-disks#repartitionrootpd
_gce_disk_maybe_resize_then_reboot() {
# Determine the size in blocks, of the whole disk and the first partition.
local dev_sda="$(fdisk -s /dev/sda)"
local dev_sda1="$(fdisk -s /dev/sda1)"
local dev_sda1_start="$(sudo fdisk -l /dev/sda | grep /dev/sda1 | sed -e 's/ \+/ /g' | cut -d' ' -f 3)"
# Use fdisk to
# - first see if the partion 1 is using as much of the disk as it should
# - then to resize the partition if it's not
#
# fdisk(1) flags:
# -c: disable DOS compatibility mode
# -u: change display mode to sectors (from cylinders)
#
# fdisk(1) commands:
# d: delete partition (automatically selects the first one)
# n: new partition
# p: primary
# 1: partition number
# $dev_sda1_start: specify the value for the start sector, the default may be incorrect
# <1 blank lines>: accept the defaults for end sectors
# w: write partition table
if [ $(_gce_disk_cmp_ratio "${dev_sda}" "${dev_sda1}") -eq 1 ]; then
echo "$FUNCNAME: Updating the partition table to use full ${dev_sda} instead ${dev_sda1}"
cat <<EOF | fdisk -c -u /dev/sda
d
n
p
1
$dev_sda1_start
w
EOF
echo "$FUNCNAME: ... updated the partition table"
shutdown -r now
return 0
fi
# After repartitioning, use resize2fs to expand sda1.
local df_size="$(df -B 1K / | grep ' /$' | sed -e 's/ \+/ /g' | cut -d' ' -f 2)"
if [ $(_gce_disk_cmp_ratio "${dev_sda}" "${df_size}") -eq 1 ]; then
echo "$FUNCNAME: resizing the partition to make full use of it"
resize2fs /dev/sda1
echo "$FUNCNAME: ... resize completed"
fi
}
main() {
_gce_disk_maybe_resize_then_reboot
local script_attr='shared_startup_script_url'
_source_gs_script $script_attr || {
echo "halting, script 'attributes/$script_attr' could not be sourced"

@ -372,7 +372,9 @@ grpc_dockerfile_install() {
[[ -d $dockerfile_dir ]] || { echo "$FUNCNAME: not a valid dir: $dockerfile_dir"; return 1; }
# For specific base images, sync the ssh key into the .ssh dir in the dockerfile context
# For specific base images, sync private files.
#
# - the ssh key, ssh certs and/or service account info.
[[ $image_label == "grpc/base" ]] && {
grpc_docker_sync_github_key $dockerfile_dir/.ssh 'base_ssh_key' || return 1;
}
@ -384,6 +386,7 @@ grpc_dockerfile_install() {
}
[[ $image_label == "grpc/ruby" ]] && {
grpc_docker_sync_roots_pem $dockerfile_dir/cacerts || return 1;
grpc_docker_sync_service_account $dockerfile_dir/service_account || return 1;
}
[[ $image_label == "grpc/cxx" ]] && {
grpc_docker_sync_service_account $dockerfile_dir/service_account || return 1;

@ -91,8 +91,6 @@
<ClInclude Include="..\..\include\grpc\support\sync_posix.h" />
<ClInclude Include="..\..\include\grpc\support\sync_win32.h" />
<ClInclude Include="..\..\include\grpc\support\thd.h" />
<ClInclude Include="..\..\include\grpc\support\thd_posix.h" />
<ClInclude Include="..\..\include\grpc\support\thd_win32.h" />
<ClInclude Include="..\..\include\grpc\support\time.h" />
<ClInclude Include="..\..\include\grpc\support\time_posix.h" />
<ClInclude Include="..\..\include\grpc\support\time_win32.h" />

@ -135,12 +135,6 @@
<ClInclude Include="..\..\include\grpc\support\thd.h">
<Filter>include\grpc\support</Filter>
</ClInclude>
<ClInclude Include="..\..\include\grpc\support\thd_posix.h">
<Filter>include\grpc\support</Filter>
</ClInclude>
<ClInclude Include="..\..\include\grpc\support\thd_win32.h">
<Filter>include\grpc\support</Filter>
</ClInclude>
<ClInclude Include="..\..\include\grpc\support\time.h">
<Filter>include\grpc\support</Filter>
</ClInclude>

Loading…
Cancel
Save