Merge branch 'master' of github.com:grpc/grpc into simplify_compression_interop

pull/6839/head
David Garcia Quintas 9 years ago
commit 27e55b5b71
  1. 2
      src/core/lib/iomgr/endpoint.h
  2. 24
      src/core/lib/iomgr/ev_poll_and_epoll_posix.c
  3. 24
      src/core/lib/iomgr/ev_poll_posix.c
  4. 4
      src/core/lib/iomgr/ev_posix.c
  5. 6
      src/core/lib/iomgr/ev_posix.h
  6. 2
      src/core/lib/iomgr/tcp_posix.c
  7. 35
      src/php/bin/stress_client.sh
  8. 112
      src/php/tests/interop/interop_client.php
  9. 38
      src/php/tests/interop/messages.proto
  10. 2
      src/php/tests/interop/stress_client.php
  11. 9
      src/php/tests/interop/test.proto
  12. 4
      src/python/grpcio/grpc/__init__.py
  13. 2
      src/python/grpcio/grpc/_adapter/_types.py
  14. 4
      src/python/grpcio/grpc/_common.py
  15. 2
      src/python/grpcio/grpc/_cython/_cygrpc/records.pyx.pxi
  16. 3
      src/python/grpcio/grpc/beta/interfaces.py
  17. 2
      src/python/grpcio/tests/tests.json
  18. 14
      src/python/grpcio/tests/unit/_api_test.py
  19. 10
      src/python/grpcio/tests/unit/beta/_connectivity_channel_test.py
  20. 7
      templates/tools/dockerfile/run_tests_addons.include
  21. 6
      templates/tools/dockerfile/run_tests_addons_nocache.include
  22. 65
      templates/tools/dockerfile/stress_test/grpc_interop_stress_php/Dockerfile.template
  23. 3
      templates/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile.template
  24. 60
      test/core/iomgr/endpoint_tests.c
  25. 2
      test/core/iomgr/tcp_posix_test.c
  26. 2
      third_party/protobuf
  27. 140
      tools/dockerfile/stress_test/grpc_interop_stress_php/Dockerfile
  28. 54
      tools/dockerfile/stress_test/grpc_interop_stress_php/build_interop_stress.sh
  29. 8
      tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile
  30. 2
      tools/run_tests/run_interop_tests.py
  31. 20
      tools/run_tests/run_tests.py
  32. 4
      tools/run_tests/sanity/check_submodules.sh
  33. 93
      tools/run_tests/stress_test/configs/php-cxx.json

@ -82,7 +82,7 @@ char *grpc_endpoint_get_peer(grpc_endpoint *ep);
void grpc_endpoint_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
gpr_slice_buffer *slices, grpc_closure *cb);
/* Causes any pending read/write callbacks to run immediately with
/* Causes any pending and future read/write callbacks to run immediately with
success==0 */
void grpc_endpoint_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep);
void grpc_endpoint_destroy(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep);

@ -515,7 +515,9 @@ static void fd_unref(grpc_fd *fd) { unref_by(fd, 2); }
static void notify_on_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
grpc_closure **st, grpc_closure *closure) {
if (*st == CLOSURE_NOT_READY) {
if (fd->shutdown) {
grpc_exec_ctx_enqueue(exec_ctx, closure, false, NULL);
} else if (*st == CLOSURE_NOT_READY) {
/* not ready ==> switch to a waiting state by setting the closure */
*st = closure;
} else if (*st == CLOSURE_READY) {
@ -557,13 +559,24 @@ static void set_read_notifier_pollset_locked(
static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
gpr_mu_lock(&fd->mu);
GPR_ASSERT(!fd->shutdown);
fd->shutdown = 1;
set_ready_locked(exec_ctx, fd, &fd->read_closure);
set_ready_locked(exec_ctx, fd, &fd->write_closure);
/* only shutdown once */
if (!fd->shutdown) {
fd->shutdown = 1;
/* signal read/write closed to OS so that future operations fail */
shutdown(fd->fd, SHUT_RDWR);
set_ready_locked(exec_ctx, fd, &fd->read_closure);
set_ready_locked(exec_ctx, fd, &fd->write_closure);
}
gpr_mu_unlock(&fd->mu);
}
static bool fd_is_shutdown(grpc_fd *fd) {
gpr_mu_lock(&fd->mu);
bool r = fd->shutdown;
gpr_mu_unlock(&fd->mu);
return r;
}
static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
grpc_closure *closure) {
gpr_mu_lock(&fd->mu);
@ -1938,6 +1951,7 @@ static const grpc_event_engine_vtable vtable = {
.fd_wrapped_fd = fd_wrapped_fd,
.fd_orphan = fd_orphan,
.fd_shutdown = fd_shutdown,
.fd_is_shutdown = fd_is_shutdown,
.fd_notify_on_read = fd_notify_on_read,
.fd_notify_on_write = fd_notify_on_write,
.fd_get_read_notifier_pollset = fd_get_read_notifier_pollset,

@ -421,7 +421,9 @@ static void fd_unref(grpc_fd *fd) { unref_by(fd, 2); }
static void notify_on_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
grpc_closure **st, grpc_closure *closure) {
if (*st == CLOSURE_NOT_READY) {
if (fd->shutdown) {
grpc_exec_ctx_enqueue(exec_ctx, closure, false, NULL);
} else if (*st == CLOSURE_NOT_READY) {
/* not ready ==> switch to a waiting state by setting the closure */
*st = closure;
} else if (*st == CLOSURE_READY) {
@ -463,13 +465,24 @@ static void set_read_notifier_pollset_locked(
static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
gpr_mu_lock(&fd->mu);
GPR_ASSERT(!fd->shutdown);
fd->shutdown = 1;
set_ready_locked(exec_ctx, fd, &fd->read_closure);
set_ready_locked(exec_ctx, fd, &fd->write_closure);
/* only shutdown once */
if (!fd->shutdown) {
fd->shutdown = 1;
/* signal read/write closed to OS so that future operations fail */
shutdown(fd->fd, SHUT_RDWR);
set_ready_locked(exec_ctx, fd, &fd->read_closure);
set_ready_locked(exec_ctx, fd, &fd->write_closure);
}
gpr_mu_unlock(&fd->mu);
}
static bool fd_is_shutdown(grpc_fd *fd) {
gpr_mu_lock(&fd->mu);
bool r = fd->shutdown;
gpr_mu_unlock(&fd->mu);
return r;
}
static void fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
grpc_closure *closure) {
gpr_mu_lock(&fd->mu);
@ -1172,6 +1185,7 @@ static const grpc_event_engine_vtable vtable = {
.fd_wrapped_fd = fd_wrapped_fd,
.fd_orphan = fd_orphan,
.fd_shutdown = fd_shutdown,
.fd_is_shutdown = fd_is_shutdown,
.fd_notify_on_read = fd_notify_on_read,
.fd_notify_on_write = fd_notify_on_write,
.fd_get_read_notifier_pollset = fd_get_read_notifier_pollset,

@ -153,6 +153,10 @@ void grpc_fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
g_event_engine->fd_shutdown(exec_ctx, fd);
}
bool grpc_fd_is_shutdown(grpc_fd *fd) {
return g_event_engine->fd_is_shutdown(fd);
}
void grpc_fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
grpc_closure *closure) {
g_event_engine->fd_notify_on_read(exec_ctx, fd, closure);

@ -55,6 +55,7 @@ typedef struct grpc_event_engine_vtable {
grpc_closure *closure);
void (*fd_notify_on_write)(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
grpc_closure *closure);
bool (*fd_is_shutdown)(grpc_fd *fd);
grpc_pollset *(*fd_get_read_notifier_pollset)(grpc_exec_ctx *exec_ctx,
grpc_fd *fd);
@ -116,7 +117,10 @@ int grpc_fd_wrapped_fd(grpc_fd *fd);
void grpc_fd_orphan(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure *on_done,
int *release_fd, const char *reason);
/* Cause any current callbacks to error out with GRPC_CALLBACK_CANCELLED. */
/* Has grpc_fd_shutdown been called on an fd? */
bool grpc_fd_is_shutdown(grpc_fd *fd);
/* Cause any current and future callbacks to fail. */
void grpc_fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd);
/* Register read interest, causing read_cb to be called once when fd becomes

@ -408,7 +408,7 @@ static void tcp_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep,
if (buf->length == 0) {
GPR_TIMER_END("tcp_write", 0);
grpc_exec_ctx_enqueue(exec_ctx, cb, true, NULL);
grpc_exec_ctx_enqueue(exec_ctx, cb, !grpc_fd_is_shutdown(tcp->em_fd), NULL);
return;
}
tcp->outgoing_buffer = buf;

@ -0,0 +1,35 @@
#!/bin/bash
# Copyright 2015, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
set -e
cd $(dirname $0)
source ./determine_extension_dir.sh
php $extension_dir -d max_execution_time=300 \
../tests/interop/stress_client.php $@ 1>&2

@ -388,6 +388,103 @@ function timeoutOnSleepingServer($stub)
'Call status was not DEADLINE_EXCEEDED');
}
function customMetadata($stub)
{
$ECHO_INITIAL_KEY = 'x-grpc-test-echo-initial';
$ECHO_INITIAL_VALUE = 'test_initial_metadata_value';
$ECHO_TRAILING_KEY = 'x-grpc-test-echo-trailing-bin';
$ECHO_TRAILING_VALUE = 'ababab';
$request_len = 271828;
$response_len = 314159;
$request = new grpc\testing\SimpleRequest();
$request->setResponseType(grpc\testing\PayloadType::COMPRESSABLE);
$request->setResponseSize($response_len);
$payload = new grpc\testing\Payload();
$payload->setType(grpc\testing\PayloadType::COMPRESSABLE);
$payload->setBody(str_repeat("\0", $request_len));
$request->setPayload($payload);
$metadata = [
$ECHO_INITIAL_KEY => [$ECHO_INITIAL_VALUE],
$ECHO_TRAILING_KEY => [$ECHO_TRAILING_VALUE],
];
$call = $stub->UnaryCall($request, $metadata);
$initial_metadata = $call->getMetadata();
hardAssert(array_key_exists($ECHO_INITIAL_KEY, $initial_metadata),
'Initial metadata does not contain expected key');
hardAssert($initial_metadata[$ECHO_INITIAL_KEY][0] ==
$ECHO_INITIAL_VALUE,
'Incorrect initial metadata value');
list($result, $status) = $call->wait();
hardAssert($status->code === Grpc\STATUS_OK,
'Call did not complete successfully');
$trailing_metadata = $call->getTrailingMetadata();
hardAssert(array_key_exists($ECHO_TRAILING_KEY, $trailing_metadata),
'Trailing metadata does not contain expected key');
hardAssert($trailing_metadata[$ECHO_TRAILING_KEY][0] ==
$ECHO_TRAILING_VALUE, 'Incorrect trailing metadata value');
$streaming_call = $stub->FullDuplexCall($metadata);
$streaming_request = new grpc\testing\StreamingOutputCallRequest();
$streaming_request->setPayload($payload);
$streaming_call->write($streaming_request);
$streaming_call->writesDone();
hardAssert($streaming_call->getStatus()->code === Grpc\STATUS_OK,
'Call did not complete successfully');
$streaming_trailing_metadata = $streaming_call->getTrailingMetadata();
hardAssert(array_key_exists($ECHO_TRAILING_KEY,
$streaming_trailing_metadata),
'Trailing metadata does not contain expected key');
hardAssert($streaming_trailing_metadata[$ECHO_TRAILING_KEY][0] ==
$ECHO_TRAILING_VALUE, 'Incorrect trailing metadata value');
}
function statusCodeAndMessage($stub)
{
$echo_status = new grpc\testing\EchoStatus();
$echo_status->setCode(2);
$echo_status->setMessage("test status message");
$request = new grpc\testing\SimpleRequest();
$request->setResponseStatus($echo_status);
$call = $stub->UnaryCall($request);
list($result, $status) = $call->wait();
hardAssert($status->code === 2,
'Received unexpected status code');
hardAssert($status->details === "test status message",
'Received unexpected status details');
$streaming_call = $stub->FullDuplexCall();
$streaming_request = new grpc\testing\StreamingOutputCallRequest();
$streaming_request->setResponseStatus($echo_status);
$streaming_call->write($streaming_request);
$streaming_call->writesDone();
$status = $streaming_call->getStatus();
hardAssert($status->code === 2,
'Received unexpected status code');
hardAssert($status->details === "test status message",
'Received unexpected status details');
}
function unimplementedMethod($stub)
{
$call = $stub->UnimplementedCall(new grpc\testing\EmptyMessage());
list($result, $status) = $call->wait();
hardAssert($status->code === Grpc\STATUS_UNIMPLEMENTED,
'Received unexpected status code');
}
function _makeStub($args)
{
if (!array_key_exists('server_host', $args)) {
@ -472,7 +569,11 @@ function _makeStub($args)
$opts['update_metadata'] = $update_metadata;
}
$stub = new grpc\testing\TestServiceClient($server_address, $opts);
if ($test_case == 'unimplemented_method') {
$stub = new grpc\testing\UnimplementedServiceClient($server_address, $opts);
} else {
$stub = new grpc\testing\TestServiceClient($server_address, $opts);
}
return $stub;
}
@ -514,6 +615,15 @@ function interop_main($args, $stub = false)
case 'timeout_on_sleeping_server':
timeoutOnSleepingServer($stub);
break;
case 'custom_metadata':
customMetadata($stub);
break;
case 'status_code_and_message':
statusCodeAndMessage($stub);
break;
case 'unimplemented_method':
unimplementedMethod($stub);
break;
case 'service_account_creds':
serviceAccountCreds($stub, $args);
break;

@ -1,5 +1,5 @@
// Copyright 2015, Google Inc.
// Copyright 2015-2016, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
@ -41,9 +41,6 @@ enum PayloadType {
// Uncompressable binary format.
UNCOMPRESSABLE = 1;
// Randomly chosen from all other formats defined in this enum.
RANDOM = 2;
}
// A block of data, to simply increase gRPC message size.
@ -54,6 +51,13 @@ message Payload {
optional bytes body = 2;
}
// A protobuf representation for grpc status. This is used by test
// clients to specify a status that the server should attempt to return.
message EchoStatus {
optional int32 code = 1;
optional string message = 2;
}
// Unary request.
message SimpleRequest {
// Desired payload type in the response from the server.
@ -72,6 +76,12 @@ message SimpleRequest {
// Whether SimpleResponse should include OAuth scope.
optional bool fill_oauth_scope = 5;
// Whether to request the server to compress the response.
optional bool request_compressed_response = 6;
// Whether server should return a given status
optional EchoStatus response_status = 7;
}
// Unary response, as configured by the request.
@ -123,6 +133,12 @@ message StreamingOutputCallRequest {
// Optional input payload sent along with the request.
optional Payload payload = 3;
// Whether to request the server to compress the response.
optional bool request_compressed_response = 6;
// Whether server should return a given status
optional EchoStatus response_status = 7;
}
// Server-streaming response, as configured by the request and parameters.
@ -130,3 +146,17 @@ message StreamingOutputCallResponse {
// Payload to increase response size.
optional Payload payload = 1;
}
// For reconnect interop test only.
// Client tells server what reconnection parameters it used.
message ReconnectParams {
optional int32 max_reconnect_backoff_ms = 1;
}
// For reconnect interop test only.
// Server tells client whether its reconnects are following the spec and the
// reconnect backoffs it saw.
message ReconnectInfo {
optional bool passed = 1;
repeated int32 backoff_ms = 2;
}

@ -102,7 +102,7 @@ if (empty($raw_args['server_addresses'])) {
}
$args['metrics_port'] = empty($raw_args['metrics_port']) ?
'8081' : $args['metrics_port'];
'8081' : $raw_args['metrics_port'];
$args['test_duration_secs'] = empty($raw_args['test_duration_secs']) ||
$raw_args['test_duration_secs'] == -1 ?

@ -1,5 +1,5 @@
// Copyright 2015, Google Inc.
// Copyright 2015-2016, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
@ -69,3 +69,10 @@ service TestService {
rpc HalfDuplexCall(stream StreamingOutputCallRequest)
returns (stream StreamingOutputCallResponse);
}
// A simple service NOT implemented at servers so clients can test for
// that case.
service UnimplementedService {
// A call that no server should implement
rpc UnimplementedCall(grpc.testing.EmptyMessage) returns (grpc.testing.EmptyMessage);
}

@ -212,14 +212,14 @@ class ChannelConnectivity(enum.Enum):
READY: The channel is ready to conduct RPCs.
TRANSIENT_FAILURE: The channel has seen a failure from which it expects to
recover.
FATAL_FAILURE: The channel has seen a failure from which it cannot recover.
SHUTDOWN: The channel has seen a failure from which it cannot recover.
"""
IDLE = (_cygrpc.ConnectivityState.idle, 'idle')
CONNECTING = (_cygrpc.ConnectivityState.connecting, 'connecting')
READY = (_cygrpc.ConnectivityState.ready, 'ready')
TRANSIENT_FAILURE = (
_cygrpc.ConnectivityState.transient_failure, 'transient failure')
FATAL_FAILURE = (_cygrpc.ConnectivityState.fatal_failure, 'fatal failure')
SHUTDOWN = (_cygrpc.ConnectivityState.shutdown, 'shutdown')
@enum.unique

@ -114,7 +114,7 @@ class ConnectivityState(enum.IntEnum):
CONNECTING = cygrpc.ConnectivityState.connecting
READY = cygrpc.ConnectivityState.ready
TRANSIENT_FAILURE = cygrpc.ConnectivityState.transient_failure
FATAL_FAILURE = cygrpc.ConnectivityState.fatal_failure
FATAL_FAILURE = cygrpc.ConnectivityState.shutdown
class Status(collections.namedtuple(

@ -46,8 +46,8 @@ CYGRPC_CONNECTIVITY_STATE_TO_CHANNEL_CONNECTIVITY = {
cygrpc.ConnectivityState.ready: grpc.ChannelConnectivity.READY,
cygrpc.ConnectivityState.transient_failure:
grpc.ChannelConnectivity.TRANSIENT_FAILURE,
cygrpc.ConnectivityState.fatal_failure:
grpc.ChannelConnectivity.FATAL_FAILURE,
cygrpc.ConnectivityState.shutdown:
grpc.ChannelConnectivity.SHUTDOWN,
}
CYGRPC_STATUS_CODE_TO_STATUS_CODE = {

@ -33,7 +33,7 @@ class ConnectivityState:
connecting = GRPC_CHANNEL_CONNECTING
ready = GRPC_CHANNEL_READY
transient_failure = GRPC_CHANNEL_TRANSIENT_FAILURE
fatal_failure = GRPC_CHANNEL_SHUTDOWN
shutdown = GRPC_CHANNEL_SHUTDOWN
class ChannelArgKey:

@ -36,6 +36,9 @@ import six
import grpc
ChannelConnectivity = grpc.ChannelConnectivity
# FATAL_FAILURE was a Beta-API name for SHUTDOWN
ChannelConnectivity.FATAL_FAILURE = ChannelConnectivity.SHUTDOWN
StatusCode = grpc.StatusCode

@ -1,5 +1,6 @@
[
"_api_test.AllTest",
"_api_test.ChannelConnectivityTest",
"_auth_test.AccessTokenCallCredentialsTest",
"_auth_test.GoogleCallCredentialsTest",
"_base_interface_test.AsyncEasyTest",
@ -13,6 +14,7 @@
"_channel_ready_future_test.ChannelReadyFutureTest",
"_channel_test.ChannelTest",
"_connectivity_channel_test.ChannelConnectivityTest",
"_connectivity_channel_test.ConnectivityStatesTest",
"_core_over_links_base_interface_test.AsyncEasyTest",
"_core_over_links_base_interface_test.AsyncPeasyTest",
"_core_over_links_base_interface_test.SyncEasyTest",

@ -33,6 +33,8 @@ import unittest
import six
import grpc
from tests.unit import _from_grpc_import_star
@ -86,5 +88,17 @@ class AllTest(unittest.TestCase):
_from_grpc_import_star.GRPC_ELEMENTS)
class ChannelConnectivityTest(unittest.TestCase):
def testChannelConnectivity(self):
self.assertSequenceEqual(
(grpc.ChannelConnectivity.IDLE,
grpc.ChannelConnectivity.CONNECTING,
grpc.ChannelConnectivity.READY,
grpc.ChannelConnectivity.TRANSIENT_FAILURE,
grpc.ChannelConnectivity.SHUTDOWN,),
tuple(grpc.ChannelConnectivity))
if __name__ == '__main__':
unittest.main(verbosity=2)

@ -187,5 +187,15 @@ class ChannelConnectivityTest(unittest.TestCase):
server_completion_queue_thread.join()
class ConnectivityStatesTest(unittest.TestCase):
def testBetaConnectivityStates(self):
self.assertIsNotNone(interfaces.ChannelConnectivity.IDLE)
self.assertIsNotNone(interfaces.ChannelConnectivity.CONNECTING)
self.assertIsNotNone(interfaces.ChannelConnectivity.READY)
self.assertIsNotNone(interfaces.ChannelConnectivity.TRANSIENT_FAILURE)
self.assertIsNotNone(interfaces.ChannelConnectivity.FATAL_FAILURE)
if __name__ == '__main__':
unittest.main(verbosity=2)

@ -1,7 +1,2 @@
<%include file="ccache_setup.include"/>
#======================
# Zookeeper dependencies
# TODO(jtattermusch): is zookeeper still needed?
RUN apt-get install -y libzookeeper-mt-dev
RUN mkdir /var/local/jenkins
<%include file="run_tests_addons_nocache.include"/>

@ -0,0 +1,6 @@
#======================
# Zookeeper dependencies
# TODO(jtattermusch): is zookeeper still needed?
RUN apt-get install -y libzookeeper-mt-dev
RUN mkdir /var/local/jenkins

@ -0,0 +1,65 @@
%YAML 1.2
--- |
# Copyright 2015, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
FROM debian:jessie
<%include file="../../apt_get_basic.include"/>
<%include file="../../ruby_deps.include"/>
<%include file="../../gcp_api_libraries.include"/>
<%include file="../../php_deps.include"/>
<%include file="../../run_tests_addons.include"/>
# ronn: a ruby tool used to convert markdown to man pages, used during the
# install of Protobuf extensions
#
# rake: a ruby version of make used to build the PHP Protobuf extension
RUN /bin/bash -l -c "rvm all do gem install ronn rake"
# Install composer
RUN curl -sS https://getcomposer.org/installer | php
RUN mv composer.phar /usr/local/bin/composer
# As an attempt to work around #4212, try to prefetch Protobuf-PHP dependency
# into composer cache to prevent "composer install" from cloning on each build.
RUN git clone --mirror https://github.com/stanley-cheung/Protobuf-PHP.git ${'\\'}
/root/.composer/cache/vcs/git-github.com-stanley-cheung-Protobuf-PHP.git/
# Download the patched PHP protobuf so that PHP gRPC clients can be generated
# from proto3 schemas.
RUN git clone https://github.com/stanley-cheung/Protobuf-PHP.git /var/local/git/protobuf-php
RUN /bin/bash -l -c "rvm use ruby-2.1 ${'\\'}
&& cd /var/local/git/protobuf-php ${'\\'}
&& rvm all do rake pear:package version=1.0 ${'\\'}
&& pear install Protobuf-1.0.tgz"
# Define the default command.
CMD ["bash"]

@ -33,7 +33,6 @@
<%include file="../../apt_get_basic.include"/>
<%include file="../../cxx_deps.include"/>
<%include file="../../run_tests_addons.include"/>
<%include file="../../run_tests_addons_nocache.include"/>
# Define the default command.
CMD ["bash"]

@ -222,7 +222,7 @@ static void read_and_write_test(grpc_endpoint_test_config config,
even when bytes_written is unsigned. */
state.bytes_written -= state.current_write_size;
read_and_write_test_write_handler(&exec_ctx, &state, 1);
grpc_exec_ctx_finish(&exec_ctx);
grpc_exec_ctx_flush(&exec_ctx);
grpc_endpoint_read(&exec_ctx, state.read_ep, &state.incoming,
&state.done_read);
@ -233,7 +233,7 @@ static void read_and_write_test(grpc_endpoint_test_config config,
gpr_log(GPR_DEBUG, "shutdown write");
grpc_endpoint_shutdown(&exec_ctx, state.write_ep);
}
grpc_exec_ctx_finish(&exec_ctx);
grpc_exec_ctx_flush(&exec_ctx);
gpr_mu_lock(g_mu);
while (!state.read_done || !state.write_done) {
@ -243,7 +243,7 @@ static void read_and_write_test(grpc_endpoint_test_config config,
gpr_now(GPR_CLOCK_MONOTONIC), deadline);
}
gpr_mu_unlock(g_mu);
grpc_exec_ctx_finish(&exec_ctx);
grpc_exec_ctx_flush(&exec_ctx);
end_test(config);
gpr_slice_buffer_destroy(&state.outgoing);
@ -253,11 +253,64 @@ static void read_and_write_test(grpc_endpoint_test_config config,
grpc_exec_ctx_finish(&exec_ctx);
}
static void inc_on_failure(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
*(int *)arg += (success == false);
}
static void wait_for_fail_count(grpc_exec_ctx *exec_ctx, int *fail_count,
int want_fail_count) {
grpc_exec_ctx_flush(exec_ctx);
for (int i = 0; i < 5 && *fail_count < want_fail_count; i++) {
grpc_pollset_worker *worker = NULL;
gpr_timespec now = gpr_now(GPR_CLOCK_REALTIME);
gpr_timespec deadline =
gpr_time_add(now, gpr_time_from_seconds(1, GPR_TIMESPAN));
gpr_mu_lock(g_mu);
grpc_pollset_work(exec_ctx, g_pollset, &worker, now, deadline);
gpr_mu_unlock(g_mu);
grpc_exec_ctx_flush(exec_ctx);
}
GPR_ASSERT(*fail_count == want_fail_count);
}
static void multiple_shutdown_test(grpc_endpoint_test_config config) {
grpc_endpoint_test_fixture f =
begin_test(config, "multiple_shutdown_test", 128);
int fail_count = 0;
gpr_slice_buffer slice_buffer;
gpr_slice_buffer_init(&slice_buffer);
grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
grpc_endpoint_add_to_pollset(&exec_ctx, f.client_ep, g_pollset);
grpc_endpoint_read(&exec_ctx, f.client_ep, &slice_buffer,
grpc_closure_create(inc_on_failure, &fail_count));
wait_for_fail_count(&exec_ctx, &fail_count, 0);
grpc_endpoint_shutdown(&exec_ctx, f.client_ep);
wait_for_fail_count(&exec_ctx, &fail_count, 1);
grpc_endpoint_read(&exec_ctx, f.client_ep, &slice_buffer,
grpc_closure_create(inc_on_failure, &fail_count));
wait_for_fail_count(&exec_ctx, &fail_count, 2);
gpr_slice_buffer_add(&slice_buffer, gpr_slice_from_copied_string("a"));
grpc_endpoint_write(&exec_ctx, f.client_ep, &slice_buffer,
grpc_closure_create(inc_on_failure, &fail_count));
wait_for_fail_count(&exec_ctx, &fail_count, 3);
grpc_endpoint_shutdown(&exec_ctx, f.client_ep);
wait_for_fail_count(&exec_ctx, &fail_count, 3);
gpr_slice_buffer_destroy(&slice_buffer);
grpc_endpoint_destroy(&exec_ctx, f.client_ep);
grpc_endpoint_destroy(&exec_ctx, f.server_ep);
grpc_exec_ctx_finish(&exec_ctx);
}
void grpc_endpoint_tests(grpc_endpoint_test_config config,
grpc_pollset *pollset, gpr_mu *mu) {
size_t i;
g_pollset = pollset;
g_mu = mu;
multiple_shutdown_test(config);
read_and_write_test(config, 10000000, 100000, 8192, 0);
read_and_write_test(config, 1000000, 100000, 1, 0);
read_and_write_test(config, 100000000, 100000, 1, 1);
@ -265,4 +318,5 @@ void grpc_endpoint_tests(grpc_endpoint_test_config config,
read_and_write_test(config, 40320, i, i, 0);
}
g_pollset = NULL;
g_mu = NULL;
}

@ -517,8 +517,8 @@ int main(int argc, char **argv) {
grpc_init();
g_pollset = gpr_malloc(grpc_pollset_size());
grpc_pollset_init(g_pollset, &g_mu);
run_tests();
grpc_endpoint_tests(configs[0], g_pollset, g_mu);
run_tests();
grpc_closure_init(&destroyed, destroy_pollset, g_pollset);
grpc_pollset_shutdown(&exec_ctx, g_pollset, &destroyed);
grpc_exec_ctx_finish(&exec_ctx);

@ -1 +1 @@
Subproject commit 3470b6895aa659b7559ed678e029a5338e535f14
Subproject commit d4d13a4349e4e59d67f311185ddcc1890d956d7a

@ -0,0 +1,140 @@
# Copyright 2015, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
FROM debian:jessie
# Install Git and basic packages.
RUN apt-get update && apt-get install -y \
autoconf \
autotools-dev \
build-essential \
bzip2 \
ccache \
curl \
gcc \
gcc-multilib \
git \
golang \
gyp \
lcov \
libc6 \
libc6-dbg \
libc6-dev \
libgtest-dev \
libtool \
make \
perl \
strace \
python-dev \
python-setuptools \
python-yaml \
telnet \
unzip \
wget \
zip && apt-get clean
#================
# Build profiling
RUN apt-get update && apt-get install -y time && apt-get clean
#==================
# Ruby dependencies
# Install rvm
RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
RUN \curl -sSL https://get.rvm.io | bash -s stable
# Install Ruby 2.1
RUN /bin/bash -l -c "rvm install ruby-2.1"
RUN /bin/bash -l -c "rvm use --default ruby-2.1"
RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc"
RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc"
RUN /bin/bash -l -c "echo 'rvm --default use ruby-2.1' >> ~/.bashrc"
RUN /bin/bash -l -c "gem install bundler --no-ri --no-rdoc"
# Google Cloud platform API libraries
RUN apt-get update && apt-get install -y python-pip && apt-get clean
RUN pip install --upgrade google-api-python-client
#=================
# PHP dependencies
# Install dependencies
RUN /bin/bash -l -c "echo 'deb http://packages.dotdeb.org wheezy-php55 all' \
>> /etc/apt/sources.list.d/dotdeb.list"
RUN /bin/bash -l -c "echo 'deb-src http://packages.dotdeb.org wheezy-php55 all' \
>> /etc/apt/sources.list.d/dotdeb.list"
RUN wget http://www.dotdeb.org/dotdeb.gpg -O- | apt-key add -
RUN apt-get update && apt-get install -y \
git php5 php5-dev phpunit unzip
# Prepare ccache
RUN ln -s /usr/bin/ccache /usr/local/bin/gcc
RUN ln -s /usr/bin/ccache /usr/local/bin/g++
RUN ln -s /usr/bin/ccache /usr/local/bin/cc
RUN ln -s /usr/bin/ccache /usr/local/bin/c++
RUN ln -s /usr/bin/ccache /usr/local/bin/clang
RUN ln -s /usr/bin/ccache /usr/local/bin/clang++
#======================
# Zookeeper dependencies
# TODO(jtattermusch): is zookeeper still needed?
RUN apt-get install -y libzookeeper-mt-dev
RUN mkdir /var/local/jenkins
# ronn: a ruby tool used to convert markdown to man pages, used during the
# install of Protobuf extensions
#
# rake: a ruby version of make used to build the PHP Protobuf extension
RUN /bin/bash -l -c "rvm all do gem install ronn rake"
# Install composer
RUN curl -sS https://getcomposer.org/installer | php
RUN mv composer.phar /usr/local/bin/composer
# As an attempt to work around #4212, try to prefetch Protobuf-PHP dependency
# into composer cache to prevent "composer install" from cloning on each build.
RUN git clone --mirror https://github.com/stanley-cheung/Protobuf-PHP.git \
/root/.composer/cache/vcs/git-github.com-stanley-cheung-Protobuf-PHP.git/
# Download the patched PHP protobuf so that PHP gRPC clients can be generated
# from proto3 schemas.
RUN git clone https://github.com/stanley-cheung/Protobuf-PHP.git /var/local/git/protobuf-php
RUN /bin/bash -l -c "rvm use ruby-2.1 \
&& cd /var/local/git/protobuf-php \
&& rvm all do rake pear:package version=1.0 \
&& pear install Protobuf-1.0.tgz"
# Define the default command.
CMD ["bash"]

@ -0,0 +1,54 @@
#!/bin/bash
# Copyright 2015, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Builds PHP interop server and client in a base image.
set -ex
mkdir -p /var/local/git
git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc
# copy service account keys if available
cp -r /var/local/jenkins/service_account $HOME || true
cd /var/local/git/grpc
rvm --default use ruby-2.1
make install-certs
# gRPC core and protobuf need to be installed
make install
(cd src/php/ext/grpc && phpize && ./configure && make)
(cd third_party/protobuf && make install)
(cd src/php && composer install)
(cd src/php && protoc-gen-php -i tests/interop/ -o tests/interop/ tests/interop/test.proto)

@ -67,14 +67,6 @@ RUN apt-get update && apt-get install -y time && apt-get clean
# C++ dependencies
RUN apt-get update && apt-get -y install libgflags-dev libgtest-dev libc++-dev clang && apt-get clean
# Prepare ccache
RUN ln -s /usr/bin/ccache /usr/local/bin/gcc
RUN ln -s /usr/bin/ccache /usr/local/bin/g++
RUN ln -s /usr/bin/ccache /usr/local/bin/cc
RUN ln -s /usr/bin/ccache /usr/local/bin/c++
RUN ln -s /usr/bin/ccache /usr/local/bin/clang
RUN ln -s /usr/bin/ccache /usr/local/bin/clang++
#======================
# Zookeeper dependencies
# TODO(jtattermusch): is zookeeper still needed?

@ -252,7 +252,7 @@ class PHPLanguage:
return {}
def unimplemented_test_cases(self):
return _SKIP_ADVANCED + _SKIP_COMPRESSION
return _SKIP_COMPRESSION
def unimplemented_test_cases_server(self):
return []

@ -62,6 +62,11 @@ os.chdir(_ROOT)
_FORCE_ENVIRON_FOR_WRAPPERS = {}
_POLLING_STRATEGIES = {
'linux': ['poll', 'legacy']
}
def platform_string():
return jobset.platform_string()
@ -153,14 +158,8 @@ class CLanguage(object):
def test_specs(self):
out = []
binaries = get_c_tests(self.args.travis, self.test_lang)
POLLING_STRATEGIES = {
'windows': ['all'],
'mac': ['all'],
'posix': ['all'],
'linux': ['poll', 'legacy']
}
for target in binaries:
polling_strategies = (POLLING_STRATEGIES[self.platform]
polling_strategies = (_POLLING_STRATEGIES.get(self.platform, ['all'])
if target.get('uses_polling', True)
else ['all'])
for polling_strategy in polling_strategies:
@ -395,7 +394,7 @@ class PythonLanguage(object):
tests_json = json.load(tests_json_file)
environment = dict(_FORCE_ENVIRON_FOR_WRAPPERS)
environment['PYTHONPATH'] = '{}:{}'.format(
os.path.abspath('src/python/gens'),
os.path.abspath('src/python/gens'),
os.path.abspath('src/python/grpcio_health_checking'))
if self.config.build_config != 'gcov':
return [self.config.job_spec(
@ -854,8 +853,13 @@ argp.add_argument('--update_submodules', default=[], nargs='*',
argp.add_argument('-a', '--antagonists', default=0, type=int)
argp.add_argument('-x', '--xml_report', default=None, type=str,
help='Generates a JUnit-compatible XML report')
argp.add_argument('--force_default_poller', default=False, action='store_const', const=True,
help='Dont try to iterate over many polling strategies when they exist')
args = argp.parse_args()
if args.force_default_poller:
_POLLING_STRATEGIES = {}
jobset.measure_cpu_costs = args.measure_cpu_costs
# update submodules if necessary

@ -41,11 +41,11 @@ want_submodules=`mktemp /tmp/submXXXXXX`
git submodule | awk '{ print $1 }' | sort > $submodules
cat << EOF | awk '{ print $1 }' | sort > $want_submodules
c880e42ba1c8032d4cdde2aba0541d8a9d9fa2e9 third_party/boringssl (heads/2661)
c880e42ba1c8032d4cdde2aba0541d8a9d9fa2e9 third_party/boringssl (version_for_cocoapods_2.0-100-gc880e42)
05b155ff59114735ec8cd089f669c4c3d8f59029 third_party/gflags (v2.1.0-45-g05b155f)
c99458533a9b4c743ed51537e25989ea55944908 third_party/googletest (release-1.7.0)
f8ac463766281625ad710900479130c7fcb4d63b third_party/nanopb (nanopb-0.3.4-29-gf8ac463)
3470b6895aa659b7559ed678e029a5338e535f14 third_party/protobuf (v3.0.0-beta-2-441-g3470b68)
d4d13a4349e4e59d67f311185ddcc1890d956d7a third_party/protobuf (v3.0.0-beta-3.2)
50893291621658f355bc5b4d450a8d06a563053d third_party/zlib (v1.2.8)
EOF

@ -0,0 +1,93 @@
{
"dockerImages": {
"grpc_stress_cxx_opt" : {
"buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh",
"dockerFileDir": "grpc_interop_stress_cxx",
"buildType": "opt"
},
"grpc_stress_php": {
"buildScript": "tools/run_tests/dockerize/build_interop_stress_image.sh",
"dockerFileDir": "grpc_interop_stress_php"
}
},
"clientTemplates": {
"baseTemplates": {
"default": {
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_client.py",
"pollIntervalSecs": 60,
"clientArgs": {
"num_channels_per_server":5,
"num_stubs_per_channel":10,
"test_cases": "empty_unary:1,large_unary:1,client_streaming:1,server_streaming:1,empty_stream:1",
"metrics_port": 8081
},
"metricsPort": 8081,
"metricsArgs": {
"metrics_server_address": "localhost:8081"
}
}
},
"templates": {
"php_client": {
"baseTemplate": "default",
"stressClientCmd": [
"/var/local/git/grpc/src/php/bin/stress_client.sh"
],
"metricsClientCmd": [
"php",
"/var/local/git/grpc/src/php/tests/interop/metrics_client.php"
]
}
}
},
"serverTemplates": {
"baseTemplates":{
"default": {
"wrapperScriptPath": "/var/local/git/grpc/tools/gcp/stress_test/run_server.py",
"serverPort": 8080,
"serverArgs": {
"port": 8080
}
}
},
"templates": {
"cxx_server_opt": {
"baseTemplate": "default",
"stressServerCmd": ["/var/local/git/grpc/bins/opt/interop_server"]
}
}
},
"testMatrix": {
"serverPodSpecs": {
"stress-server-cxx-php": {
"serverTemplate": "cxx_server_opt",
"dockerImage": "grpc_stress_cxx_opt",
"numInstances": 1
}
},
"clientPodSpecs": {
"stress-client-php": {
"clientTemplate": "php_client",
"dockerImage": "grpc_stress_php",
"numInstances": 20,
"serverPodSpec": "stress-server-cxx-php"
}
}
},
"globalSettings": {
"buildDockerImages": true,
"pollIntervalSecs": 60,
"testDurationSecs": 7200,
"kubernetesProxyPort": 8010,
"datasetIdNamePrefix": "stress_test_php_cxx_opt",
"summaryTableId": "summary",
"qpsTableId": "qps",
"podWarmupSecs": 60
}
}
Loading…
Cancel
Save