Merge github.com:grpc/grpc into idempotent

pull/5691/head
Craig Tiller 9 years ago
commit f0ec923498
  1. 8
      CONTRIBUTING.md
  2. 8
      setup.py
  3. 29
      src/core/channel/client_channel.c
  4. 18
      src/core/channel/subchannel_call_holder.c
  5. 22
      src/core/iomgr/timer_heap.c
  6. 6
      src/core/transport/chttp2_transport.c
  7. 74
      src/cpp/README.md
  8. 2
      src/node/src/client.js
  9. 12
      src/python/grpcio/README.rst
  10. 3
      src/python/grpcio/commands.py
  11. 1
      src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi
  12. 5
      src/python/grpcio/grpc/_cython/_cygrpc/completion_queue.pyx.pxi
  13. 1
      src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi
  14. 13
      src/python/grpcio/precompiled.py
  15. 3
      src/ruby/.rubocop.yml
  16. 97
      src/ruby/lib/grpc/generic/rpc_server.rb
  17. 25
      src/ruby/spec/generic/rpc_server_spec.rb
  18. 194
      test/core/iomgr/timer_heap_test.c
  19. 14
      tools/distrib/python/docgen.py
  20. 20
      tools/run_tests/artifact_targets.py
  21. 6
      tools/run_tests/build_python.sh
  22. 2
      tools/run_tests/run_python.sh
  23. 2
      tools/run_tests/run_tests.py

@ -45,9 +45,13 @@ In order to run most of the available tests, one would need to run:
`./tools/run_tests/run_tests.py`
If you want to run all the possible tests for any of the languages {c, c++, node, php, python}, do this:
If you want to run tests for any of the languages {c, c++, csharp, node, objc, php, python, ruby}, do this:
`./tools/run_tests/run_tests.py -l <lang> -c all`
`./tools/run_tests/run_tests.py -l <lang>`
To know about the list of available commands, do this:
`./tools/run_tests/run_tests.py -h`
## Adding or removing source code

@ -108,8 +108,13 @@ if "linux" in sys.platform or "darwin" in sys.platform:
def cython_extensions(package_names, module_names, extra_sources, include_dirs,
libraries, define_macros, build_with_cython=False):
# Set compiler directives linetrace argument only if we care about tracing;
# this is due to Cython having different behavior between linetrace being
# False and linetrace being unset. See issue #5689.
cython_compiler_directives = {}
if ENABLE_CYTHON_TRACING:
define_macros = define_macros + [('CYTHON_TRACE_NOGIL', 1)]
cython_compiler_directives['linetrace'] = True
file_extension = 'pyx' if build_with_cython else 'c'
module_files = [os.path.join(PYTHON_STEM,
name.replace('.', '/') + '.' + file_extension)
@ -129,7 +134,7 @@ def cython_extensions(package_names, module_names, extra_sources, include_dirs,
return Cython.Build.cythonize(
extensions,
include_path=include_dirs,
compiler_directives={'linetrace': bool(ENABLE_CYTHON_TRACING)})
compiler_directives=cython_compiler_directives)
else:
return extensions
@ -154,6 +159,7 @@ INSTALL_REQUIRES = (
SETUP_REQUIRES = (
'sphinx>=1.3',
'sphinx_rtd_theme>=0.1.8'
) + INSTALL_REQUIRES
COMMAND_CLASS = {

@ -165,7 +165,6 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg,
channel_data *chand = arg;
grpc_lb_policy *lb_policy = NULL;
grpc_lb_policy *old_lb_policy;
grpc_resolver *old_resolver;
grpc_connectivity_state state = GRPC_CHANNEL_TRANSIENT_FAILURE;
int exit_idle = 0;
@ -201,28 +200,25 @@ static void cc_on_config_changed(grpc_exec_ctx *exec_ctx, void *arg,
}
if (iomgr_success && chand->resolver) {
grpc_resolver *resolver = chand->resolver;
GRPC_RESOLVER_REF(resolver, "channel-next");
grpc_connectivity_state_set(exec_ctx, &chand->state_tracker, state,
"new_lb+resolver");
if (lb_policy != NULL) {
watch_lb_policy(exec_ctx, chand, lb_policy, state);
}
gpr_mu_unlock(&chand->mu_config);
GRPC_CHANNEL_STACK_REF(chand->owning_stack, "resolver");
grpc_resolver_next(exec_ctx, resolver, &chand->incoming_configuration,
grpc_resolver_next(exec_ctx, chand->resolver,
&chand->incoming_configuration,
&chand->on_config_changed);
GRPC_RESOLVER_UNREF(exec_ctx, resolver, "channel-next");
gpr_mu_unlock(&chand->mu_config);
} else {
old_resolver = chand->resolver;
chand->resolver = NULL;
if (chand->resolver != NULL) {
grpc_resolver_shutdown(exec_ctx, chand->resolver);
GRPC_RESOLVER_UNREF(exec_ctx, chand->resolver, "channel");
chand->resolver = NULL;
}
grpc_connectivity_state_set(exec_ctx, &chand->state_tracker,
GRPC_CHANNEL_FATAL_FAILURE, "resolver_gone");
gpr_mu_unlock(&chand->mu_config);
if (old_resolver != NULL) {
grpc_resolver_shutdown(exec_ctx, old_resolver);
GRPC_RESOLVER_UNREF(exec_ctx, old_resolver, "channel");
}
}
if (exit_idle) {
@ -247,7 +243,6 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx,
grpc_channel_element *elem,
grpc_transport_op *op) {
channel_data *chand = elem->channel_data;
grpc_resolver *destroy_resolver = NULL;
grpc_exec_ctx_enqueue(exec_ctx, op->on_consumed, true, NULL);
@ -279,7 +274,8 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx,
if (op->disconnect && chand->resolver != NULL) {
grpc_connectivity_state_set(exec_ctx, &chand->state_tracker,
GRPC_CHANNEL_FATAL_FAILURE, "disconnect");
destroy_resolver = chand->resolver;
grpc_resolver_shutdown(exec_ctx, chand->resolver);
GRPC_RESOLVER_UNREF(exec_ctx, chand->resolver, "channel");
chand->resolver = NULL;
if (chand->lb_policy != NULL) {
grpc_pollset_set_del_pollset_set(exec_ctx,
@ -290,11 +286,6 @@ static void cc_start_transport_op(grpc_exec_ctx *exec_ctx,
}
}
gpr_mu_unlock(&chand->mu_config);
if (destroy_resolver) {
grpc_resolver_shutdown(exec_ctx, destroy_resolver);
GRPC_RESOLVER_UNREF(exec_ctx, destroy_resolver, "channel");
}
}
typedef struct {

@ -174,17 +174,15 @@ static void subchannel_ready(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
holder->creation_phase = GRPC_SUBCHANNEL_CALL_HOLDER_NOT_CREATING;
if (holder->connected_subchannel == NULL) {
fail_locked(exec_ctx, holder);
} else if (1 == gpr_atm_acq_load(&holder->subchannel_call)) {
/* already cancelled before subchannel became ready */
fail_locked(exec_ctx, holder);
} else {
if (!gpr_atm_rel_cas(
&holder->subchannel_call, 0,
(gpr_atm)(uintptr_t)grpc_connected_subchannel_create_call(
exec_ctx, holder->connected_subchannel, holder->pollset))) {
GPR_ASSERT(gpr_atm_acq_load(&holder->subchannel_call) == 1);
/* if this cas fails, the call was cancelled before the pick completed */
fail_locked(exec_ctx, holder);
} else {
retry_waiting_locked(exec_ctx, holder);
}
gpr_atm_rel_store(
&holder->subchannel_call,
(gpr_atm)(uintptr_t)grpc_connected_subchannel_create_call(
exec_ctx, holder->connected_subchannel, holder->pollset));
retry_waiting_locked(exec_ctx, holder);
}
gpr_mu_unlock(&holder->mu);
GRPC_CALL_STACK_UNREF(exec_ctx, holder->owning_call, "pick_subchannel");

@ -1,6 +1,6 @@
/*
*
* Copyright 2015, Google Inc.
* Copyright 2015-2016, Google Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -46,7 +46,7 @@
static void adjust_upwards(grpc_timer **first, uint32_t i, grpc_timer *t) {
while (i > 0) {
uint32_t parent = (uint32_t)(((int)i - 1) / 2);
if (gpr_time_cmp(first[parent]->deadline, t->deadline) >= 0) break;
if (gpr_time_cmp(first[parent]->deadline, t->deadline) <= 0) break;
first[i] = first[parent];
first[i]->heap_index = i;
i = parent;
@ -62,16 +62,14 @@ static void adjust_downwards(grpc_timer **first, uint32_t i, uint32_t length,
grpc_timer *t) {
for (;;) {
uint32_t left_child = 1u + 2u * i;
uint32_t right_child;
uint32_t next_i;
if (left_child >= length) break;
right_child = left_child + 1;
next_i = right_child < length &&
gpr_time_cmp(first[left_child]->deadline,
first[right_child]->deadline) < 0
? right_child
: left_child;
if (gpr_time_cmp(t->deadline, first[next_i]->deadline) >= 0) break;
uint32_t right_child = left_child + 1;
uint32_t next_i = right_child < length &&
gpr_time_cmp(first[left_child]->deadline,
first[right_child]->deadline) > 0
? right_child
: left_child;
if (gpr_time_cmp(t->deadline, first[next_i]->deadline) <= 0) break;
first[i] = first[next_i];
first[i]->heap_index = i;
i = next_i;
@ -95,7 +93,7 @@ static void maybe_shrink(grpc_timer_heap *heap) {
static void note_changed_priority(grpc_timer_heap *heap, grpc_timer *timer) {
uint32_t i = timer->heap_index;
uint32_t parent = (uint32_t)(((int)i - 1) / 2);
if (gpr_time_cmp(heap->timers[parent]->deadline, timer->deadline) < 0) {
if (gpr_time_cmp(heap->timers[parent]->deadline, timer->deadline) > 0) {
adjust_upwards(heap->timers, i, timer);
} else {
adjust_downwards(heap->timers, i, heap->timer_count, timer);

@ -851,9 +851,11 @@ static void perform_stream_op_locked(
if (stream_global->write_closed) {
grpc_chttp2_complete_closure_step(
exec_ctx, &stream_global->send_message_finished, 0);
} else if (stream_global->id != 0) {
} else {
stream_global->send_message = op->send_message;
grpc_chttp2_become_writable(transport_global, stream_global);
if (stream_global->id != 0) {
grpc_chttp2_become_writable(transport_global, stream_global);
}
}
}

@ -6,3 +6,77 @@ This directory contains source code for C++ implementation of gRPC.
#Status
Beta
#Pre-requisites
##Linux
```sh
$ [sudo] apt-get install build-essential autoconf libtool
```
##Mac OSX
For a Mac system, git is not available by default. You will first need to
install Xcode from the Mac AppStore and then run the following command from a
terminal:
```sh
$ [sudo] xcode-select --install
```
##Protoc
By default gRPC uses [protocol buffers](https://github.com/google/protobuf),
you will need the `protoc` compiler to generate stub server and client code.
If you compile gRPC from source, as described below, this also installs the
`protoc` compiler.
If it hasn't been installed, you can run the following commands to install it.
```sh
$ cd grpc/third_party/protobuf
$ sudo make install # 'make' should have been run by core grpc
```
Alternatively, you can download `protoc` binaries from
[the protocol buffers Github repository](https://github.com/google/protobuf/releases).
#Installation
Currently to install gRPC for C++, you need to build from source as described
below.
#Build from Source
```sh
$ git clone https://github.com/grpc/grpc.git
$ cd grpc
$ git submodule update --init
$ make
$ [sudo] make install
```
#Documentation
You can find out how to build and run our simplest gRPC C++ example in our
[C++ quick start](https://github.com/grpc/grpc/tree/{{ site.data.config.grpc_release_branch }}/examples/cpp).
For more detailed documentation on using gRPC in C++ , see our main
documentation site at [grpc.io](http://grpc.io), specifically:
* [Overview](http://www.grpc.io/docs/): An introduction to gRPC with a simple
Hello World example in all our supported languages, including C++.
* [gRPC Basics - C++](http://www.grpc.io/docs/tutorials/basic/c.html):
A tutorial that steps you through creating a simple gRPC C++ example
application.
* [Asynchronous Basics - C++](http://www.grpc.io/docs/tutorials/async/helloasync-cpp.html):
A tutorial that shows you how to use gRPC C++'s asynchronous/non-blocking
APIs.
# Examples
Code examples for gRPC C++ live in this repository's
[examples/cpp](https://github.com/grpc/grpc/tree/{{ site.data.config.grpc_release_branch }}/examples/cpp) directory.

@ -408,7 +408,7 @@ function makeUnaryRequestFunction(method, serialize, deserialize) {
}
}
if (status.code !== grpc.status.OK) {
error = new Error(response.status.details);
error = new Error(status.details);
error.code = status.code;
error.metadata = status.metadata;
callback(error);

@ -6,7 +6,7 @@ Package for gRPC Python.
Installation
------------
gRPC Python is available for Linux and Mac OS X running Python 2.7.
gRPC Python is available for Linux, Mac OS X, and Windows running Python 2.7.
From PyPI
~~~~~~~~~
@ -23,11 +23,15 @@ Else system wide (on Ubuntu)...
$ sudo pip install grpcio
n.b. On Windows and on Mac OS X one *must* have a recent release of :code:`pip`
to retrieve the proper wheel from PyPI. Be sure to upgrade to the latest
version!
From Source
~~~~~~~~~~~
Building from source requires that you have the Python headers (usually a
package named `python-dev`).
package named :code:`python-dev`).
::
@ -36,8 +40,8 @@ package named `python-dev`).
$ cd $REPO_ROOT
$ pip install .
Note that `$REPO_ROOT` can be assigned to whatever directory name floats your
fancy.
Note that :code:`$REPO_ROOT` can be assigned to whatever directory name floats
your fancy.
Troubleshooting
~~~~~~~~~~~~~~~

@ -119,8 +119,7 @@ class SphinxDocumentation(setuptools.Command):
import sphinx
import sphinx.apidoc
metadata = self.distribution.metadata
src_dir = os.path.join(
PYTHON_STEM, self.distribution.package_dir[''], 'grpc')
src_dir = os.path.join(PYTHON_STEM, 'grpc')
sys.path.append(src_dir)
sphinx.apidoc.main([
'', '--force', '--full', '-H', metadata.name, '-A', metadata.author,

@ -95,7 +95,6 @@ cdef class Channel:
self, grpc_connectivity_state last_observed_state,
Timespec deadline not None, CompletionQueue queue not None, tag):
cdef OperationTag operation_tag = OperationTag(tag)
operation_tag.references = [self, queue]
cpython.Py_INCREF(operation_tag)
grpc_channel_watch_connectivity_state(
self.c_channel, last_observed_state, deadline.c_time,

@ -140,7 +140,8 @@ cdef class CompletionQueue:
grpc_completion_queue_shutdown(self.c_completion_queue)
# Pump the queue
while not self.is_shutdown:
event = grpc_completion_queue_next(
self.c_completion_queue, c_deadline, NULL)
with nogil:
event = grpc_completion_queue_next(
self.c_completion_queue, c_deadline, NULL)
self._interpret_event(event)
grpc_completion_queue_destroy(self.c_completion_queue)

@ -106,7 +106,6 @@ cdef class Server:
self.is_shutting_down = True
operation_tag = OperationTag(tag)
operation_tag.shutting_down_server = self
operation_tag.references.extend([self, queue])
cpython.Py_INCREF(operation_tag)
grpc_server_shutdown_and_notify(
self.c_server, queue.c_completion_queue,

@ -31,6 +31,7 @@ import os
import platform
import shutil
import sys
import sysconfig
import setuptools
@ -51,9 +52,15 @@ USE_PRECOMPILED_BINARIES = bool(int(os.environ.get(
def _tagged_ext_name(base):
uname = platform.uname()
tags = '-'.join((grpc_version.VERSION, uname[0], uname[4]))
flavor = 'ucs2' if sys.maxunicode == 65535 else 'ucs4'
return '{base}-{tags}-{flavor}'.format(base=base, tags=tags, flavor=flavor)
tags = (
grpc_version.VERSION,
'py{}'.format(sysconfig.get_python_version()),
uname[0],
uname[4],
)
ucs = 'ucs{}'.format(sysconfig.get_config_var('Py_UNICODE_SIZE'))
return '{base}-{tags}-{ucs}'.format(
base=base, tags='-'.join(tags), ucs=ucs)
class BuildTaggedExt(setuptools.Command):

@ -15,3 +15,6 @@ Metrics/CyclomaticComplexity:
Metrics/PerceivedComplexity:
Max: 8
Metrics/ClassLength:
Max: 250

@ -107,7 +107,9 @@ module GRPC
# Starts running the jobs in the thread pool.
def start
fail 'already stopped' if @stopped
@stop_mutex.synchronize do
fail 'already stopped' if @stopped
end
until @workers.size == @size.to_i
next_thread = Thread.new do
catch(:exit) do # allows { throw :exit } to kill a thread
@ -264,10 +266,10 @@ module GRPC
@pool = Pool.new(@pool_size)
@run_cond = ConditionVariable.new
@run_mutex = Mutex.new
@running = false
# running_state can take 4 values: :not_started, :running, :stopping, and
# :stopped. State transitions can only proceed in that order.
@running_state = :not_started
@server = RpcServer.setup_srv(server_override, @cq, **kw)
@stopped = false
@stop_mutex = Mutex.new
end
# stops a running server
@ -275,27 +277,42 @@ module GRPC
# the call has no impact if the server is already stopped, otherwise
# server's current call loop is it's last.
def stop
return unless @running
@stop_mutex.synchronize do
@stopped = true
@run_mutex.synchronize do
fail 'Cannot stop before starting' if @running_state == :not_started
return if @running_state != :running
transition_running_state(:stopping)
end
deadline = from_relative_time(@poll_period)
return if @server.close(@cq, deadline)
deadline = from_relative_time(@poll_period)
@server.close(@cq, deadline)
@pool.stop
end
# determines if the server has been stopped
def stopped?
@stop_mutex.synchronize do
return @stopped
def running_state
@run_mutex.synchronize do
return @running_state
end
end
# Can only be called while holding @run_mutex
def transition_running_state(target_state)
state_transitions = {
not_started: :running,
running: :stopping,
stopping: :stopped
}
if state_transitions[@running_state] == target_state
@running_state = target_state
else
fail "Bad server state transition: #{@running_state}->#{target_state}"
end
end
# determines if the server is currently running
def running?
@running
running_state == :running
end
def stopped?
running_state == :stopped
end
# Is called from other threads to wait for #run to start up the server.
@ -304,13 +321,11 @@ module GRPC
#
# @param timeout [Numeric] number of seconds to wait
# @result [true, false] true if the server is running, false otherwise
def wait_till_running(timeout = 0.1)
end_time, sleep_period = Time.now + timeout, (1.0 * timeout) / 100
while Time.now < end_time
@run_mutex.synchronize { @run_cond.wait(@run_mutex) } unless running?
sleep(sleep_period)
def wait_till_running(timeout = nil)
@run_mutex.synchronize do
@run_cond.wait(@run_mutex, timeout) if @running_state == :not_started
return @running_state == :running
end
running?
end
# Runs the server in its own thread, then waits for signal INT or TERM on
@ -360,11 +375,14 @@ module GRPC
# @param service [Object|Class] a service class or object as described
# above
def handle(service)
fail 'cannot add services if the server is running' if running?
fail 'cannot add services if the server is stopped' if stopped?
cls = service.is_a?(Class) ? service : service.class
assert_valid_service_class(cls)
add_rpc_descs_for(service)
@run_mutex.synchronize do
unless @running_state == :not_started
fail 'cannot add services if the server has been started'
end
cls = service.is_a?(Class) ? service : service.class
assert_valid_service_class(cls)
add_rpc_descs_for(service)
end
end
# runs the server
@ -375,16 +393,13 @@ module GRPC
# - #running? returns true after this is called, until #stop cause the
# the server to stop.
def run
if rpc_descs.size.zero?
GRPC.logger.warn('did not run as no services were present')
return
end
@run_mutex.synchronize do
@running = true
@run_cond.signal
fail 'cannot run without registering services' if rpc_descs.size.zero?
@pool.start
@server.start
transition_running_state(:running)
@run_cond.broadcast
end
@pool.start
@server.start
loop_handle_server_calls
end
@ -413,9 +428,9 @@ module GRPC
# handles calls to the server
def loop_handle_server_calls
fail 'not running' unless @running
fail 'not started' if running_state == :not_started
loop_tag = Object.new
until stopped?
while running_state == :running
begin
an_rpc = @server.request_call(@cq, loop_tag, INFINITE_FUTURE)
break if (!an_rpc.nil?) && an_rpc.call.nil?
@ -430,11 +445,14 @@ module GRPC
rescue Core::CallError, RuntimeError => e
# these might happen for various reasonse. The correct behaviour of
# the server is to log them and continue, if it's not shutting down.
GRPC.logger.warn("server call failed: #{e}") unless stopped?
if running_state == :running
GRPC.logger.warn("server call failed: #{e}")
end
next
end
end
@running = false
# @running_state should be :stopping here
@run_mutex.synchronize { transition_running_state(:stopped) }
GRPC.logger.info("stopped: #{self}")
end
@ -484,9 +502,10 @@ module GRPC
cls.assert_rpc_descs_have_methods
end
# This should be called while holding @run_mutex
def add_rpc_descs_for(service)
cls = service.is_a?(Class) ? service : service.class
specs, handlers = rpc_descs, rpc_handlers
specs, handlers = (@rpc_descs ||= {}), (@rpc_handlers ||= {})
cls.rpc_descs.each_pair do |name, spec|
route = "/#{cls.service_name}/#{name}".to_sym
fail "already registered: rpc #{route} from #{spec}" if specs.key? route

@ -1,4 +1,4 @@
# Copyright 2015, Google Inc.
# Copyright 2015-2016, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
@ -220,19 +220,10 @@ describe GRPC::RpcServer do
@srv = RpcServer.new(**opts)
end
after(:each) do
@srv.stop
end
it 'starts out false' do
expect(@srv.stopped?).to be(false)
end
it 'stays false after a #stop is called before #run' do
@srv.stop
expect(@srv.stopped?).to be(false)
end
it 'stays false after the server starts running', server: true do
@srv.handle(EchoService)
t = Thread.new { @srv.run }
@ -247,8 +238,8 @@ describe GRPC::RpcServer do
t = Thread.new { @srv.run }
@srv.wait_till_running
@srv.stop
expect(@srv.stopped?).to be(true)
t.join
expect(@srv.stopped?).to be(true)
end
end
@ -266,9 +257,7 @@ describe GRPC::RpcServer do
server_override: @server
}
r = RpcServer.new(**opts)
r.run
expect(r.running?).to be(false)
r.stop
expect { r.run }.to raise_error(RuntimeError)
end
it 'is true after run is called with a registered service' do
@ -293,10 +282,6 @@ describe GRPC::RpcServer do
@srv = RpcServer.new(**@opts)
end
after(:each) do
@srv.stop
end
it 'raises if #run has already been called' do
@srv.handle(EchoService)
t = Thread.new { @srv.run }
@ -528,10 +513,6 @@ describe GRPC::RpcServer do
@srv = RpcServer.new(**server_opts)
end
after(:each) do
@srv.stop
end
it 'should be added to BadStatus when requests fail', server: true do
service = FailingService.new
@srv.handle(service)

@ -1,6 +1,6 @@
/*
*
* Copyright 2015, Google Inc.
* Copyright 2015-2016, Google Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -38,6 +38,8 @@
#include <grpc/support/alloc.h>
#include <grpc/support/log.h>
#include <grpc/support/useful.h>
#include "test/core/util/test_config.h"
static gpr_timespec random_deadline(void) {
@ -57,79 +59,6 @@ static grpc_timer *create_test_elements(size_t num_elements) {
return elems;
}
static int cmp_elem(const void *a, const void *b) {
int i = *(const int *)a;
int j = *(const int *)b;
return i - j;
}
static size_t *all_top(grpc_timer_heap *pq, size_t *n) {
size_t *vec = NULL;
size_t *need_to_check_children;
size_t num_need_to_check_children = 0;
*n = 0;
if (pq->timer_count == 0) return vec;
need_to_check_children =
gpr_malloc(pq->timer_count * sizeof(*need_to_check_children));
need_to_check_children[num_need_to_check_children++] = 0;
vec = gpr_malloc(pq->timer_count * sizeof(*vec));
while (num_need_to_check_children > 0) {
size_t ind = need_to_check_children[0];
size_t leftchild, rightchild;
num_need_to_check_children--;
memmove(need_to_check_children, need_to_check_children + 1,
num_need_to_check_children * sizeof(*need_to_check_children));
vec[(*n)++] = ind;
leftchild = 1u + 2u * ind;
if (leftchild < pq->timer_count) {
if (gpr_time_cmp(pq->timers[leftchild]->deadline,
pq->timers[ind]->deadline) >= 0) {
need_to_check_children[num_need_to_check_children++] = leftchild;
}
rightchild = leftchild + 1;
if (rightchild < pq->timer_count &&
gpr_time_cmp(pq->timers[rightchild]->deadline,
pq->timers[ind]->deadline) >= 0) {
need_to_check_children[num_need_to_check_children++] = rightchild;
}
}
}
gpr_free(need_to_check_children);
return vec;
}
static void check_pq_top(grpc_timer *elements, grpc_timer_heap *pq,
uint8_t *inpq, size_t num_elements) {
gpr_timespec max_deadline = gpr_inf_past(GPR_CLOCK_REALTIME);
size_t *max_deadline_indices =
gpr_malloc(num_elements * sizeof(*max_deadline_indices));
size_t *top_elements;
size_t num_max_deadline_indices = 0;
size_t num_top_elements;
size_t i;
for (i = 0; i < num_elements; ++i) {
if (inpq[i] && gpr_time_cmp(elements[i].deadline, max_deadline) >= 0) {
if (gpr_time_cmp(elements[i].deadline, max_deadline) > 0) {
num_max_deadline_indices = 0;
max_deadline = elements[i].deadline;
}
max_deadline_indices[num_max_deadline_indices++] = elements[i].heap_index;
}
}
qsort(max_deadline_indices, num_max_deadline_indices,
sizeof(*max_deadline_indices), cmp_elem);
top_elements = all_top(pq, &num_top_elements);
GPR_ASSERT(num_top_elements == num_max_deadline_indices);
for (i = 0; i < num_top_elements; i++) {
GPR_ASSERT(max_deadline_indices[i] == top_elements[i]);
}
gpr_free(max_deadline_indices);
gpr_free(top_elements);
}
static int contains(grpc_timer_heap *pq, grpc_timer *el) {
size_t i;
for (i = 0; i < pq->timer_count; i++) {
@ -145,15 +74,19 @@ static void check_valid(grpc_timer_heap *pq) {
size_t right_child = left_child + 1u;
if (left_child < pq->timer_count) {
GPR_ASSERT(gpr_time_cmp(pq->timers[i]->deadline,
pq->timers[left_child]->deadline) >= 0);
pq->timers[left_child]->deadline) <= 0);
}
if (right_child < pq->timer_count) {
GPR_ASSERT(gpr_time_cmp(pq->timers[i]->deadline,
pq->timers[right_child]->deadline) >= 0);
pq->timers[right_child]->deadline) <= 0);
}
}
}
/*******************************************************************************
* test1
*/
static void test1(void) {
grpc_timer_heap pq;
const size_t num_test_elements = 200;
@ -162,6 +95,8 @@ static void test1(void) {
grpc_timer *test_elements = create_test_elements(num_test_elements);
uint8_t *inpq = gpr_malloc(num_test_elements);
gpr_log(GPR_INFO, "test1");
grpc_timer_heap_init(&pq);
memset(inpq, 0, num_test_elements);
GPR_ASSERT(grpc_timer_heap_is_empty(&pq));
@ -172,7 +107,6 @@ static void test1(void) {
check_valid(&pq);
GPR_ASSERT(contains(&pq, &test_elements[i]));
inpq[i] = 1;
check_pq_top(test_elements, &pq, inpq, num_test_elements);
}
for (i = 0; i < num_test_elements; ++i) {
/* Test that check still succeeds even for element that wasn't just
@ -182,7 +116,7 @@ static void test1(void) {
GPR_ASSERT(pq.timer_count == num_test_elements);
check_pq_top(test_elements, &pq, inpq, num_test_elements);
check_valid(&pq);
for (i = 0; i < num_test_operations; ++i) {
size_t elem_num = (size_t)rand() % num_test_elements;
@ -193,14 +127,12 @@ static void test1(void) {
grpc_timer_heap_add(&pq, el);
GPR_ASSERT(contains(&pq, el));
inpq[elem_num] = 1;
check_pq_top(test_elements, &pq, inpq, num_test_elements);
check_valid(&pq);
} else {
GPR_ASSERT(contains(&pq, el));
grpc_timer_heap_remove(&pq, el);
GPR_ASSERT(!contains(&pq, el));
inpq[elem_num] = 0;
check_pq_top(test_elements, &pq, inpq, num_test_elements);
check_valid(&pq);
}
}
@ -210,7 +142,108 @@ static void test1(void) {
gpr_free(inpq);
}
/*******************************************************************************
* test2
*/
typedef struct {
grpc_timer elem;
bool inserted;
} elem_struct;
static elem_struct *search_elems(elem_struct *elems, size_t count,
bool inserted) {
size_t *search_order = gpr_malloc(count * sizeof(*search_order));
for (size_t i = 0; i < count; i++) {
search_order[i] = i;
}
for (size_t i = 0; i < count * 2; i++) {
size_t a = (size_t)rand() % count;
size_t b = (size_t)rand() % count;
GPR_SWAP(size_t, search_order[a], search_order[b]);
}
elem_struct *out = NULL;
for (size_t i = 0; out == NULL && i < count; i++) {
if (elems[search_order[i]].inserted == inserted) {
out = &elems[search_order[i]];
}
}
gpr_free(search_order);
return out;
}
static void test2(void) {
gpr_log(GPR_INFO, "test2");
grpc_timer_heap pq;
elem_struct elems[1000];
size_t num_inserted = 0;
grpc_timer_heap_init(&pq);
memset(elems, 0, sizeof(elems));
for (size_t round = 0; round < 10000; round++) {
int r = rand() % 1000;
if (r <= 550) {
/* 55% of the time we try to add something */
elem_struct *el = search_elems(elems, GPR_ARRAY_SIZE(elems), false);
if (el != NULL) {
el->elem.deadline = random_deadline();
grpc_timer_heap_add(&pq, &el->elem);
el->inserted = true;
num_inserted++;
check_valid(&pq);
}
} else if (r <= 650) {
/* 10% of the time we try to remove something */
elem_struct *el = search_elems(elems, GPR_ARRAY_SIZE(elems), true);
if (el != NULL) {
grpc_timer_heap_remove(&pq, &el->elem);
el->inserted = false;
num_inserted--;
check_valid(&pq);
}
} else {
/* the remaining times we pop */
if (num_inserted > 0) {
grpc_timer *top = grpc_timer_heap_top(&pq);
grpc_timer_heap_pop(&pq);
for (size_t i = 0; i < GPR_ARRAY_SIZE(elems); i++) {
if (top == &elems[i].elem) {
GPR_ASSERT(elems[i].inserted);
elems[i].inserted = false;
}
}
num_inserted--;
check_valid(&pq);
}
}
if (num_inserted) {
gpr_timespec *min_deadline = NULL;
for (size_t i = 0; i < GPR_ARRAY_SIZE(elems); i++) {
if (elems[i].inserted) {
if (min_deadline == NULL) {
min_deadline = &elems[i].elem.deadline;
} else {
if (gpr_time_cmp(elems[i].elem.deadline, *min_deadline) < 0) {
min_deadline = &elems[i].elem.deadline;
}
}
}
}
GPR_ASSERT(
0 == gpr_time_cmp(grpc_timer_heap_top(&pq)->deadline, *min_deadline));
}
}
grpc_timer_heap_destroy(&pq);
}
static void shrink_test(void) {
gpr_log(GPR_INFO, "shrink_test");
grpc_timer_heap pq;
size_t i;
size_t expected_size;
@ -274,6 +307,7 @@ int main(int argc, char **argv) {
for (i = 0; i < 5; i++) {
test1();
test2();
shrink_test();
}

@ -1,5 +1,5 @@
#!/usr/bin/env python2.7
# Copyright 2015, Google Inc.
# Copyright 2015-2016, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
@ -51,29 +51,35 @@ SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
PROJECT_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, '..', '..', '..'))
CONFIG = args.config
SETUP_PATH = os.path.join(PROJECT_ROOT, 'src/python/grpcio/setup.py')
DOC_PATH = os.path.join(PROJECT_ROOT, 'src/python/grpcio/doc/build')
SETUP_PATH = os.path.join(PROJECT_ROOT, 'setup.py')
REQUIREMENTS_PATH = os.path.join(PROJECT_ROOT, 'requirements.txt')
DOC_PATH = os.path.join(PROJECT_ROOT, 'doc/build')
INCLUDE_PATH = os.path.join(PROJECT_ROOT, 'include')
LIBRARY_PATH = os.path.join(PROJECT_ROOT, 'libs/{}'.format(CONFIG))
VIRTUALENV_DIR = os.path.join(SCRIPT_DIR, 'distrib_virtualenv')
VIRTUALENV_PYTHON_PATH = os.path.join(VIRTUALENV_DIR, 'bin', 'python')
VIRTUALENV_PIP_PATH = os.path.join(VIRTUALENV_DIR, 'bin', 'pip')
environment = os.environ.copy()
environment.update({
'CONFIG': CONFIG,
'CFLAGS': '-I{}'.format(INCLUDE_PATH),
'LDFLAGS': '-L{}'.format(LIBRARY_PATH),
'LD_LIBRARY_PATH': LIBRARY_PATH
'LD_LIBRARY_PATH': LIBRARY_PATH,
'GRPC_PYTHON_BUILD_WITH_CYTHON': '1',
})
subprocess_arguments_list = [
{'args': ['make'], 'cwd': PROJECT_ROOT},
{'args': ['virtualenv', VIRTUALENV_DIR], 'env': environment},
{'args': [VIRTUALENV_PIP_PATH, 'install', '-r', REQUIREMENTS_PATH],
'env': environment},
{'args': [VIRTUALENV_PYTHON_PATH, SETUP_PATH, 'build'], 'env': environment},
{'args': [VIRTUALENV_PYTHON_PATH, SETUP_PATH, 'doc'], 'env': environment},
]
for subprocess_arguments in subprocess_arguments_list:
print('Running command: {}'.format(subprocess_arguments['args']))
subprocess.check_call(**subprocess_arguments)
if args.submit:

@ -69,16 +69,6 @@ def create_jobspec(name, cmdline, environ=None, shell=False,
return jobspec
def macos_arch_env(arch):
"""Returns environ specifying -arch arguments for make."""
if arch == 'x86':
arch_arg = '-arch i386'
elif arch == 'x64':
arch_arg = '-arch x86_64'
else:
raise Exception('Unsupported arch')
return {'CFLAGS': arch_arg, 'LDFLAGS': arch_arg}
_MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.7'
_ARCH_FLAG_MAP = {
@ -191,13 +181,17 @@ class CSharpExtArtifact:
environ = {'CONFIG': 'opt',
'EMBED_OPENSSL': 'true',
'EMBED_ZLIB': 'true',
'CFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE'}
'CFLAGS': '-DGPR_BACKWARDS_COMPATIBILITY_MODE',
'LDFLAGS': ''}
if self.platform == 'linux':
return create_docker_jobspec(self.name,
'tools/dockerfile/grpc_artifact_linux_%s' % self.arch,
'tools/run_tests/build_artifact_csharp.sh')
'tools/run_tests/build_artifact_csharp.sh',
environ=environ)
else:
environ.update(macos_arch_env(self.arch))
archflag = _ARCH_FLAG_MAP[self.arch]
environ['CFLAGS'] += ' %s %s' % (archflag, _MACOS_COMPAT_FLAG)
environ['LDFLAGS'] += ' %s' % archflag
return create_jobspec(self.name,
['tools/run_tests/build_artifact_csharp.sh'],
environ=environ)

@ -40,7 +40,11 @@ export PATH=$ROOT/bins/$CONFIG:$ROOT/bins/$CONFIG/protobuf:$PATH
export CFLAGS="-I$ROOT/include -std=gnu99"
export LDFLAGS="-L$ROOT/libs/$CONFIG"
export GRPC_PYTHON_BUILD_WITH_CYTHON=1
export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1
if [ "$CONFIG" = "gcov" ]
then
export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1
fi
tox --notest

@ -40,10 +40,10 @@ export PATH=$ROOT/bins/$CONFIG:$ROOT/bins/$CONFIG/protobuf:$PATH
export CFLAGS="-I$ROOT/include -std=c89"
export LDFLAGS="-L$ROOT/libs/$CONFIG"
export GRPC_PYTHON_BUILD_WITH_CYTHON=1
export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1
if [ "$CONFIG" = "gcov" ]
then
export GRPC_PYTHON_ENABLE_CYTHON_TRACING=1
tox
else
$ROOT/.tox/py27/bin/python $ROOT/setup.py test_lite

@ -360,7 +360,7 @@ class PythonLanguage(object):
['tools/run_tests/run_python.sh'],
None,
environ=dict(environment.items() +
[('GPRC_PYTHON_TESTRUNNER_FILTER', suite_name)]),
[('GRPC_PYTHON_TESTRUNNER_FILTER', suite_name)]),
shortname='py.test.%s' % suite_name,
timeout_seconds=5*60)
for suite_name in tests_json]

Loading…
Cancel
Save