Merge remote-tracking branch 'upstream/master' into seperate_generated_files

pull/7451/head
Yuchen Zeng 9 years ago
commit 543c5886ee
  1. 1
      doc/statuscodes.md
  2. 2
      include/grpc++/impl/codegen/async_unary_call.h
  3. 17
      include/grpc/impl/codegen/compression_types.h
  4. 38
      include/grpc/impl/codegen/grpc_types.h
  5. 28
      src/core/lib/iomgr/ev_poll_posix.c
  6. 8
      src/objective-c/GRPCClient/GRPCCall.m
  7. 2
      tools/gcp/stress_test/stress_test_utils.py
  8. 28
      tools/profiling/latency_profile/profile_analyzer.py

@ -18,6 +18,7 @@ Only a subset of the pre-defined status codes are generated by the gRPC librarie
| Could not decompress, but compression algorithm supported (Server -> Client) | INTERNAL | Client | | Could not decompress, but compression algorithm supported (Server -> Client) | INTERNAL | Client |
| Compression mechanism used by client not supported at server | UNIMPLEMENTED | Server | | Compression mechanism used by client not supported at server | UNIMPLEMENTED | Server |
| Server temporarily out of resources (e.g., Flow-control resource limits reached) | RESOURCE_EXHAUSTED | Server| | Server temporarily out of resources (e.g., Flow-control resource limits reached) | RESOURCE_EXHAUSTED | Server|
| Client does not have enough memory to hold the server response | RESOURCE_EXHAUSTED | Client |
| Flow-control protocol violation | INTERNAL | Both | | Flow-control protocol violation | INTERNAL | Both |
| Error parsing returned status | UNKNOWN | Client | | Error parsing returned status | UNKNOWN | Client |
| Incorrect Auth metadata ( Credentials failed to get metadata, Incompatible credentials set on channel and call, Invalid host set in `:authority` metadata, etc.) | UNAUTHENTICATED | Both | | Incorrect Auth metadata ( Credentials failed to get metadata, Incompatible credentials set on channel and call, Invalid host set in `:authority` metadata, etc.) | UNAUTHENTICATED | Both |

@ -65,7 +65,7 @@ class ClientAsyncResponseReader GRPC_FINAL
const W& request) const W& request)
: context_(context), : context_(context),
call_(channel->CreateCall(method, context, cq)), call_(channel->CreateCall(method, context, cq)),
collection_(new CallOpSetCollection) { collection_(std::make_shared<CallOpSetCollection>()) {
collection_->init_buf_.SetCollection(collection_); collection_->init_buf_.SetCollection(collection_);
collection_->init_buf_.SendInitialMetadata( collection_->init_buf_.SendInitialMetadata(
context->send_initial_metadata_, context->initial_metadata_flags()); context->send_initial_metadata_, context->initial_metadata_flags());

@ -46,12 +46,27 @@ extern "C" {
#define GRPC_COMPRESSION_REQUEST_ALGORITHM_MD_KEY \ #define GRPC_COMPRESSION_REQUEST_ALGORITHM_MD_KEY \
"grpc-internal-encoding-request" "grpc-internal-encoding-request"
/** To be used in channel arguments */ /** To be used in channel arguments.
*
* \addtogroup grpc_arg_keys
* \{ */
/** Default compression algorithm for the channel.
* Its value is an int from the \a grpc_compression_algorithm enum. */
#define GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM \ #define GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM \
"grpc.default_compression_algorithm" "grpc.default_compression_algorithm"
/** Default compression level for the channel.
* Its value is an int from the \a grpc_compression_level enum. */
#define GRPC_COMPRESSION_CHANNEL_DEFAULT_LEVEL "grpc.default_compression_level" #define GRPC_COMPRESSION_CHANNEL_DEFAULT_LEVEL "grpc.default_compression_level"
/** Compression algorithms supported by the channel.
* Its value is a bitset (an int). Bits correspond to algorithms in \a
* grpc_compression_algorithm. For example, its LSB corresponds to
* GRPC_COMPRESS_NONE, the next bit to GRPC_COMPRESS_DEFLATE, etc.
* Unset bits disable support for the algorithm. By default all algorithms are
* supported. It's not possible to disable GRPC_COMPRESS_NONE (the attempt will
* be ignored). */
#define GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET \ #define GRPC_COMPRESSION_CHANNEL_ENABLED_ALGORITHMS_BITSET \
"grpc.compression_enabled_algorithms_bitset" "grpc.compression_enabled_algorithms_bitset"
/** \} */
/* The various compression algorithms supported by gRPC */ /* The various compression algorithms supported by gRPC */
typedef enum { typedef enum {

@ -106,58 +106,66 @@ typedef struct {
by grpc_arg; keys are strings to allow easy backwards-compatible extension by grpc_arg; keys are strings to allow easy backwards-compatible extension
by arbitrary parties. by arbitrary parties.
All evaluation is performed at channel creation time (i.e. the values in All evaluation is performed at channel creation time (i.e. the values in
this structure need only live through the creation invocation). */ this structure need only live through the creation invocation).
See the description of the \ref grpc_arg_keys "available args" for more
details. */
typedef struct { typedef struct {
size_t num_args; size_t num_args;
grpc_arg *args; grpc_arg *args;
} grpc_channel_args; } grpc_channel_args;
/* Channel argument keys: */ /** \defgroup grpc_arg_keys
/** Enable census for tracing and stats collection */ * Channel argument keys.
* \{
*/
/** If non-zero, enable census for tracing and stats collection. */
#define GRPC_ARG_ENABLE_CENSUS "grpc.census" #define GRPC_ARG_ENABLE_CENSUS "grpc.census"
/** Enable load reporting */ /** If non-zero, enable load reporting. */
#define GRPC_ARG_ENABLE_LOAD_REPORTING "grpc.loadreporting" #define GRPC_ARG_ENABLE_LOAD_REPORTING "grpc.loadreporting"
/** Maximum number of concurrent incoming streams to allow on a http2 /** Maximum number of concurrent incoming streams to allow on a http2
connection */ connection. Int valued. */
#define GRPC_ARG_MAX_CONCURRENT_STREAMS "grpc.max_concurrent_streams" #define GRPC_ARG_MAX_CONCURRENT_STREAMS "grpc.max_concurrent_streams"
/** Maximum message length that the channel can receive */ /** Maximum message length that the channel can receive. Int valued, bytes. */
#define GRPC_ARG_MAX_MESSAGE_LENGTH "grpc.max_message_length" #define GRPC_ARG_MAX_MESSAGE_LENGTH "grpc.max_message_length"
/** Initial sequence number for http2 transports */ /** Initial sequence number for http2 transports. Int valued. */
#define GRPC_ARG_HTTP2_INITIAL_SEQUENCE_NUMBER \ #define GRPC_ARG_HTTP2_INITIAL_SEQUENCE_NUMBER \
"grpc.http2.initial_sequence_number" "grpc.http2.initial_sequence_number"
/** Amount to read ahead on individual streams. Defaults to 64kb, larger /** Amount to read ahead on individual streams. Defaults to 64kb, larger
values can help throughput on high-latency connections. values can help throughput on high-latency connections.
NOTE: at some point we'd like to auto-tune this, and this parameter NOTE: at some point we'd like to auto-tune this, and this parameter
will become a no-op. */ will become a no-op. Int valued, bytes. */
#define GRPC_ARG_HTTP2_STREAM_LOOKAHEAD_BYTES "grpc.http2.lookahead_bytes" #define GRPC_ARG_HTTP2_STREAM_LOOKAHEAD_BYTES "grpc.http2.lookahead_bytes"
/** How much memory to use for hpack decoding */ /** How much memory to use for hpack decoding. Int valued, bytes. */
#define GRPC_ARG_HTTP2_HPACK_TABLE_SIZE_DECODER \ #define GRPC_ARG_HTTP2_HPACK_TABLE_SIZE_DECODER \
"grpc.http2.hpack_table_size.decoder" "grpc.http2.hpack_table_size.decoder"
/** How much memory to use for hpack encoding */ /** How much memory to use for hpack encoding. Int valued, bytes. */
#define GRPC_ARG_HTTP2_HPACK_TABLE_SIZE_ENCODER \ #define GRPC_ARG_HTTP2_HPACK_TABLE_SIZE_ENCODER \
"grpc.http2.hpack_table_size.encoder" "grpc.http2.hpack_table_size.encoder"
/** Default authority to pass if none specified on call construction */ /** Default authority to pass if none specified on call construction. A string.
* */
#define GRPC_ARG_DEFAULT_AUTHORITY "grpc.default_authority" #define GRPC_ARG_DEFAULT_AUTHORITY "grpc.default_authority"
/** Primary user agent: goes at the start of the user-agent metadata /** Primary user agent: goes at the start of the user-agent metadata
sent on each request */ sent on each request. A string. */
#define GRPC_ARG_PRIMARY_USER_AGENT_STRING "grpc.primary_user_agent" #define GRPC_ARG_PRIMARY_USER_AGENT_STRING "grpc.primary_user_agent"
/** Secondary user agent: goes at the end of the user-agent metadata /** Secondary user agent: goes at the end of the user-agent metadata
sent on each request */ sent on each request. A string. */
#define GRPC_ARG_SECONDARY_USER_AGENT_STRING "grpc.secondary_user_agent" #define GRPC_ARG_SECONDARY_USER_AGENT_STRING "grpc.secondary_user_agent"
/** The maximum time between subsequent connection attempts, in ms */ /** The maximum time between subsequent connection attempts, in ms */
#define GRPC_ARG_MAX_RECONNECT_BACKOFF_MS "grpc.max_reconnect_backoff_ms" #define GRPC_ARG_MAX_RECONNECT_BACKOFF_MS "grpc.max_reconnect_backoff_ms"
/* The caller of the secure_channel_create functions may override the target /* The caller of the secure_channel_create functions may override the target
name used for SSL host name checking using this channel argument which is of name used for SSL host name checking using this channel argument which is of
type GRPC_ARG_STRING. This *should* be used for testing only. type \a GRPC_ARG_STRING. This *should* be used for testing only.
If this argument is not specified, the name used for SSL host name checking If this argument is not specified, the name used for SSL host name checking
will be the target parameter (assuming that the secure channel is an SSL will be the target parameter (assuming that the secure channel is an SSL
channel). If this parameter is specified and the underlying is not an SSL channel). If this parameter is specified and the underlying is not an SSL
channel, it will just be ignored. */ channel, it will just be ignored. */
#define GRPC_SSL_TARGET_NAME_OVERRIDE_ARG "grpc.ssl_target_name_override" #define GRPC_SSL_TARGET_NAME_OVERRIDE_ARG "grpc.ssl_target_name_override"
/* Maximum metadata size */ /* Maximum metadata size, in bytes. */
#define GRPC_ARG_MAX_METADATA_SIZE "grpc.max_metadata_size" #define GRPC_ARG_MAX_METADATA_SIZE "grpc.max_metadata_size"
/** If non-zero, allow the use of SO_REUSEPORT if it's available (default 1) */ /** If non-zero, allow the use of SO_REUSEPORT if it's available (default 1) */
#define GRPC_ARG_ALLOW_REUSEPORT "grpc.so_reuseport" #define GRPC_ARG_ALLOW_REUSEPORT "grpc.so_reuseport"
/** \} */
/** Result of a grpc call. If the caller satisfies the prerequisites of a /** Result of a grpc call. If the caller satisfies the prerequisites of a
particular operation, the grpc_call_error returned will be GRPC_CALL_OK. particular operation, the grpc_call_error returned will be GRPC_CALL_OK.

@ -844,6 +844,11 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
*worker_hdl = &worker; *worker_hdl = &worker;
grpc_error *error = GRPC_ERROR_NONE; grpc_error *error = GRPC_ERROR_NONE;
/* Avoid malloc for small number of elements. */
enum { inline_elements = 96 };
struct pollfd pollfd_space[inline_elements];
struct grpc_fd_watcher watcher_space[inline_elements];
/* pollset->mu already held */ /* pollset->mu already held */
int added_worker = 0; int added_worker = 0;
int locked = 1; int locked = 1;
@ -899,15 +904,23 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
int r; int r;
size_t i, fd_count; size_t i, fd_count;
nfds_t pfd_count; nfds_t pfd_count;
/* TODO(ctiller): inline some elements to avoid an allocation */
grpc_fd_watcher *watchers; grpc_fd_watcher *watchers;
struct pollfd *pfds; struct pollfd *pfds;
timeout = poll_deadline_to_millis_timeout(deadline, now); timeout = poll_deadline_to_millis_timeout(deadline, now);
/* TODO(ctiller): perform just one malloc here if we exceed the inline
* case */ if (pollset->fd_count + 2 <= inline_elements) {
pfds = gpr_malloc(sizeof(*pfds) * (pollset->fd_count + 2)); pfds = pollfd_space;
watchers = gpr_malloc(sizeof(*watchers) * (pollset->fd_count + 2)); watchers = watcher_space;
} else {
/* Allocate one buffer to hold both pfds and watchers arrays */
const size_t pfd_size = sizeof(*pfds) * (pollset->fd_count + 2);
const size_t watch_size = sizeof(*watchers) * (pollset->fd_count + 2);
void *buf = gpr_malloc(pfd_size + watch_size);
pfds = buf;
watchers = (void *)((char *)buf + pfd_size);
}
fd_count = 0; fd_count = 0;
pfd_count = 2; pfd_count = 2;
pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd); pfds[0].fd = GRPC_WAKEUP_FD_GET_READ_FD(&grpc_global_wakeup_fd);
@ -974,8 +987,11 @@ static grpc_error *pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
} }
} }
if (pfds != pollfd_space) {
/* pfds and watchers are in the same memory block pointed to by pfds */
gpr_free(pfds); gpr_free(pfds);
gpr_free(watchers); }
GPR_TIMER_END("maybe_work_and_unlock", 0); GPR_TIMER_END("maybe_work_and_unlock", 0);
locked = 0; locked = 0;
} else { } else {

@ -208,13 +208,9 @@ NSString * const kGRPCTrailersKey = @"io.grpc.TrailersKey";
// don't want to throw, because the app shouldn't crash for a behavior // don't want to throw, because the app shouldn't crash for a behavior
// that's on the hands of any server to have. Instead we finish and ask // that's on the hands of any server to have. Instead we finish and ask
// the server to cancel. // the server to cancel.
//
// TODO(jcanizales): No canonical code is appropriate for this situation
// (because it's just a client problem). Use another domain and an
// appropriately-documented code.
[weakSelf finishWithError:[NSError errorWithDomain:kGRPCErrorDomain [weakSelf finishWithError:[NSError errorWithDomain:kGRPCErrorDomain
code:GRPCErrorCodeInternal code:GRPCErrorCodeResourceExhausted
userInfo:nil]]; userInfo:@{NSLocalizedDescriptionKey: @"Client does not have enough memory to hold the server response."}]];
[weakSelf cancelCall]; [weakSelf cancelCall];
return; return;
} }

@ -121,7 +121,7 @@ class BigQueryHelper:
if not page['jobComplete']: if not page['jobComplete']:
print('TIMEOUT ERROR: The query %s timed out. Current timeout value is' print('TIMEOUT ERROR: The query %s timed out. Current timeout value is'
' %d msec. Returning False (i.e assuming there are no failures)' ' %d msec. Returning False (i.e assuming there are no failures)'
) % (query, timeoout_msec) ) % (query, timeout_msec)
return False return False
num_failures = int(page['totalRows']) num_failures = int(page['totalRows'])

@ -43,6 +43,7 @@ TIME_FROM_SCOPE_START = object()
TIME_TO_SCOPE_END = object() TIME_TO_SCOPE_END = object()
TIME_FROM_STACK_START = object() TIME_FROM_STACK_START = object()
TIME_TO_STACK_END = object() TIME_TO_STACK_END = object()
TIME_FROM_LAST_IMPORTANT = object()
argp = argparse.ArgumentParser(description='Process output of basic_prof builds') argp = argparse.ArgumentParser(description='Process output of basic_prof builds')
@ -78,10 +79,14 @@ class ScopeBuilder(object):
self.call_stack_builder.lines.append(line_item) self.call_stack_builder.lines.append(line_item)
def finish(self, line): def finish(self, line):
assert line['tag'] == self.top_line.tag, 'expected %s, got %s; thread=%s; t0=%f t1=%f' % (self.top_line.tag, line['tag'], line['thd'], self.top_line.start_time, line['t']) assert line['tag'] == self.top_line.tag, (
'expected %s, got %s; thread=%s; t0=%f t1=%f' %
(self.top_line.tag, line['tag'], line['thd'], self.top_line.start_time,
line['t']))
final_time_stamp = line['t'] final_time_stamp = line['t']
assert self.top_line.end_time is None assert self.top_line.end_time is None
self.top_line.end_time = final_time_stamp self.top_line.end_time = final_time_stamp
self.top_line.important = self.top_line.important or line['imp']
assert SELF_TIME not in self.top_line.times assert SELF_TIME not in self.top_line.times
self.top_line.times[SELF_TIME] = final_time_stamp - self.top_line.start_time self.top_line.times[SELF_TIME] = final_time_stamp - self.top_line.start_time
for line in self.call_stack_builder.lines[self.first_child_pos:]: for line in self.call_stack_builder.lines[self.first_child_pos:]:
@ -101,9 +106,14 @@ class CallStackBuilder(object):
start_time = self.lines[0].start_time start_time = self.lines[0].start_time
end_time = self.lines[0].end_time end_time = self.lines[0].end_time
self.signature = self.signature.hexdigest() self.signature = self.signature.hexdigest()
last_important = start_time
for line in self.lines: for line in self.lines:
line.times[TIME_FROM_STACK_START] = line.start_time - start_time line.times[TIME_FROM_STACK_START] = line.start_time - start_time
line.times[TIME_TO_STACK_END] = end_time - line.end_time line.times[TIME_TO_STACK_END] = end_time - line.end_time
line.times[TIME_FROM_LAST_IMPORTANT] = line.start_time - last_important
if line.important:
last_important = line.end_time
last_important = end_time
def add(self, line): def add(self, line):
line_type = line['type'] line_type = line['type']
@ -113,7 +123,9 @@ class CallStackBuilder(object):
self.stk.append(ScopeBuilder(self, line)) self.stk.append(ScopeBuilder(self, line))
return False return False
elif line_type == '}': elif line_type == '}':
assert self.stk, 'expected non-empty stack for closing %s; thread=%s; t=%f' % (line['tag'], line['thd'], line['t']) assert self.stk, (
'expected non-empty stack for closing %s; thread=%s; t=%f' %
(line['tag'], line['thd'], line['t']))
self.stk.pop().finish(line) self.stk.pop().finish(line)
if not self.stk: if not self.stk:
self.finish() self.finish()
@ -216,9 +228,16 @@ def time_format(idx):
return '' return ''
return ent return ent
BANNER = {
'simple': 'Count: %(count)d',
'html': '<h1>Count: %(count)d</h1>'
}
FORMAT = [ FORMAT = [
('TAG', lambda line: '..'*line.indent + tidy_tag(line.tag)), ('TAG', lambda line: '..'*line.indent + tidy_tag(line.tag)),
('LOC', lambda line: '%s:%d' % (line.filename[line.filename.rfind('/')+1:], line.fileline)), ('LOC', lambda line: '%s:%d' % (line.filename[line.filename.rfind('/')+1:], line.fileline)),
('IMP', lambda line: '*' if line.important else ''),
('FROM_IMP', time_format(TIME_FROM_LAST_IMPORTANT)),
('FROM_STACK_START', time_format(TIME_FROM_STACK_START)), ('FROM_STACK_START', time_format(TIME_FROM_STACK_START)),
('SELF', time_format(SELF_TIME)), ('SELF', time_format(SELF_TIME)),
('TO_STACK_END', time_format(TIME_TO_STACK_END)), ('TO_STACK_END', time_format(TIME_TO_STACK_END)),
@ -227,11 +246,6 @@ FORMAT = [
('TO_SCOPE_END', time_format(TIME_TO_SCOPE_END)), ('TO_SCOPE_END', time_format(TIME_TO_SCOPE_END)),
] ]
BANNER = {
'simple': 'Count: %(count)d',
'html': '<h1>Count: %(count)d</h1>'
}
if args.fmt == 'html': if args.fmt == 'html':
print '<html>' print '<html>'
print '<head>' print '<head>'

Loading…
Cancel
Save