From 38b32c077bf3095c3dfbdb84c2c3afe47df35c1b Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Thu, 13 Aug 2015 19:58:27 +0200 Subject: [PATCH 01/48] Better handling of _WIN32_WINNT. --- include/grpc/support/port_platform.h | 9 +++++---- vsprojects/global.props | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/include/grpc/support/port_platform.h b/include/grpc/support/port_platform.h index 57fed18cf68..d257164547a 100644 --- a/include/grpc/support/port_platform.h +++ b/include/grpc/support/port_platform.h @@ -46,10 +46,11 @@ #define NOMINMAX #endif /* NOMINMAX */ -#if defined(_WIN32_WINNT) -#if _WIN32_WINNT < 0x0600 -#undef _WIN32_WINNT -#define _WIN32_WINNT 0x0600 +#ifndef _WIN32_WINNT +#error "Please compile grpc with _WIN32_WINNT of at least 0x600 (aka Windows Vista)" +#else /* !defined(_WIN32_WINNT) */ +#if (_WIN32_WINNT < 0x0600) +#error "Please compile grpc with _WIN32_WINNT of at least 0x600 (aka Windows Vista)" #endif /* _WIN32_WINNT < 0x0600 */ #endif /* defined(_WIN32_WINNT) */ diff --git a/vsprojects/global.props b/vsprojects/global.props index 6858bd7b9d4..4647436e9e6 100644 --- a/vsprojects/global.props +++ b/vsprojects/global.props @@ -6,7 +6,7 @@ $(ProjectDir)\..\..;$(ProjectDir)\..\..\include;$(ProjectDir)\..\..\third_party\protobuf\src;$(ProjectDir)\..\packages\grpc.dependencies.openssl.1.0.2.2\build\native\include;$(ProjectDir)\..\..\third_party\gflags\include;%(AdditionalIncludeDirectories) - _SCL_SECURE_NO_WARNINGS;_CRT_SECURE_NO_WARNINGS;_UNICODE;UNICODE;%(PreprocessorDefinitions) + _WIN32_WINNT=0x600;_SCL_SECURE_NO_WARNINGS;_CRT_SECURE_NO_WARNINGS;_UNICODE;UNICODE;%(PreprocessorDefinitions) EnableAllWarnings From 41a3bc7784321b94bf014865d2ac2ac7d8b74886 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Thu, 17 Sep 2015 14:59:32 -0700 Subject: [PATCH 02/48] Update test for changed core HTTP header handling --- src/python/grpcio_test/.gitignore | 1 + .../grpc_test/_adapter/_low_test.py | 23 +++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/python/grpcio_test/.gitignore b/src/python/grpcio_test/.gitignore index 218e3a15ab4..e3540baa7cc 100644 --- a/src/python/grpcio_test/.gitignore +++ b/src/python/grpcio_test/.gitignore @@ -7,4 +7,5 @@ dist/ *.eggs/ .coverage .coverage.* +.cache nosetests.xml diff --git a/src/python/grpcio_test/grpc_test/_adapter/_low_test.py b/src/python/grpcio_test/grpc_test/_adapter/_low_test.py index 70149127da3..8115cd0e833 100644 --- a/src/python/grpcio_test/grpc_test/_adapter/_low_test.py +++ b/src/python/grpcio_test/grpc_test/_adapter/_low_test.py @@ -34,6 +34,7 @@ import unittest from grpc import _grpcio_metadata from grpc._adapter import _types from grpc._adapter import _low +from grpc_test import test_common def wait_for_events(completion_queues, deadline): @@ -140,16 +141,16 @@ class InsecureServerInsecureClient(unittest.TestCase): self.assertIsInstance(request_event.call, _low.Call) self.assertIs(server_request_tag, request_event.tag) self.assertEqual(1, len(request_event.results)) - received_initial_metadata = dict(request_event.results[0].initial_metadata) + received_initial_metadata = request_event.results[0].initial_metadata # Check that our metadata were transmitted - self.assertEqual( - dict(client_initial_metadata), - dict((x, received_initial_metadata[x]) - for x in zip(*client_initial_metadata)[0])) + self.assertTrue(test_common.metadata_transmitted(client_initial_metadata, + received_initial_metadata)) # Check that Python's user agent string is a part of the full user agent # string + received_initial_metadata_dict = dict(received_initial_metadata) + self.assertIn('user-agent', received_initial_metadata_dict) self.assertIn('Python-gRPC-{}'.format(_grpcio_metadata.__version__), - received_initial_metadata['user-agent']) + received_initial_metadata_dict['user-agent']) self.assertEqual(method, request_event.call_details.method) self.assertEqual(host, request_event.call_details.host) self.assertLess(abs(deadline - request_event.call_details.deadline), @@ -193,13 +194,15 @@ class InsecureServerInsecureClient(unittest.TestCase): self.assertNotIn(client_result.type, found_client_op_types) found_client_op_types.add(client_result.type) if client_result.type == _types.OpType.RECV_INITIAL_METADATA: - self.assertEqual(dict(server_initial_metadata), - dict(client_result.initial_metadata)) + self.assertTrue( + test_common.metadata_transmitted(server_initial_metadata, + client_result.initial_metadata)) elif client_result.type == _types.OpType.RECV_MESSAGE: self.assertEqual(response, client_result.message) elif client_result.type == _types.OpType.RECV_STATUS_ON_CLIENT: - self.assertEqual(dict(server_trailing_metadata), - dict(client_result.trailing_metadata)) + self.assertTrue( + test_common.metadata_transmitted(server_trailing_metadata, + client_result.trailing_metadata)) self.assertEqual(server_status_details, client_result.status.details) self.assertEqual(server_status_code, client_result.status.code) self.assertEqual(set([ From 91318bc256991c9946f450219721665148480cd8 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 24 Sep 2015 08:58:39 -0700 Subject: [PATCH 03/48] Add retries for flaky tests (enabled by default for now) --- tools/run_tests/jobset.py | 50 ++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index 2a863191259..e696a0e9698 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -81,6 +81,7 @@ _CLEAR_LINE = '\x1b[2K' _TAG_COLOR = { 'FAILED': 'red', + 'FLAKE': 'red', 'WARNING': 'yellow', 'TIMEOUT': 'red', 'PASSED': 'green', @@ -131,7 +132,7 @@ class JobSpec(object): """Specifies what to run for a job.""" def __init__(self, cmdline, shortname=None, environ=None, hash_targets=None, - cwd=None, shell=False, timeout_seconds=5*60): + cwd=None, shell=False, timeout_seconds=5*60, flake_retries=5): """ Arguments: cmdline: a list of arguments to pass as the command line @@ -150,6 +151,7 @@ class JobSpec(object): self.cwd = cwd self.shell = shell self.timeout_seconds = timeout_seconds + self.flake_retries = flake_retries def identity(self): return '%r %r %r' % (self.cmdline, self.environ, self.hash_targets) @@ -167,25 +169,28 @@ class Job(object): def __init__(self, spec, bin_hash, newline_on_success, travis, add_env, xml_report): self._spec = spec self._bin_hash = bin_hash + self._newline_on_success = newline_on_success + self._travis = travis + self._add_env = add_env.copy() + self._xml_test = ET.SubElement(xml_report, 'testcase', + name=self._spec.shortname) if xml_report is not None else None + self._retries = 0 + message('START', spec.shortname, do_newline=self._travis) + self.start() + + def start(self): self._tempfile = tempfile.TemporaryFile() - env = os.environ.copy() - for k, v in spec.environ.iteritems(): - env[k] = v - for k, v in add_env.iteritems(): - env[k] = v + env = dict(os.environ) + env.update(self._spec.environ) + env.update(self._add_env) self._start = time.time() - self._process = subprocess.Popen(args=spec.cmdline, + self._process = subprocess.Popen(args=self._spec.cmdline, stderr=subprocess.STDOUT, stdout=self._tempfile, - cwd=spec.cwd, - shell=spec.shell, + cwd=self._spec.cwd, + shell=self._spec.shell, env=env) self._state = _RUNNING - self._newline_on_success = newline_on_success - self._travis = travis - self._xml_test = ET.SubElement(xml_report, 'testcase', - name=self._spec.shortname) if xml_report is not None else None - message('START', spec.shortname, do_newline=self._travis) def state(self, update_cache): """Poll current state of the job. Prints messages at completion.""" @@ -202,15 +207,22 @@ class Job(object): self._xml_test.set('time', str(elapsed)) ET.SubElement(self._xml_test, 'system-out').text = filtered_stdout if self._process.returncode != 0: - self._state = _FAILURE - message('FAILED', '%s [ret=%d, pid=%d]' % ( + if self._retries < self._spec.flake_retries: + message('FLAKE', '%s [ret=%d, pid=%d]' % ( self._spec.shortname, self._process.returncode, self._process.pid), stdout, do_newline=True) - if self._xml_test is not None: - ET.SubElement(self._xml_test, 'failure', message='Failure').text + self._retries += 1 + self.start() + else: + self._state = _FAILURE + message('FAILED', '%s [ret=%d, pid=%d]' % ( + self._spec.shortname, self._process.returncode, self._process.pid), + stdout, do_newline=True) + if self._xml_test is not None: + ET.SubElement(self._xml_test, 'failure', message='Failure').text else: self._state = _SUCCESS - message('PASSED', '%s [time=%.1fsec]' % (self._spec.shortname, elapsed), + message('PASSED', '%s [time=%.1fsec; retries=%d]' % (self._spec.shortname, elapsed, self._retries), do_newline=self._newline_on_success or self._travis) if self._bin_hash: update_cache.finished(self._spec.identity(), self._bin_hash) From 3dedb70b38683a491ae3d150fdd206adabab8b4e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 24 Sep 2015 10:31:26 -0700 Subject: [PATCH 04/48] Implement turnstile polling for Windows --- src/core/iomgr/iocp_windows.c | 61 +++++++------- src/core/iomgr/iocp_windows.h | 1 + src/core/iomgr/iomgr.c | 2 + src/core/iomgr/iomgr_internal.h | 3 + src/core/iomgr/iomgr_posix.c | 2 - src/core/iomgr/iomgr_posix.h | 3 - src/core/iomgr/pollset_windows.c | 133 ++++++++++++++++++++++++------- src/core/iomgr/pollset_windows.h | 16 +++- 8 files changed, 153 insertions(+), 68 deletions(-) diff --git a/src/core/iomgr/iocp_windows.c b/src/core/iomgr/iocp_windows.c index c2f62a41b83..af846f680f7 100644 --- a/src/core/iomgr/iocp_windows.c +++ b/src/core/iomgr/iocp_windows.c @@ -50,13 +50,28 @@ static ULONG g_iocp_kick_token; static OVERLAPPED g_iocp_custom_overlap; -static gpr_event g_shutdown_iocp; -static gpr_event g_iocp_done; static gpr_atm g_custom_events = 0; static HANDLE g_iocp; -static void do_iocp_work(grpc_exec_ctx *exec_ctx) { +static DWORD deadline_to_millis_timeout(gpr_timespec deadline, + gpr_timespec now) { + gpr_timespec timeout; + static const int max_spin_polling_us = 10; + if (gpr_time_cmp(deadline, gpr_inf_future(deadline.clock_type)) == 0) { + return INFINITE; + } + if (gpr_time_cmp(deadline, gpr_time_add(now, gpr_time_from_micros( + max_spin_polling_us, + GPR_TIMESPAN))) <= 0) { + return 0; + } + timeout = gpr_time_sub(deadline, now); + return gpr_time_to_millis(gpr_time_add( + timeout, gpr_time_from_nanos(GPR_NS_PER_SEC - 1, GPR_TIMESPAN))); +} + +void grpc_iocp_work(grpc_exec_ctx *exec_ctx, gpr_timespec deadline) { BOOL success; DWORD bytes = 0; DWORD flags = 0; @@ -66,10 +81,10 @@ static void do_iocp_work(grpc_exec_ctx *exec_ctx) { grpc_winsocket_callback_info *info; grpc_closure *closure = NULL; success = GetQueuedCompletionStatus(g_iocp, &bytes, &completion_key, - &overlapped, INFINITE); - /* success = 0 and overlapped = NULL means the deadline got attained. - Which is impossible. since our wait time is +inf */ - GPR_ASSERT(success || overlapped); + &overlapped, deadline_to_millis_timeout(deadline, gpr_now(deadline.clock_type))); + if (success == 0 && overlapped == NULL) { + return; + } GPR_ASSERT(completion_key && overlapped); if (overlapped == &g_iocp_custom_overlap) { gpr_atm_full_fetch_add(&g_custom_events, -1); @@ -109,29 +124,10 @@ static void do_iocp_work(grpc_exec_ctx *exec_ctx) { } } -static void iocp_loop(void *p) { - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - - while (gpr_atm_acq_load(&g_custom_events) || - !gpr_event_get(&g_shutdown_iocp)) { - do_iocp_work(&exec_ctx); - grpc_exec_ctx_flush(&exec_ctx); - } - grpc_exec_ctx_finish(&exec_ctx); - - gpr_event_set(&g_iocp_done, (void *)1); -} - void grpc_iocp_init(void) { - gpr_thd_id id; - g_iocp = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, (ULONG_PTR)NULL, 0); GPR_ASSERT(g_iocp); - - gpr_event_init(&g_iocp_done); - gpr_event_init(&g_shutdown_iocp); - gpr_thd_new(&id, iocp_loop, NULL, NULL); } void grpc_iocp_kick(void) { @@ -144,12 +140,13 @@ void grpc_iocp_kick(void) { } void grpc_iocp_shutdown(void) { - BOOL success; - gpr_event_set(&g_shutdown_iocp, (void *)1); - grpc_iocp_kick(); - gpr_event_wait(&g_iocp_done, gpr_inf_future(GPR_CLOCK_REALTIME)); - success = CloseHandle(g_iocp); - GPR_ASSERT(success); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + while (gpr_atm_acq_load(&g_custom_events)) { + grpc_iocp_work(&exec_ctx, gpr_inf_future(GPR_CLOCK_MONOTONIC)); + grpc_exec_ctx_flush(&exec_ctx); + } + grpc_exec_ctx_finish(&exec_ctx); + GPR_ASSERT(CloseHandle(g_iocp)); } void grpc_iocp_add_socket(grpc_winsocket *socket) { diff --git a/src/core/iomgr/iocp_windows.h b/src/core/iomgr/iocp_windows.h index b0209e04e33..7e330e7ce2f 100644 --- a/src/core/iomgr/iocp_windows.h +++ b/src/core/iomgr/iocp_windows.h @@ -38,6 +38,7 @@ #include "src/core/iomgr/socket_windows.h" +void grpc_iocp_work(grpc_exec_ctx *exec_ctx, gpr_timespec deadline); void grpc_iocp_init(void); void grpc_iocp_kick(void); void grpc_iocp_shutdown(void); diff --git a/src/core/iomgr/iomgr.c b/src/core/iomgr/iomgr.c index ef222416af7..fe5c1d4e8fb 100644 --- a/src/core/iomgr/iomgr.c +++ b/src/core/iomgr/iomgr.c @@ -66,6 +66,7 @@ void grpc_iomgr_init(void) { g_root_object.next = g_root_object.prev = &g_root_object; g_root_object.name = "root"; grpc_iomgr_platform_init(); + grpc_pollset_global_init(); } static size_t count_objects(void) { @@ -138,6 +139,7 @@ void grpc_iomgr_shutdown(void) { gpr_mu_lock(&g_mu); gpr_mu_unlock(&g_mu); + grpc_pollset_global_shutdown(); grpc_iomgr_platform_shutdown(); gpr_mu_destroy(&g_mu); gpr_cv_destroy(&g_rcv); diff --git a/src/core/iomgr/iomgr_internal.h b/src/core/iomgr/iomgr_internal.h index f266732c96b..1a0724b431c 100644 --- a/src/core/iomgr/iomgr_internal.h +++ b/src/core/iomgr/iomgr_internal.h @@ -43,6 +43,9 @@ typedef struct grpc_iomgr_object { struct grpc_iomgr_object *prev; } grpc_iomgr_object; +void grpc_pollset_global_init(void); +void grpc_pollset_global_shutdown(void); + void grpc_iomgr_register_object(grpc_iomgr_object *obj, const char *name); void grpc_iomgr_unregister_object(grpc_iomgr_object *obj); diff --git a/src/core/iomgr/iomgr_posix.c b/src/core/iomgr/iomgr_posix.c index 2425e599415..db93d0a7561 100644 --- a/src/core/iomgr/iomgr_posix.c +++ b/src/core/iomgr/iomgr_posix.c @@ -42,12 +42,10 @@ void grpc_iomgr_platform_init(void) { grpc_fd_global_init(); - grpc_pollset_global_init(); grpc_register_tracer("tcp", &grpc_tcp_trace); } void grpc_iomgr_platform_shutdown(void) { - grpc_pollset_global_shutdown(); grpc_fd_global_shutdown(); } diff --git a/src/core/iomgr/iomgr_posix.h b/src/core/iomgr/iomgr_posix.h index 716fedb6368..068a5c6d7cf 100644 --- a/src/core/iomgr/iomgr_posix.h +++ b/src/core/iomgr/iomgr_posix.h @@ -36,7 +36,4 @@ #include "src/core/iomgr/iomgr_internal.h" -void grpc_pollset_global_init(void); -void grpc_pollset_global_shutdown(void); - #endif /* GRPC_INTERNAL_CORE_IOMGR_IOMGR_POSIX_H */ diff --git a/src/core/iomgr/pollset_windows.c b/src/core/iomgr/pollset_windows.c index 6182eb35328..cb0aad0e33c 100644 --- a/src/core/iomgr/pollset_windows.c +++ b/src/core/iomgr/pollset_windows.c @@ -39,38 +39,66 @@ #include "src/core/iomgr/alarm_internal.h" #include "src/core/iomgr/iomgr_internal.h" +#include "src/core/iomgr/iocp_windows.h" #include "src/core/iomgr/pollset.h" #include "src/core/iomgr/pollset_windows.h" -static void remove_worker(grpc_pollset *p, grpc_pollset_worker *worker) { - worker->prev->next = worker->next; - worker->next->prev = worker->prev; +static gpr_mu g_polling_mu; +static grpc_pollset_worker *g_active_poller; +static grpc_pollset_worker g_global_root_worker; + +void grpc_pollset_global_init() { + gpr_mu_init(&g_polling_mu); + g_active_poller = NULL; + g_global_root_worker.links[GRPC_POLLSET_WORKER_LINK_GLOBAL].next = + g_global_root_worker.links[GRPC_POLLSET_WORKER_LINK_GLOBAL].prev = + &g_global_root_worker; +} + +void grpc_pollset_global_shutdown() { + gpr_mu_destroy(&g_polling_mu); } -static int has_workers(grpc_pollset *p) { - return p->root_worker.next != &p->root_worker; +static void remove_worker(grpc_pollset_worker *worker, + grpc_pollset_worker_link_type type) { + worker->links[type].prev->links[type].next = worker->links[type].next; + worker->links[type].next->links[type].prev = worker->links[type].prev; + worker->links[type].next = worker->links[type].prev = worker; } -static grpc_pollset_worker *pop_front_worker(grpc_pollset *p) { - if (has_workers(p)) { - grpc_pollset_worker *w = p->root_worker.next; - remove_worker(p, w); +static int has_workers(grpc_pollset_worker *root, grpc_pollset_worker_link_type type) { + return root->links[type].next != root; +} + +static grpc_pollset_worker *pop_front_worker( + grpc_pollset_worker *root, grpc_pollset_worker_link_type type) { + if (has_workers(root, type)) { + grpc_pollset_worker *w = root->links[type].next; + remove_worker(w, type); return w; } else { return NULL; } } -static void push_back_worker(grpc_pollset *p, grpc_pollset_worker *worker) { - worker->next = &p->root_worker; - worker->prev = worker->next->prev; - worker->prev->next = worker->next->prev = worker; +static void push_back_worker(grpc_pollset_worker *root, + grpc_pollset_worker_link_type type, + grpc_pollset_worker *worker) { + worker->links[type].next = root; + worker->links[type].prev = worker->links[type].next->links[type].prev; + worker->links[type].prev->links[type].next = + worker->links[type].next->links[type].prev = + worker; } -static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { - worker->prev = &p->root_worker; - worker->next = worker->prev->next; - worker->prev->next = worker->next->prev = worker; +static void push_front_worker(grpc_pollset_worker *root, + grpc_pollset_worker_link_type type, + grpc_pollset_worker *worker) { + worker->links[type].prev = root; + worker->links[type].next = worker->links[type].prev->links[type].next; + worker->links[type].prev->links[type].next = + worker->links[type].next->links[type].prev = + worker; } /* There isn't really any such thing as a pollset under Windows, due to the @@ -81,8 +109,9 @@ static void push_front_worker(grpc_pollset *p, grpc_pollset_worker *worker) { void grpc_pollset_init(grpc_pollset *pollset) { memset(pollset, 0, sizeof(*pollset)); gpr_mu_init(&pollset->mu); - pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker; - pollset->kicked_without_pollers = 0; + pollset->root_worker.links[GRPC_POLLSET_WORKER_LINK_POLLSET].next = + pollset->root_worker.links[GRPC_POLLSET_WORKER_LINK_POLLSET].prev = + &pollset->root_worker; } void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, @@ -90,8 +119,12 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, gpr_mu_lock(&pollset->mu); pollset->shutting_down = 1; grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); + if (!pollset->is_iocp_worker) { + grpc_exec_ctx_enqueue(exec_ctx, closure, 1); + } else { + pollset->on_shutdown = closure; + } gpr_mu_unlock(&pollset->mu); - grpc_exec_ctx_enqueue(exec_ctx, closure, 1); } void grpc_pollset_destroy(grpc_pollset *pollset) { @@ -102,13 +135,42 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_pollset_worker *worker, gpr_timespec now, gpr_timespec deadline) { int added_worker = 0; - worker->next = worker->prev = NULL; + worker->links[GRPC_POLLSET_WORKER_LINK_POLLSET].next = + worker->links[GRPC_POLLSET_WORKER_LINK_POLLSET].prev = + NULL; gpr_cv_init(&worker->cv); if (grpc_alarm_check(exec_ctx, now, &deadline)) { goto done; } if (!pollset->kicked_without_pollers && !pollset->shutting_down) { - push_front_worker(pollset, worker); + gpr_mu_lock(&g_polling_mu); + if (g_active_poller == NULL) { + grpc_pollset_worker *next_worker; + /* become poller */ + pollset->is_iocp_worker = 1; + g_active_poller = worker; + gpr_mu_unlock(&g_polling_mu); + gpr_mu_unlock(&pollset->mu); + grpc_iocp_work(exec_ctx, deadline); + gpr_mu_lock(&pollset->mu); + gpr_mu_lock(&g_polling_mu); + pollset->is_iocp_worker = 0; + g_active_poller = NULL; + next_worker = pop_front_worker(&g_global_root_worker, GRPC_POLLSET_WORKER_LINK_GLOBAL); + if (next_worker != NULL) { + gpr_cv_signal(&next_worker->cv); + } + gpr_mu_unlock(&g_polling_mu); + + if (pollset->shutting_down && pollset->on_shutdown != NULL) { + grpc_exec_ctx_enqueue(exec_ctx, pollset->on_shutdown, 1); + pollset->on_shutdown = NULL; + } + goto done; + } + push_front_worker(&g_global_root_worker, GRPC_POLLSET_WORKER_LINK_GLOBAL, worker); + gpr_mu_unlock(&g_polling_mu); + push_front_worker(&pollset->root_worker, GRPC_POLLSET_WORKER_LINK_POLLSET, worker); added_worker = 1; gpr_cv_wait(&worker->cv, &pollset->mu, deadline); } else { @@ -122,27 +184,40 @@ done: } gpr_cv_destroy(&worker->cv); if (added_worker) { - remove_worker(pollset, worker); + remove_worker(worker, GRPC_POLLSET_WORKER_LINK_GLOBAL); + remove_worker(worker, GRPC_POLLSET_WORKER_LINK_POLLSET); } } void grpc_pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) { if (specific_worker != NULL) { if (specific_worker == GRPC_POLLSET_KICK_BROADCAST) { - for (specific_worker = p->root_worker.next; + for (specific_worker = p->root_worker.links[GRPC_POLLSET_WORKER_LINK_POLLSET].next; specific_worker != &p->root_worker; - specific_worker = specific_worker->next) { + specific_worker = specific_worker->links[GRPC_POLLSET_WORKER_LINK_POLLSET].next) { gpr_cv_signal(&specific_worker->cv); } p->kicked_without_pollers = 1; + if (p->is_iocp_worker) { + grpc_iocp_kick(); + } } else { - gpr_cv_signal(&specific_worker->cv); + if (p->is_iocp_worker) { + gpr_mu_lock(&g_polling_mu); + if (g_active_poller == specific_worker) { + grpc_iocp_kick(); + } + gpr_mu_unlock(&g_polling_mu); + } else { + gpr_cv_signal(&specific_worker->cv); + } } } else { - specific_worker = pop_front_worker(p); + specific_worker = pop_front_worker(&p->root_worker, GRPC_POLLSET_WORKER_LINK_POLLSET); if (specific_worker != NULL) { - push_back_worker(p, specific_worker); - gpr_cv_signal(&specific_worker->cv); + grpc_pollset_kick(p, specific_worker); + } else if (p->is_iocp_worker) { + grpc_iocp_kick(); } else { p->kicked_without_pollers = 1; } diff --git a/src/core/iomgr/pollset_windows.h b/src/core/iomgr/pollset_windows.h index 4efa5a1717c..55f87aca729 100644 --- a/src/core/iomgr/pollset_windows.h +++ b/src/core/iomgr/pollset_windows.h @@ -43,17 +43,29 @@ used to synchronize with the IOCP, and workers are condition variables used to block threads until work is ready. */ -typedef struct grpc_pollset_worker { - gpr_cv cv; +typedef enum { + GRPC_POLLSET_WORKER_LINK_POLLSET = 0, + GRPC_POLLSET_WORKER_LINK_GLOBAL, + GRPC_POLLSET_WORKER_LINK_TYPES +} grpc_pollset_worker_link_type; + +typedef struct grpc_pollset_worker_link { struct grpc_pollset_worker *next; struct grpc_pollset_worker *prev; +} grpc_pollset_worker_link; + +typedef struct grpc_pollset_worker { + gpr_cv cv; + grpc_pollset_worker_link links[GRPC_POLLSET_WORKER_LINK_TYPES]; } grpc_pollset_worker; typedef struct grpc_pollset { gpr_mu mu; int shutting_down; int kicked_without_pollers; + int is_iocp_worker; grpc_pollset_worker root_worker; + grpc_closure *on_shutdown; } grpc_pollset; #define GRPC_POLLSET_MU(pollset) (&(pollset)->mu) From ddf3a511aaf519f699c3cd0fdee4f5275ecf600a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 24 Sep 2015 13:03:44 -0700 Subject: [PATCH 05/48] Add clang-format to sanity --- include/grpc++/impl/call.h | 3 +- .../grpc++/security/auth_metadata_processor.h | 1 - include/grpc++/server.h | 7 ++--- include/grpc/census.h | 4 +-- include/grpc/compression.h | 2 +- include/grpc/grpc.h | 4 +-- include/grpc/grpc_security.h | 4 +-- src/core/channel/channel_args.h | 4 +-- .../client_config/lb_policies/round_robin.h | 1 - src/core/client_config/subchannel.h | 16 +++++----- src/core/security/credentials.h | 1 - src/core/security/handshake.h | 1 - src/core/security/security_connector.h | 8 ++--- src/core/support/time_precise.h | 2 +- src/cpp/server/secure_server_credentials.h | 2 +- test/cpp/interop/client_helper.h | 2 +- tools/distrib/clang_format_code.sh | 13 ++++++++ tools/dockerfile/grpc_clang_format/Dockerfile | 6 ++++ .../clang_format_all_the_things.sh | 30 +++++++++++++++++++ tools/run_tests/run_sanity.sh | 2 ++ 20 files changed, 78 insertions(+), 35 deletions(-) create mode 100755 tools/distrib/clang_format_code.sh create mode 100644 tools/dockerfile/grpc_clang_format/Dockerfile create mode 100755 tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh diff --git a/include/grpc++/impl/call.h b/include/grpc++/impl/call.h index fca56030479..4602c9cdb05 100644 --- a/include/grpc++/impl/call.h +++ b/include/grpc++/impl/call.h @@ -248,8 +248,7 @@ class CallOpRecvMessage { if (*status) { got_message = true; *status = SerializationTraits::Deserialize(recv_buf_, message_, - max_message_size) - .ok(); + max_message_size).ok(); } else { got_message = false; grpc_byte_buffer_destroy(recv_buf_); diff --git a/include/grpc++/security/auth_metadata_processor.h b/include/grpc++/security/auth_metadata_processor.h index 9b9c06e3b63..25011f33baf 100644 --- a/include/grpc++/security/auth_metadata_processor.h +++ b/include/grpc++/security/auth_metadata_processor.h @@ -71,4 +71,3 @@ class AuthMetadataProcessor { } // namespace grpc #endif // GRPCXX_AUTH_METADATA_PROCESSOR_H_ - diff --git a/include/grpc++/server.h b/include/grpc++/server.h index 210fe020ad1..1a62df5698f 100644 --- a/include/grpc++/server.h +++ b/include/grpc++/server.h @@ -215,11 +215,10 @@ class Server GRPC_FINAL : public GrpcLibrary, private CallHook { bool FinalizeResult(void** tag, bool* status) GRPC_OVERRIDE { bool serialization_status = *status && payload_ && - SerializationTraits::Deserialize(payload_, request_, - server_->max_message_size_) - .ok(); + SerializationTraits::Deserialize( + payload_, request_, server_->max_message_size_).ok(); bool ret = RegisteredAsyncRequest::FinalizeResult(tag, status); - *status = serialization_status && *status; + *status = serialization_status&&* status; return ret; } diff --git a/include/grpc/census.h b/include/grpc/census.h index 2f36665d465..d0bc90420c0 100644 --- a/include/grpc/census.h +++ b/include/grpc/census.h @@ -424,8 +424,8 @@ extern census_aggregation_ops census_agg_window; construction via census_define_view(). */ typedef struct { const census_aggregation_ops *ops; - const void - *create_arg; /* Argument to be used for aggregation initialization. */ + const void * + create_arg; /* Argument to be used for aggregation initialization. */ } census_aggregation; /** A census view type. Opaque. */ diff --git a/include/grpc/compression.h b/include/grpc/compression.h index 82e326fe0ec..30163be2ff4 100644 --- a/include/grpc/compression.h +++ b/include/grpc/compression.h @@ -95,7 +95,7 @@ void grpc_compression_options_init(grpc_compression_options *opts); /** Mark \a algorithm as enabled in \a opts. */ void grpc_compression_options_enable_algorithm( - grpc_compression_options *opts, grpc_compression_algorithm algorithm); + grpc_compression_options *opts, grpc_compression_algorithm algorithm); /** Mark \a algorithm as disabled in \a opts. */ void grpc_compression_options_disable_algorithm( diff --git a/include/grpc/grpc.h b/include/grpc/grpc.h index 47f3df6605a..0c854a5b0cd 100644 --- a/include/grpc/grpc.h +++ b/include/grpc/grpc.h @@ -595,8 +595,8 @@ grpc_call_error grpc_call_cancel_with_status(grpc_call *call, void grpc_call_destroy(grpc_call *call); /** Request notification of a new call. - Once a call is received, a notification tagged with \a tag_new is added to - \a cq_for_notification. \a call, \a details and \a request_metadata are + Once a call is received, a notification tagged with \a tag_new is added to + \a cq_for_notification. \a call, \a details and \a request_metadata are updated with the appropriate call information. \a cq_bound_to_call is bound to \a call, and batch operation notifications for that call will be posted to \a cq_bound_to_call. diff --git a/include/grpc/grpc_security.h b/include/grpc/grpc_security.h index 44ced4fb434..7a442e2ace2 100644 --- a/include/grpc/grpc_security.h +++ b/include/grpc/grpc_security.h @@ -123,8 +123,8 @@ grpc_credentials *grpc_google_refresh_token_credentials_create( /* Creates an Oauth2 Access Token credentials with an access token that was aquired by an out of band mechanism. */ -grpc_credentials *grpc_access_token_credentials_create( - const char *access_token, void *reserved); +grpc_credentials *grpc_access_token_credentials_create(const char *access_token, + void *reserved); /* Creates an IAM credentials object for connecting to Google. */ grpc_credentials *grpc_google_iam_credentials_create( diff --git a/src/core/channel/channel_args.h b/src/core/channel/channel_args.h index 1a6be91359e..480cc9aec2f 100644 --- a/src/core/channel/channel_args.h +++ b/src/core/channel/channel_args.h @@ -75,9 +75,7 @@ grpc_channel_args *grpc_channel_args_set_compression_algorithm( * modified to point to the returned instance (which may be different from the * input value of \a a). */ grpc_channel_args *grpc_channel_args_compression_algorithm_set_state( - grpc_channel_args **a, - grpc_compression_algorithm algorithm, - int enabled); + grpc_channel_args **a, grpc_compression_algorithm algorithm, int enabled); /** Returns the bitset representing the support state (true for enabled, false * for disabled) for compression algorithms. diff --git a/src/core/client_config/lb_policies/round_robin.h b/src/core/client_config/lb_policies/round_robin.h index 2c81b9ef179..cf1f69c85f6 100644 --- a/src/core/client_config/lb_policies/round_robin.h +++ b/src/core/client_config/lb_policies/round_robin.h @@ -43,5 +43,4 @@ extern int grpc_lb_round_robin_trace; /** Returns a load balancing factory for the round robin policy */ grpc_lb_policy_factory *grpc_round_robin_lb_factory_create(); - #endif diff --git a/src/core/client_config/subchannel.h b/src/core/client_config/subchannel.h index 2e36c69134a..a2695999289 100644 --- a/src/core/client_config/subchannel.h +++ b/src/core/client_config/subchannel.h @@ -62,14 +62,14 @@ typedef struct grpc_subchannel_args grpc_subchannel_args; #define GRPC_SUBCHANNEL_REF_EXTRA_ARGS #endif -void grpc_subchannel_ref( - grpc_subchannel *channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS); -void grpc_subchannel_unref( - grpc_subchannel *channel GRPC_SUBCHANNEL_REF_EXTRA_ARGS); -void grpc_subchannel_call_ref( - grpc_subchannel_call *call GRPC_SUBCHANNEL_REF_EXTRA_ARGS); -void grpc_subchannel_call_unref( - grpc_subchannel_call *call GRPC_SUBCHANNEL_REF_EXTRA_ARGS); +void grpc_subchannel_ref(grpc_subchannel *channel + GRPC_SUBCHANNEL_REF_EXTRA_ARGS); +void grpc_subchannel_unref(grpc_subchannel *channel + GRPC_SUBCHANNEL_REF_EXTRA_ARGS); +void grpc_subchannel_call_ref(grpc_subchannel_call *call + GRPC_SUBCHANNEL_REF_EXTRA_ARGS); +void grpc_subchannel_call_unref(grpc_subchannel_call *call + GRPC_SUBCHANNEL_REF_EXTRA_ARGS); /** construct a call (possibly asynchronously) */ void grpc_subchannel_create_call(grpc_subchannel *subchannel, diff --git a/src/core/security/credentials.h b/src/core/security/credentials.h index 38ce0f8ba6e..6c6be56d15f 100644 --- a/src/core/security/credentials.h +++ b/src/core/security/credentials.h @@ -216,7 +216,6 @@ typedef struct { grpc_server_credentials *c, grpc_security_connector **sc); } grpc_server_credentials_vtable; - /* TODO(jboeuf): Add a refcount. */ struct grpc_server_credentials { const grpc_server_credentials_vtable *vtable; diff --git a/src/core/security/handshake.h b/src/core/security/handshake.h index d7e4a305800..7a7749c8b43 100644 --- a/src/core/security/handshake.h +++ b/src/core/security/handshake.h @@ -37,7 +37,6 @@ #include "src/core/iomgr/endpoint.h" #include "src/core/security/security_connector.h" - /* Calls the callback upon completion. Takes owership of handshaker. */ void grpc_do_security_handshake(tsi_handshaker *handshaker, grpc_security_connector *connector, diff --git a/src/core/security/security_connector.h b/src/core/security/security_connector.h index 5fc1db382e5..5eea77ad4b5 100644 --- a/src/core/security/security_connector.h +++ b/src/core/security/security_connector.h @@ -63,11 +63,11 @@ typedef struct grpc_security_connector grpc_security_connector; typedef void (*grpc_security_check_cb)(void *user_data, grpc_security_status status); - /* Ownership of the secure_endpoint is transfered. */ -typedef void (*grpc_security_handshake_done_cb)( - void *user_data, grpc_security_status status, - grpc_endpoint *wrapped_endpoint, grpc_endpoint *secure_endpoint); +typedef void (*grpc_security_handshake_done_cb)(void *user_data, + grpc_security_status status, + grpc_endpoint *wrapped_endpoint, + grpc_endpoint *secure_endpoint); typedef struct { void (*destroy)(grpc_security_connector *sc); diff --git a/src/core/support/time_precise.h b/src/core/support/time_precise.h index 574ebb84489..cd201faab95 100644 --- a/src/core/support/time_precise.h +++ b/src/core/support/time_precise.h @@ -83,7 +83,7 @@ static void gpr_precise_clock_now(gpr_timespec *clk) { clk->tv_nsec = counter % cycles_per_second; } -#else /* GRPC_TIMERS_RDTSC */ +#else /* GRPC_TIMERS_RDTSC */ static void gpr_precise_clock_now(gpr_timespec *clk) { *clk = gpr_now(GPR_CLOCK_REALTIME); clk->clock_type = GPR_CLOCK_PRECISE; diff --git a/src/cpp/server/secure_server_credentials.h b/src/cpp/server/secure_server_credentials.h index 4f003c6b7e5..5460f4a02c3 100644 --- a/src/cpp/server/secure_server_credentials.h +++ b/src/cpp/server/secure_server_credentials.h @@ -46,7 +46,7 @@ namespace grpc { class AuthMetadataProcessorAyncWrapper GRPC_FINAL { public: - static void Destroy(void *wrapper); + static void Destroy(void* wrapper); static void Process(void* wrapper, grpc_auth_context* context, const grpc_metadata* md, size_t num_md, diff --git a/test/cpp/interop/client_helper.h b/test/cpp/interop/client_helper.h index 92d5078f48b..0221df93db5 100644 --- a/test/cpp/interop/client_helper.h +++ b/test/cpp/interop/client_helper.h @@ -53,7 +53,7 @@ std::shared_ptr CreateChannelForTestCase( class InteropClientContextInspector { public: InteropClientContextInspector(const ::grpc::ClientContext& context) - : context_(context) {} + : context_(context) {} // Inspector methods, able to peek inside ClientContext, follow. grpc_compression_algorithm GetCallCompressionAlgorithm() const { diff --git a/tools/distrib/clang_format_code.sh b/tools/distrib/clang_format_code.sh new file mode 100755 index 00000000000..55f4c52ec27 --- /dev/null +++ b/tools/distrib/clang_format_code.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -ex + +# change to root directory +cd $(dirname $0)/../.. + +# build clang-format docker image +docker build -t grpc_clang_format tools/dockerfile/grpc_clang_format + +# run clang-format against the checked out codebase +docker run -e TEST=$TEST --rm=true -v `pwd`:/local-code -t grpc_clang_format /clang_format_all_the_things.sh + diff --git a/tools/dockerfile/grpc_clang_format/Dockerfile b/tools/dockerfile/grpc_clang_format/Dockerfile new file mode 100644 index 00000000000..a0fff2f2b52 --- /dev/null +++ b/tools/dockerfile/grpc_clang_format/Dockerfile @@ -0,0 +1,6 @@ +FROM ubuntu:vivid +RUN apt-get update +RUN apt-get -y install clang-format-3.6 +ADD clang_format_all_the_things.sh / +CMD ["echo 'Run with tools/distrib/clang_format_code.sh'"] + diff --git a/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh new file mode 100755 index 00000000000..5aa265800de --- /dev/null +++ b/tools/dockerfile/grpc_clang_format/clang_format_all_the_things.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# directories to run against +DIRS="src/core src/cpp test/core test/cpp include" + +# file matching patterns to check +GLOB="*.h *.cpp" + +# clang format command +CLANG_FORMAT=clang-format-3.6 + +files= +for dir in $DIRS +do + for glob in $GLOB + do + files="$files `find /local-code/$dir -name $glob`" + done +done + +if [ "x$TEST" = "x" ] +then + echo $files | xargs $CLANG_FORMAT -i +else + for file in $files + do + $CLANG_FORMAT $file | diff $file - + done +fi + diff --git a/tools/run_tests/run_sanity.sh b/tools/run_tests/run_sanity.sh index 4b367dcbc77..1e8fe5c1a7e 100755 --- a/tools/run_tests/run_sanity.sh +++ b/tools/run_tests/run_sanity.sh @@ -58,3 +58,5 @@ if [ -f cache.mk ] ; then fi ./tools/buildgen/generate_projects.sh +./tools/distrib/clang_format_code.sh + From d383b0f7e79220966e65b55342ec37f3fbba0be6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 24 Sep 2015 13:22:51 -0700 Subject: [PATCH 06/48] Integrate docker-based clang_format with Jenkins --- tools/jenkins/build_docker_and_run_tests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/jenkins/build_docker_and_run_tests.sh b/tools/jenkins/build_docker_and_run_tests.sh index fa6bd44e180..9f97f39e75d 100755 --- a/tools/jenkins/build_docker_and_run_tests.sh +++ b/tools/jenkins/build_docker_and_run_tests.sh @@ -60,6 +60,8 @@ docker run \ -i $TTY_FLAG \ -v "$git_root:/var/local/jenkins/grpc" \ -v /tmp/ccache:/tmp/ccache \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v $(which docker):/bin/docker \ -w /var/local/git/grpc \ --cidfile=docker.cid \ $DOCKER_IMAGE_NAME \ From cc5dc5bbafd4ef3f54b655bf98afcc3b71435ef2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 24 Sep 2015 17:02:22 -0700 Subject: [PATCH 07/48] Fast path for proto serialization for small protos --- src/cpp/proto/proto_utils.cc | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/cpp/proto/proto_utils.cc b/src/cpp/proto/proto_utils.cc index f47acc8f8dc..3c0be589194 100644 --- a/src/cpp/proto/proto_utils.cc +++ b/src/cpp/proto/proto_utils.cc @@ -158,10 +158,19 @@ namespace grpc { Status SerializeProto(const grpc::protobuf::Message& msg, grpc_byte_buffer** bp) { - GrpcBufferWriter writer(bp); - return msg.SerializeToZeroCopyStream(&writer) - ? Status::OK - : Status(StatusCode::INTERNAL, "Failed to serialize message"); + int byte_size = msg.ByteSize(); + if (byte_size <= kMaxBufferLength) { + gpr_slice slice = gpr_slice_malloc(byte_size); + GPR_ASSERT(GPR_SLICE_END_PTR(slice) == msg.SerializeWithCachedSizesToArray(GPR_SLICE_START_PTR(slice))); + *bp = grpc_raw_byte_buffer_create(&slice, 1); + gpr_slice_unref(slice); + return Status::OK; + } else { + GrpcBufferWriter writer(bp); + return msg.SerializeToZeroCopyStream(&writer) + ? Status::OK + : Status(StatusCode::INTERNAL, "Failed to serialize message"); + } } Status DeserializeProto(grpc_byte_buffer* buffer, grpc::protobuf::Message* msg, From 649deebf7f50d29ed3550ce9dcd241cbeacb7df6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 24 Sep 2015 23:19:40 -0700 Subject: [PATCH 08/48] Recycle partially filled buffers on the next read --- include/grpc/support/slice_buffer.h | 2 +- src/core/iomgr/tcp_posix.c | 9 ++++++++- src/core/support/slice_buffer.c | 9 +++++---- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/include/grpc/support/slice_buffer.h b/include/grpc/support/slice_buffer.h index 321ba288fd9..f3f14a6aa0b 100644 --- a/include/grpc/support/slice_buffer.h +++ b/include/grpc/support/slice_buffer.h @@ -87,7 +87,7 @@ void gpr_slice_buffer_swap(gpr_slice_buffer *a, gpr_slice_buffer *b); /* move all of the elements of src into dst */ void gpr_slice_buffer_move_into(gpr_slice_buffer *src, gpr_slice_buffer *dst); /* remove n bytes from the end of a slice buffer */ -void gpr_slice_buffer_trim_end(gpr_slice_buffer *src, size_t n); +void gpr_slice_buffer_trim_end(gpr_slice_buffer *src, size_t n, gpr_slice_buffer *garbage); #ifdef __cplusplus } diff --git a/src/core/iomgr/tcp_posix.c b/src/core/iomgr/tcp_posix.c index c98f0125f8a..54ebad7dbc7 100644 --- a/src/core/iomgr/tcp_posix.c +++ b/src/core/iomgr/tcp_posix.c @@ -78,6 +78,9 @@ typedef struct { size_t slice_size; gpr_refcount refcount; + /* garbage after the last read */ + gpr_slice_buffer last_read_buffer; + gpr_slice_buffer *incoming_buffer; gpr_slice_buffer *outgoing_buffer; /** slice within outgoing_buffer to write next */ @@ -106,6 +109,7 @@ static void tcp_shutdown(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep) { static void tcp_free(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { grpc_fd_orphan(exec_ctx, tcp->em_fd, NULL, "tcp_unref_orphan"); + gpr_slice_buffer_destroy(&tcp->last_read_buffer); gpr_free(tcp->peer_string); gpr_free(tcp); } @@ -226,7 +230,8 @@ static void tcp_continue_read(grpc_exec_ctx *exec_ctx, grpc_tcp *tcp) { if ((size_t)read_bytes < tcp->incoming_buffer->length) { gpr_slice_buffer_trim_end( tcp->incoming_buffer, - tcp->incoming_buffer->length - (size_t)read_bytes); + tcp->incoming_buffer->length - (size_t)read_bytes, + &tcp->last_read_buffer); } else if (tcp->iov_size < MAX_READ_IOVEC) { ++tcp->iov_size; } @@ -259,6 +264,7 @@ static void tcp_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, tcp->read_cb = cb; tcp->incoming_buffer = incoming_buffer; gpr_slice_buffer_reset_and_unref(incoming_buffer); + gpr_slice_buffer_swap(incoming_buffer, &tcp->last_read_buffer); TCP_REF(tcp, "read"); if (tcp->finished_edge) { tcp->finished_edge = 0; @@ -457,6 +463,7 @@ grpc_endpoint *grpc_tcp_create(grpc_fd *em_fd, size_t slice_size, tcp->read_closure.cb_arg = tcp; tcp->write_closure.cb = tcp_handle_write; tcp->write_closure.cb_arg = tcp; + gpr_slice_buffer_init(&tcp->last_read_buffer); return &tcp->base; } diff --git a/src/core/support/slice_buffer.c b/src/core/support/slice_buffer.c index 8873d459d56..a1aa56fd722 100644 --- a/src/core/support/slice_buffer.c +++ b/src/core/support/slice_buffer.c @@ -208,7 +208,7 @@ void gpr_slice_buffer_move_into(gpr_slice_buffer *src, gpr_slice_buffer *dst) { src->length = 0; } -void gpr_slice_buffer_trim_end(gpr_slice_buffer *sb, size_t n) { +void gpr_slice_buffer_trim_end(gpr_slice_buffer *sb, size_t n, gpr_slice_buffer *garbage) { GPR_ASSERT(n <= sb->length); sb->length -= n; for (;;) { @@ -216,14 +216,15 @@ void gpr_slice_buffer_trim_end(gpr_slice_buffer *sb, size_t n) { gpr_slice slice = sb->slices[idx]; size_t slice_len = GPR_SLICE_LENGTH(slice); if (slice_len > n) { - sb->slices[idx] = gpr_slice_sub_no_ref(slice, 0, slice_len - n); + sb->slices[idx] = gpr_slice_split_head(&slice, slice_len - n); + gpr_slice_buffer_add_indexed(garbage, slice); return; } else if (slice_len == n) { - gpr_slice_unref(slice); + gpr_slice_buffer_add_indexed(garbage, slice); sb->count = idx; return; } else { - gpr_slice_unref(slice); + gpr_slice_buffer_add_indexed(garbage, slice); n -= slice_len; sb->count = idx; } From 0799ff3684d55f002cf3d8d5f5213181e923e631 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 24 Sep 2015 23:39:33 -0700 Subject: [PATCH 09/48] Fix a memory corruption bug --- src/core/transport/chttp2/incoming_metadata.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/transport/chttp2/incoming_metadata.c b/src/core/transport/chttp2/incoming_metadata.c index d216c421139..10c64f33565 100644 --- a/src/core/transport/chttp2/incoming_metadata.c +++ b/src/core/transport/chttp2/incoming_metadata.c @@ -171,7 +171,7 @@ void grpc_chttp2_incoming_metadata_buffer_postprocess_sopb_and_begin_live_op( size_t copy_bytes = sizeof(*buffer->elems) * new_count; GPR_ASSERT(mdidx < buffer->count); buffer->elems = gpr_malloc(copy_bytes); - memcpy(live_op_buffer->elems + mdidx, buffer->elems, copy_bytes); + memcpy(buffer->elems, live_op_buffer->elems + mdidx, copy_bytes); buffer->count = buffer->capacity = new_count; } else { buffer->elems = NULL; From 25834347e216173b3db24405e794ac658833a74b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 25 Sep 2015 08:08:24 -0700 Subject: [PATCH 10/48] Enforce a more mergable build.yaml format --- build.yaml | 2215 +++++++++++++++++++++++-------- tools/buildgen/build-cleaner.py | 2 +- 2 files changed, 1684 insertions(+), 533 deletions(-) diff --git a/build.yaml b/build.yaml index 4c6c6dc100e..f86a1add9f5 100644 --- a/build.yaml +++ b/build.yaml @@ -3,879 +3,1823 @@ '#3': Please refer to the templates directory for more information. settings: '#': The public version number of the library. - version: {major: 0, minor: 11, micro: 0, build: 0} + version: + major: 0 + minor: 11 + micro: 0 + build: 0 filegroups: - name: census - public_headers: [include/grpc/census.h] - headers: [src/core/census/aggregation.h, src/core/census/context.h, src/core/census/rpc_metric_id.h] - src: [src/core/census/context.c, src/core/census/initialize.c, src/core/census/operation.c, - src/core/census/tracing.c] + public_headers: + - include/grpc/census.h + headers: + - src/core/census/aggregation.h + - src/core/census/context.h + - src/core/census/rpc_metric_id.h + src: + - src/core/census/context.c + - src/core/census/initialize.c + - src/core/census/operation.c + - src/core/census/tracing.c - name: grpc++_base - public_headers: [include/grpc++/channel.h, include/grpc++/client_context.h, include/grpc++/completion_queue.h, - include/grpc++/create_channel.h, include/grpc++/generic/async_generic_service.h, - include/grpc++/generic/generic_stub.h, include/grpc++/grpc++.h, include/grpc++/impl/call.h, - include/grpc++/impl/client_unary_call.h, include/grpc++/impl/grpc_library.h, include/grpc++/impl/proto_utils.h, - include/grpc++/impl/rpc_method.h, include/grpc++/impl/rpc_service_method.h, include/grpc++/impl/serialization_traits.h, - include/grpc++/impl/service_type.h, include/grpc++/impl/sync.h, include/grpc++/impl/sync_cxx11.h, - include/grpc++/impl/sync_no_cxx11.h, include/grpc++/impl/thd.h, include/grpc++/impl/thd_cxx11.h, - include/grpc++/impl/thd_no_cxx11.h, include/grpc++/security/auth_context.h, include/grpc++/security/auth_metadata_processor.h, - include/grpc++/security/credentials.h, include/grpc++/security/server_credentials.h, - include/grpc++/server.h, include/grpc++/server_builder.h, include/grpc++/server_context.h, - include/grpc++/support/async_stream.h, include/grpc++/support/async_unary_call.h, - include/grpc++/support/byte_buffer.h, include/grpc++/support/channel_arguments.h, - include/grpc++/support/config.h, include/grpc++/support/config_protobuf.h, include/grpc++/support/slice.h, - include/grpc++/support/status.h, include/grpc++/support/status_code_enum.h, include/grpc++/support/string_ref.h, - include/grpc++/support/stub_options.h, include/grpc++/support/sync_stream.h, include/grpc++/support/time.h] - headers: [src/cpp/client/create_channel_internal.h, src/cpp/common/create_auth_context.h, - src/cpp/server/dynamic_thread_pool.h, src/cpp/server/fixed_size_thread_pool.h, - src/cpp/server/thread_pool_interface.h] - src: [src/cpp/client/channel.cc, src/cpp/client/channel_arguments.cc, src/cpp/client/client_context.cc, - src/cpp/client/create_channel.cc, src/cpp/client/create_channel_internal.cc, src/cpp/client/credentials.cc, - src/cpp/client/generic_stub.cc, src/cpp/client/insecure_credentials.cc, src/cpp/common/call.cc, - src/cpp/common/completion_queue.cc, src/cpp/common/rpc_method.cc, src/cpp/proto/proto_utils.cc, - src/cpp/server/async_generic_service.cc, src/cpp/server/create_default_thread_pool.cc, - src/cpp/server/dynamic_thread_pool.cc, src/cpp/server/fixed_size_thread_pool.cc, - src/cpp/server/insecure_server_credentials.cc, src/cpp/server/server.cc, src/cpp/server/server_builder.cc, - src/cpp/server/server_context.cc, src/cpp/server/server_credentials.cc, src/cpp/util/byte_buffer.cc, - src/cpp/util/slice.cc, src/cpp/util/status.cc, src/cpp/util/string_ref.cc, src/cpp/util/time.cc] + public_headers: + - include/grpc++/channel.h + - include/grpc++/client_context.h + - include/grpc++/completion_queue.h + - include/grpc++/create_channel.h + - include/grpc++/generic/async_generic_service.h + - include/grpc++/generic/generic_stub.h + - include/grpc++/grpc++.h + - include/grpc++/impl/call.h + - include/grpc++/impl/client_unary_call.h + - include/grpc++/impl/grpc_library.h + - include/grpc++/impl/proto_utils.h + - include/grpc++/impl/rpc_method.h + - include/grpc++/impl/rpc_service_method.h + - include/grpc++/impl/serialization_traits.h + - include/grpc++/impl/service_type.h + - include/grpc++/impl/sync.h + - include/grpc++/impl/sync_cxx11.h + - include/grpc++/impl/sync_no_cxx11.h + - include/grpc++/impl/thd.h + - include/grpc++/impl/thd_cxx11.h + - include/grpc++/impl/thd_no_cxx11.h + - include/grpc++/security/auth_context.h + - include/grpc++/security/auth_metadata_processor.h + - include/grpc++/security/credentials.h + - include/grpc++/security/server_credentials.h + - include/grpc++/server.h + - include/grpc++/server_builder.h + - include/grpc++/server_context.h + - include/grpc++/support/async_stream.h + - include/grpc++/support/async_unary_call.h + - include/grpc++/support/byte_buffer.h + - include/grpc++/support/channel_arguments.h + - include/grpc++/support/config.h + - include/grpc++/support/config_protobuf.h + - include/grpc++/support/slice.h + - include/grpc++/support/status.h + - include/grpc++/support/status_code_enum.h + - include/grpc++/support/string_ref.h + - include/grpc++/support/stub_options.h + - include/grpc++/support/sync_stream.h + - include/grpc++/support/time.h + headers: + - src/cpp/client/create_channel_internal.h + - src/cpp/common/create_auth_context.h + - src/cpp/server/dynamic_thread_pool.h + - src/cpp/server/fixed_size_thread_pool.h + - src/cpp/server/thread_pool_interface.h + src: + - src/cpp/client/channel.cc + - src/cpp/client/channel_arguments.cc + - src/cpp/client/client_context.cc + - src/cpp/client/create_channel.cc + - src/cpp/client/create_channel_internal.cc + - src/cpp/client/credentials.cc + - src/cpp/client/generic_stub.cc + - src/cpp/client/insecure_credentials.cc + - src/cpp/common/call.cc + - src/cpp/common/completion_queue.cc + - src/cpp/common/rpc_method.cc + - src/cpp/proto/proto_utils.cc + - src/cpp/server/async_generic_service.cc + - src/cpp/server/create_default_thread_pool.cc + - src/cpp/server/dynamic_thread_pool.cc + - src/cpp/server/fixed_size_thread_pool.cc + - src/cpp/server/insecure_server_credentials.cc + - src/cpp/server/server.cc + - src/cpp/server/server_builder.cc + - src/cpp/server/server_context.cc + - src/cpp/server/server_credentials.cc + - src/cpp/util/byte_buffer.cc + - src/cpp/util/slice.cc + - src/cpp/util/status.cc + - src/cpp/util/string_ref.cc + - src/cpp/util/time.cc - name: grpc_base - public_headers: [include/grpc/byte_buffer.h, include/grpc/byte_buffer_reader.h, - include/grpc/compression.h, include/grpc/grpc.h, include/grpc/status.h] - headers: [src/core/census/grpc_filter.h, src/core/channel/channel_args.h, src/core/channel/channel_stack.h, - src/core/channel/client_channel.h, src/core/channel/compress_filter.h, src/core/channel/connected_channel.h, - src/core/channel/context.h, src/core/channel/http_client_filter.h, src/core/channel/http_server_filter.h, - src/core/channel/noop_filter.h, src/core/client_config/client_config.h, src/core/client_config/connector.h, - src/core/client_config/lb_policies/pick_first.h, src/core/client_config/lb_policies/round_robin.h, - src/core/client_config/lb_policy.h, src/core/client_config/lb_policy_factory.h, - src/core/client_config/lb_policy_registry.h, src/core/client_config/resolver.h, - src/core/client_config/resolver_factory.h, src/core/client_config/resolver_registry.h, - src/core/client_config/resolvers/dns_resolver.h, src/core/client_config/resolvers/sockaddr_resolver.h, - src/core/client_config/subchannel.h, src/core/client_config/subchannel_factory.h, - src/core/client_config/subchannel_factory_decorators/add_channel_arg.h, src/core/client_config/subchannel_factory_decorators/merge_channel_args.h, - src/core/client_config/uri_parser.h, src/core/compression/message_compress.h, - src/core/debug/trace.h, src/core/httpcli/format_request.h, src/core/httpcli/httpcli.h, - src/core/httpcli/parser.h, src/core/iomgr/alarm.h, src/core/iomgr/alarm_heap.h, - src/core/iomgr/alarm_internal.h, src/core/iomgr/closure.h, src/core/iomgr/endpoint.h, - src/core/iomgr/endpoint_pair.h, src/core/iomgr/exec_ctx.h, src/core/iomgr/fd_posix.h, - src/core/iomgr/iocp_windows.h, src/core/iomgr/iomgr.h, src/core/iomgr/iomgr_internal.h, - src/core/iomgr/iomgr_posix.h, src/core/iomgr/pollset.h, src/core/iomgr/pollset_posix.h, - src/core/iomgr/pollset_set.h, src/core/iomgr/pollset_set_posix.h, src/core/iomgr/pollset_set_windows.h, - src/core/iomgr/pollset_windows.h, src/core/iomgr/resolve_address.h, src/core/iomgr/sockaddr.h, - src/core/iomgr/sockaddr_posix.h, src/core/iomgr/sockaddr_utils.h, src/core/iomgr/sockaddr_win32.h, - src/core/iomgr/socket_utils_posix.h, src/core/iomgr/socket_windows.h, src/core/iomgr/tcp_client.h, - src/core/iomgr/tcp_posix.h, src/core/iomgr/tcp_server.h, src/core/iomgr/tcp_windows.h, - src/core/iomgr/time_averaged_stats.h, src/core/iomgr/udp_server.h, src/core/iomgr/wakeup_fd_pipe.h, - src/core/iomgr/wakeup_fd_posix.h, src/core/iomgr/workqueue.h, src/core/iomgr/workqueue_posix.h, - src/core/iomgr/workqueue_windows.h, src/core/json/json.h, src/core/json/json_common.h, - src/core/json/json_reader.h, src/core/json/json_writer.h, src/core/profiling/timers.h, - src/core/statistics/census_interface.h, src/core/statistics/census_rpc_stats.h, - src/core/surface/byte_buffer_queue.h, src/core/surface/call.h, src/core/surface/channel.h, - src/core/surface/completion_queue.h, src/core/surface/event_string.h, src/core/surface/init.h, - src/core/surface/server.h, src/core/surface/surface_trace.h, src/core/transport/chttp2/alpn.h, - src/core/transport/chttp2/bin_encoder.h, src/core/transport/chttp2/frame.h, src/core/transport/chttp2/frame_data.h, - src/core/transport/chttp2/frame_goaway.h, src/core/transport/chttp2/frame_ping.h, - src/core/transport/chttp2/frame_rst_stream.h, src/core/transport/chttp2/frame_settings.h, - src/core/transport/chttp2/frame_window_update.h, src/core/transport/chttp2/hpack_parser.h, - src/core/transport/chttp2/hpack_table.h, src/core/transport/chttp2/http2_errors.h, - src/core/transport/chttp2/huffsyms.h, src/core/transport/chttp2/incoming_metadata.h, - src/core/transport/chttp2/internal.h, src/core/transport/chttp2/status_conversion.h, - src/core/transport/chttp2/stream_encoder.h, src/core/transport/chttp2/stream_map.h, - src/core/transport/chttp2/timeout_encoding.h, src/core/transport/chttp2/varint.h, - src/core/transport/chttp2_transport.h, src/core/transport/connectivity_state.h, - src/core/transport/metadata.h, src/core/transport/stream_op.h, src/core/transport/transport.h, - src/core/transport/transport_impl.h] - src: [src/core/census/grpc_context.c, src/core/census/grpc_filter.c, src/core/channel/channel_args.c, - src/core/channel/channel_stack.c, src/core/channel/client_channel.c, src/core/channel/compress_filter.c, - src/core/channel/connected_channel.c, src/core/channel/http_client_filter.c, src/core/channel/http_server_filter.c, - src/core/channel/noop_filter.c, src/core/client_config/client_config.c, src/core/client_config/connector.c, - src/core/client_config/lb_policies/pick_first.c, src/core/client_config/lb_policies/round_robin.c, - src/core/client_config/lb_policy.c, src/core/client_config/lb_policy_factory.c, - src/core/client_config/lb_policy_registry.c, src/core/client_config/resolver.c, - src/core/client_config/resolver_factory.c, src/core/client_config/resolver_registry.c, - src/core/client_config/resolvers/dns_resolver.c, src/core/client_config/resolvers/sockaddr_resolver.c, - src/core/client_config/subchannel.c, src/core/client_config/subchannel_factory.c, - src/core/client_config/subchannel_factory_decorators/add_channel_arg.c, src/core/client_config/subchannel_factory_decorators/merge_channel_args.c, - src/core/client_config/uri_parser.c, src/core/compression/algorithm.c, src/core/compression/message_compress.c, - src/core/debug/trace.c, src/core/httpcli/format_request.c, src/core/httpcli/httpcli.c, - src/core/httpcli/parser.c, src/core/iomgr/alarm.c, src/core/iomgr/alarm_heap.c, - src/core/iomgr/closure.c, src/core/iomgr/endpoint.c, src/core/iomgr/endpoint_pair_posix.c, - src/core/iomgr/endpoint_pair_windows.c, src/core/iomgr/exec_ctx.c, src/core/iomgr/fd_posix.c, - src/core/iomgr/iocp_windows.c, src/core/iomgr/iomgr.c, src/core/iomgr/iomgr_posix.c, - src/core/iomgr/iomgr_windows.c, src/core/iomgr/pollset_multipoller_with_epoll.c, - src/core/iomgr/pollset_multipoller_with_poll_posix.c, src/core/iomgr/pollset_posix.c, - src/core/iomgr/pollset_set_posix.c, src/core/iomgr/pollset_set_windows.c, src/core/iomgr/pollset_windows.c, - src/core/iomgr/resolve_address_posix.c, src/core/iomgr/resolve_address_windows.c, - src/core/iomgr/sockaddr_utils.c, src/core/iomgr/socket_utils_common_posix.c, src/core/iomgr/socket_utils_linux.c, - src/core/iomgr/socket_utils_posix.c, src/core/iomgr/socket_windows.c, src/core/iomgr/tcp_client_posix.c, - src/core/iomgr/tcp_client_windows.c, src/core/iomgr/tcp_posix.c, src/core/iomgr/tcp_server_posix.c, - src/core/iomgr/tcp_server_windows.c, src/core/iomgr/tcp_windows.c, src/core/iomgr/time_averaged_stats.c, - src/core/iomgr/udp_server.c, src/core/iomgr/wakeup_fd_eventfd.c, src/core/iomgr/wakeup_fd_nospecial.c, - src/core/iomgr/wakeup_fd_pipe.c, src/core/iomgr/wakeup_fd_posix.c, src/core/iomgr/workqueue_posix.c, - src/core/iomgr/workqueue_windows.c, src/core/json/json.c, src/core/json/json_reader.c, - src/core/json/json_string.c, src/core/json/json_writer.c, src/core/profiling/basic_timers.c, - src/core/profiling/stap_timers.c, src/core/surface/byte_buffer.c, src/core/surface/byte_buffer_queue.c, - src/core/surface/byte_buffer_reader.c, src/core/surface/call.c, src/core/surface/call_details.c, - src/core/surface/call_log_batch.c, src/core/surface/channel.c, src/core/surface/channel_connectivity.c, - src/core/surface/channel_create.c, src/core/surface/completion_queue.c, src/core/surface/event_string.c, - src/core/surface/init.c, src/core/surface/lame_client.c, src/core/surface/metadata_array.c, - src/core/surface/server.c, src/core/surface/server_chttp2.c, src/core/surface/server_create.c, - src/core/surface/surface_trace.c, src/core/surface/version.c, src/core/transport/chttp2/alpn.c, - src/core/transport/chttp2/bin_encoder.c, src/core/transport/chttp2/frame_data.c, - src/core/transport/chttp2/frame_goaway.c, src/core/transport/chttp2/frame_ping.c, - src/core/transport/chttp2/frame_rst_stream.c, src/core/transport/chttp2/frame_settings.c, - src/core/transport/chttp2/frame_window_update.c, src/core/transport/chttp2/hpack_parser.c, - src/core/transport/chttp2/hpack_table.c, src/core/transport/chttp2/huffsyms.c, - src/core/transport/chttp2/incoming_metadata.c, src/core/transport/chttp2/parsing.c, - src/core/transport/chttp2/status_conversion.c, src/core/transport/chttp2/stream_encoder.c, - src/core/transport/chttp2/stream_lists.c, src/core/transport/chttp2/stream_map.c, - src/core/transport/chttp2/timeout_encoding.c, src/core/transport/chttp2/varint.c, - src/core/transport/chttp2/writing.c, src/core/transport/chttp2_transport.c, src/core/transport/connectivity_state.c, - src/core/transport/metadata.c, src/core/transport/stream_op.c, src/core/transport/transport.c, - src/core/transport/transport_op_string.c] + public_headers: + - include/grpc/byte_buffer.h + - include/grpc/byte_buffer_reader.h + - include/grpc/compression.h + - include/grpc/grpc.h + - include/grpc/status.h + headers: + - src/core/census/grpc_filter.h + - src/core/channel/channel_args.h + - src/core/channel/channel_stack.h + - src/core/channel/client_channel.h + - src/core/channel/compress_filter.h + - src/core/channel/connected_channel.h + - src/core/channel/context.h + - src/core/channel/http_client_filter.h + - src/core/channel/http_server_filter.h + - src/core/channel/noop_filter.h + - src/core/client_config/client_config.h + - src/core/client_config/connector.h + - src/core/client_config/lb_policies/pick_first.h + - src/core/client_config/lb_policies/round_robin.h + - src/core/client_config/lb_policy.h + - src/core/client_config/lb_policy_factory.h + - src/core/client_config/lb_policy_registry.h + - src/core/client_config/resolver.h + - src/core/client_config/resolver_factory.h + - src/core/client_config/resolver_registry.h + - src/core/client_config/resolvers/dns_resolver.h + - src/core/client_config/resolvers/sockaddr_resolver.h + - src/core/client_config/subchannel.h + - src/core/client_config/subchannel_factory.h + - src/core/client_config/subchannel_factory_decorators/add_channel_arg.h + - src/core/client_config/subchannel_factory_decorators/merge_channel_args.h + - src/core/client_config/uri_parser.h + - src/core/compression/message_compress.h + - src/core/debug/trace.h + - src/core/httpcli/format_request.h + - src/core/httpcli/httpcli.h + - src/core/httpcli/parser.h + - src/core/iomgr/alarm.h + - src/core/iomgr/alarm_heap.h + - src/core/iomgr/alarm_internal.h + - src/core/iomgr/closure.h + - src/core/iomgr/endpoint.h + - src/core/iomgr/endpoint_pair.h + - src/core/iomgr/exec_ctx.h + - src/core/iomgr/fd_posix.h + - src/core/iomgr/iocp_windows.h + - src/core/iomgr/iomgr.h + - src/core/iomgr/iomgr_internal.h + - src/core/iomgr/iomgr_posix.h + - src/core/iomgr/pollset.h + - src/core/iomgr/pollset_posix.h + - src/core/iomgr/pollset_set.h + - src/core/iomgr/pollset_set_posix.h + - src/core/iomgr/pollset_set_windows.h + - src/core/iomgr/pollset_windows.h + - src/core/iomgr/resolve_address.h + - src/core/iomgr/sockaddr.h + - src/core/iomgr/sockaddr_posix.h + - src/core/iomgr/sockaddr_utils.h + - src/core/iomgr/sockaddr_win32.h + - src/core/iomgr/socket_utils_posix.h + - src/core/iomgr/socket_windows.h + - src/core/iomgr/tcp_client.h + - src/core/iomgr/tcp_posix.h + - src/core/iomgr/tcp_server.h + - src/core/iomgr/tcp_windows.h + - src/core/iomgr/time_averaged_stats.h + - src/core/iomgr/udp_server.h + - src/core/iomgr/wakeup_fd_pipe.h + - src/core/iomgr/wakeup_fd_posix.h + - src/core/iomgr/workqueue.h + - src/core/iomgr/workqueue_posix.h + - src/core/iomgr/workqueue_windows.h + - src/core/json/json.h + - src/core/json/json_common.h + - src/core/json/json_reader.h + - src/core/json/json_writer.h + - src/core/profiling/timers.h + - src/core/statistics/census_interface.h + - src/core/statistics/census_rpc_stats.h + - src/core/surface/byte_buffer_queue.h + - src/core/surface/call.h + - src/core/surface/channel.h + - src/core/surface/completion_queue.h + - src/core/surface/event_string.h + - src/core/surface/init.h + - src/core/surface/server.h + - src/core/surface/surface_trace.h + - src/core/transport/chttp2/alpn.h + - src/core/transport/chttp2/bin_encoder.h + - src/core/transport/chttp2/frame.h + - src/core/transport/chttp2/frame_data.h + - src/core/transport/chttp2/frame_goaway.h + - src/core/transport/chttp2/frame_ping.h + - src/core/transport/chttp2/frame_rst_stream.h + - src/core/transport/chttp2/frame_settings.h + - src/core/transport/chttp2/frame_window_update.h + - src/core/transport/chttp2/hpack_parser.h + - src/core/transport/chttp2/hpack_table.h + - src/core/transport/chttp2/http2_errors.h + - src/core/transport/chttp2/huffsyms.h + - src/core/transport/chttp2/incoming_metadata.h + - src/core/transport/chttp2/internal.h + - src/core/transport/chttp2/status_conversion.h + - src/core/transport/chttp2/stream_encoder.h + - src/core/transport/chttp2/stream_map.h + - src/core/transport/chttp2/timeout_encoding.h + - src/core/transport/chttp2/varint.h + - src/core/transport/chttp2_transport.h + - src/core/transport/connectivity_state.h + - src/core/transport/metadata.h + - src/core/transport/stream_op.h + - src/core/transport/transport.h + - src/core/transport/transport_impl.h + src: + - src/core/census/grpc_context.c + - src/core/census/grpc_filter.c + - src/core/channel/channel_args.c + - src/core/channel/channel_stack.c + - src/core/channel/client_channel.c + - src/core/channel/compress_filter.c + - src/core/channel/connected_channel.c + - src/core/channel/http_client_filter.c + - src/core/channel/http_server_filter.c + - src/core/channel/noop_filter.c + - src/core/client_config/client_config.c + - src/core/client_config/connector.c + - src/core/client_config/lb_policies/pick_first.c + - src/core/client_config/lb_policies/round_robin.c + - src/core/client_config/lb_policy.c + - src/core/client_config/lb_policy_factory.c + - src/core/client_config/lb_policy_registry.c + - src/core/client_config/resolver.c + - src/core/client_config/resolver_factory.c + - src/core/client_config/resolver_registry.c + - src/core/client_config/resolvers/dns_resolver.c + - src/core/client_config/resolvers/sockaddr_resolver.c + - src/core/client_config/subchannel.c + - src/core/client_config/subchannel_factory.c + - src/core/client_config/subchannel_factory_decorators/add_channel_arg.c + - src/core/client_config/subchannel_factory_decorators/merge_channel_args.c + - src/core/client_config/uri_parser.c + - src/core/compression/algorithm.c + - src/core/compression/message_compress.c + - src/core/debug/trace.c + - src/core/httpcli/format_request.c + - src/core/httpcli/httpcli.c + - src/core/httpcli/parser.c + - src/core/iomgr/alarm.c + - src/core/iomgr/alarm_heap.c + - src/core/iomgr/closure.c + - src/core/iomgr/endpoint.c + - src/core/iomgr/endpoint_pair_posix.c + - src/core/iomgr/endpoint_pair_windows.c + - src/core/iomgr/exec_ctx.c + - src/core/iomgr/fd_posix.c + - src/core/iomgr/iocp_windows.c + - src/core/iomgr/iomgr.c + - src/core/iomgr/iomgr_posix.c + - src/core/iomgr/iomgr_windows.c + - src/core/iomgr/pollset_multipoller_with_epoll.c + - src/core/iomgr/pollset_multipoller_with_poll_posix.c + - src/core/iomgr/pollset_posix.c + - src/core/iomgr/pollset_set_posix.c + - src/core/iomgr/pollset_set_windows.c + - src/core/iomgr/pollset_windows.c + - src/core/iomgr/resolve_address_posix.c + - src/core/iomgr/resolve_address_windows.c + - src/core/iomgr/sockaddr_utils.c + - src/core/iomgr/socket_utils_common_posix.c + - src/core/iomgr/socket_utils_linux.c + - src/core/iomgr/socket_utils_posix.c + - src/core/iomgr/socket_windows.c + - src/core/iomgr/tcp_client_posix.c + - src/core/iomgr/tcp_client_windows.c + - src/core/iomgr/tcp_posix.c + - src/core/iomgr/tcp_server_posix.c + - src/core/iomgr/tcp_server_windows.c + - src/core/iomgr/tcp_windows.c + - src/core/iomgr/time_averaged_stats.c + - src/core/iomgr/udp_server.c + - src/core/iomgr/wakeup_fd_eventfd.c + - src/core/iomgr/wakeup_fd_nospecial.c + - src/core/iomgr/wakeup_fd_pipe.c + - src/core/iomgr/wakeup_fd_posix.c + - src/core/iomgr/workqueue_posix.c + - src/core/iomgr/workqueue_windows.c + - src/core/json/json.c + - src/core/json/json_reader.c + - src/core/json/json_string.c + - src/core/json/json_writer.c + - src/core/profiling/basic_timers.c + - src/core/profiling/stap_timers.c + - src/core/surface/byte_buffer.c + - src/core/surface/byte_buffer_queue.c + - src/core/surface/byte_buffer_reader.c + - src/core/surface/call.c + - src/core/surface/call_details.c + - src/core/surface/call_log_batch.c + - src/core/surface/channel.c + - src/core/surface/channel_connectivity.c + - src/core/surface/channel_create.c + - src/core/surface/completion_queue.c + - src/core/surface/event_string.c + - src/core/surface/init.c + - src/core/surface/lame_client.c + - src/core/surface/metadata_array.c + - src/core/surface/server.c + - src/core/surface/server_chttp2.c + - src/core/surface/server_create.c + - src/core/surface/surface_trace.c + - src/core/surface/version.c + - src/core/transport/chttp2/alpn.c + - src/core/transport/chttp2/bin_encoder.c + - src/core/transport/chttp2/frame_data.c + - src/core/transport/chttp2/frame_goaway.c + - src/core/transport/chttp2/frame_ping.c + - src/core/transport/chttp2/frame_rst_stream.c + - src/core/transport/chttp2/frame_settings.c + - src/core/transport/chttp2/frame_window_update.c + - src/core/transport/chttp2/hpack_parser.c + - src/core/transport/chttp2/hpack_table.c + - src/core/transport/chttp2/huffsyms.c + - src/core/transport/chttp2/incoming_metadata.c + - src/core/transport/chttp2/parsing.c + - src/core/transport/chttp2/status_conversion.c + - src/core/transport/chttp2/stream_encoder.c + - src/core/transport/chttp2/stream_lists.c + - src/core/transport/chttp2/stream_map.c + - src/core/transport/chttp2/timeout_encoding.c + - src/core/transport/chttp2/varint.c + - src/core/transport/chttp2/writing.c + - src/core/transport/chttp2_transport.c + - src/core/transport/connectivity_state.c + - src/core/transport/metadata.c + - src/core/transport/stream_op.c + - src/core/transport/transport.c + - src/core/transport/transport_op_string.c - name: grpc_test_util_base - headers: [test/core/end2end/cq_verifier.h, test/core/end2end/fixtures/proxy.h, test/core/iomgr/endpoint_tests.h, - test/core/security/oauth2_utils.h, test/core/util/grpc_profiler.h, test/core/util/parse_hexstring.h, - test/core/util/port.h, test/core/util/slice_splitter.h] - src: [test/core/end2end/cq_verifier.c, test/core/end2end/fixtures/proxy.c, test/core/iomgr/endpoint_tests.c, - test/core/security/oauth2_utils.c, test/core/util/grpc_profiler.c, test/core/util/parse_hexstring.c, - test/core/util/port_posix.c, test/core/util/port_windows.c, test/core/util/slice_splitter.c] + headers: + - test/core/end2end/cq_verifier.h + - test/core/end2end/fixtures/proxy.h + - test/core/iomgr/endpoint_tests.h + - test/core/security/oauth2_utils.h + - test/core/util/grpc_profiler.h + - test/core/util/parse_hexstring.h + - test/core/util/port.h + - test/core/util/slice_splitter.h + src: + - test/core/end2end/cq_verifier.c + - test/core/end2end/fixtures/proxy.c + - test/core/iomgr/endpoint_tests.c + - test/core/security/oauth2_utils.c + - test/core/util/grpc_profiler.c + - test/core/util/parse_hexstring.c + - test/core/util/port_posix.c + - test/core/util/port_windows.c + - test/core/util/slice_splitter.c libs: - name: gpr build: all language: c - public_headers: [include/grpc/support/alloc.h, include/grpc/support/atm.h, include/grpc/support/atm_gcc_atomic.h, - include/grpc/support/atm_gcc_sync.h, include/grpc/support/atm_win32.h, include/grpc/support/cmdline.h, - include/grpc/support/cpu.h, include/grpc/support/histogram.h, include/grpc/support/host_port.h, - include/grpc/support/log.h, include/grpc/support/log_win32.h, include/grpc/support/port_platform.h, - include/grpc/support/slice.h, include/grpc/support/slice_buffer.h, include/grpc/support/string_util.h, - include/grpc/support/subprocess.h, include/grpc/support/sync.h, include/grpc/support/sync_generic.h, - include/grpc/support/sync_posix.h, include/grpc/support/sync_win32.h, include/grpc/support/thd.h, - include/grpc/support/time.h, include/grpc/support/tls.h, include/grpc/support/tls_gcc.h, - include/grpc/support/tls_msvc.h, include/grpc/support/tls_pthread.h, include/grpc/support/useful.h] - headers: [src/core/support/env.h, src/core/support/file.h, src/core/support/murmur_hash.h, - src/core/support/stack_lockfree.h, src/core/support/string.h, src/core/support/string_win32.h, - src/core/support/thd_internal.h, src/core/support/time_precise.h] - src: [src/core/support/alloc.c, src/core/support/cmdline.c, src/core/support/cpu_iphone.c, - src/core/support/cpu_linux.c, src/core/support/cpu_posix.c, src/core/support/cpu_windows.c, - src/core/support/env_linux.c, src/core/support/env_posix.c, src/core/support/env_win32.c, - src/core/support/file.c, src/core/support/file_posix.c, src/core/support/file_win32.c, - src/core/support/histogram.c, src/core/support/host_port.c, src/core/support/log.c, - src/core/support/log_android.c, src/core/support/log_linux.c, src/core/support/log_posix.c, - src/core/support/log_win32.c, src/core/support/murmur_hash.c, src/core/support/slice.c, - src/core/support/slice_buffer.c, src/core/support/stack_lockfree.c, src/core/support/string.c, - src/core/support/string_posix.c, src/core/support/string_win32.c, src/core/support/subprocess_posix.c, - src/core/support/sync.c, src/core/support/sync_posix.c, src/core/support/sync_win32.c, - src/core/support/thd.c, src/core/support/thd_posix.c, src/core/support/thd_win32.c, - src/core/support/time.c, src/core/support/time_posix.c, src/core/support/time_win32.c, - src/core/support/tls_pthread.c] + public_headers: + - include/grpc/support/alloc.h + - include/grpc/support/atm.h + - include/grpc/support/atm_gcc_atomic.h + - include/grpc/support/atm_gcc_sync.h + - include/grpc/support/atm_win32.h + - include/grpc/support/cmdline.h + - include/grpc/support/cpu.h + - include/grpc/support/histogram.h + - include/grpc/support/host_port.h + - include/grpc/support/log.h + - include/grpc/support/log_win32.h + - include/grpc/support/port_platform.h + - include/grpc/support/slice.h + - include/grpc/support/slice_buffer.h + - include/grpc/support/string_util.h + - include/grpc/support/subprocess.h + - include/grpc/support/sync.h + - include/grpc/support/sync_generic.h + - include/grpc/support/sync_posix.h + - include/grpc/support/sync_win32.h + - include/grpc/support/thd.h + - include/grpc/support/time.h + - include/grpc/support/tls.h + - include/grpc/support/tls_gcc.h + - include/grpc/support/tls_msvc.h + - include/grpc/support/tls_pthread.h + - include/grpc/support/useful.h + headers: + - src/core/support/env.h + - src/core/support/file.h + - src/core/support/murmur_hash.h + - src/core/support/stack_lockfree.h + - src/core/support/string.h + - src/core/support/string_win32.h + - src/core/support/thd_internal.h + - src/core/support/time_precise.h + src: + - src/core/support/alloc.c + - src/core/support/cmdline.c + - src/core/support/cpu_iphone.c + - src/core/support/cpu_linux.c + - src/core/support/cpu_posix.c + - src/core/support/cpu_windows.c + - src/core/support/env_linux.c + - src/core/support/env_posix.c + - src/core/support/env_win32.c + - src/core/support/file.c + - src/core/support/file_posix.c + - src/core/support/file_win32.c + - src/core/support/histogram.c + - src/core/support/host_port.c + - src/core/support/log.c + - src/core/support/log_android.c + - src/core/support/log_linux.c + - src/core/support/log_posix.c + - src/core/support/log_win32.c + - src/core/support/murmur_hash.c + - src/core/support/slice.c + - src/core/support/slice_buffer.c + - src/core/support/stack_lockfree.c + - src/core/support/string.c + - src/core/support/string_posix.c + - src/core/support/string_win32.c + - src/core/support/subprocess_posix.c + - src/core/support/sync.c + - src/core/support/sync_posix.c + - src/core/support/sync_win32.c + - src/core/support/thd.c + - src/core/support/thd_posix.c + - src/core/support/thd_win32.c + - src/core/support/time.c + - src/core/support/time_posix.c + - src/core/support/time_win32.c + - src/core/support/tls_pthread.c secure: false vs_project_guid: '{B23D3D1A-9438-4EDA-BEB6-9A0A03D17792}' - name: gpr_test_util build: private language: c - headers: [test/core/util/test_config.h] - src: [test/core/util/test_config.c] - deps: [gpr] + headers: + - test/core/util/test_config.h + src: + - test/core/util/test_config.c + deps: + - gpr secure: false vs_project_guid: '{EAB0A629-17A9-44DB-B5FF-E91A721FE037}' - name: grpc build: all language: c - public_headers: [include/grpc/grpc_security.h] - headers: [src/core/security/auth_filters.h, src/core/security/base64.h, src/core/security/credentials.h, - src/core/security/handshake.h, src/core/security/json_token.h, src/core/security/jwt_verifier.h, - src/core/security/secure_endpoint.h, src/core/security/security_connector.h, src/core/security/security_context.h, - src/core/tsi/fake_transport_security.h, src/core/tsi/ssl_transport_security.h, - src/core/tsi/transport_security.h, src/core/tsi/transport_security_interface.h] - src: [src/core/httpcli/httpcli_security_connector.c, src/core/security/base64.c, - src/core/security/client_auth_filter.c, src/core/security/credentials.c, src/core/security/credentials_metadata.c, - src/core/security/credentials_posix.c, src/core/security/credentials_win32.c, - src/core/security/google_default_credentials.c, src/core/security/handshake.c, - src/core/security/json_token.c, src/core/security/jwt_verifier.c, src/core/security/secure_endpoint.c, - src/core/security/security_connector.c, src/core/security/security_context.c, - src/core/security/server_auth_filter.c, src/core/security/server_secure_chttp2.c, - src/core/surface/init_secure.c, src/core/surface/secure_channel_create.c, src/core/tsi/fake_transport_security.c, - src/core/tsi/ssl_transport_security.c, src/core/tsi/transport_security.c] - deps: [gpr] + public_headers: + - include/grpc/grpc_security.h + headers: + - src/core/security/auth_filters.h + - src/core/security/base64.h + - src/core/security/credentials.h + - src/core/security/handshake.h + - src/core/security/json_token.h + - src/core/security/jwt_verifier.h + - src/core/security/secure_endpoint.h + - src/core/security/security_connector.h + - src/core/security/security_context.h + - src/core/tsi/fake_transport_security.h + - src/core/tsi/ssl_transport_security.h + - src/core/tsi/transport_security.h + - src/core/tsi/transport_security_interface.h + src: + - src/core/httpcli/httpcli_security_connector.c + - src/core/security/base64.c + - src/core/security/client_auth_filter.c + - src/core/security/credentials.c + - src/core/security/credentials_metadata.c + - src/core/security/credentials_posix.c + - src/core/security/credentials_win32.c + - src/core/security/google_default_credentials.c + - src/core/security/handshake.c + - src/core/security/json_token.c + - src/core/security/jwt_verifier.c + - src/core/security/secure_endpoint.c + - src/core/security/security_connector.c + - src/core/security/security_context.c + - src/core/security/server_auth_filter.c + - src/core/security/server_secure_chttp2.c + - src/core/surface/init_secure.c + - src/core/surface/secure_channel_create.c + - src/core/tsi/fake_transport_security.c + - src/core/tsi/ssl_transport_security.c + - src/core/tsi/transport_security.c + deps: + - gpr baselib: true dll: true - filegroups: [grpc_base, census] + filegroups: + - grpc_base + - census secure: true - vs_packages: [grpc.dependencies.openssl, grpc.dependencies.zlib] + vs_packages: + - grpc.dependencies.openssl + - grpc.dependencies.zlib vs_project_guid: '{29D16885-7228-4C31-81ED-5F9187C7F2A9}' - name: grpc_test_util build: private language: c - headers: [test/core/end2end/data/ssl_test_data.h] - src: [test/core/end2end/data/server1_cert.c, test/core/end2end/data/server1_key.c, - test/core/end2end/data/test_root_cert.c] - deps: [gpr, gpr_test_util, grpc] - filegroups: [grpc_test_util_base] + headers: + - test/core/end2end/data/ssl_test_data.h + src: + - test/core/end2end/data/server1_cert.c + - test/core/end2end/data/server1_key.c + - test/core/end2end/data/test_root_cert.c + deps: + - gpr + - gpr_test_util + - grpc + filegroups: + - grpc_test_util_base vs_project_guid: '{17BCAFC0-5FDC-4C94-AEB9-95F3E220614B}' - name: grpc_test_util_unsecure build: private language: c - deps: [gpr, gpr_test_util, grpc] - filegroups: [grpc_test_util_base] + deps: + - gpr + - gpr_test_util + - grpc + filegroups: + - grpc_test_util_base secure: false vs_project_guid: '{0A7E7F92-FDEA-40F1-A9EC-3BA484F98BBF}' - name: grpc_unsecure build: all language: c - src: [src/core/surface/init_unsecure.c] - deps: [gpr] + src: + - src/core/surface/init_unsecure.c + deps: + - gpr baselib: true dll: true - filegroups: [grpc_base, census] + filegroups: + - grpc_base + - census secure: false vs_project_guid: '{46CEDFFF-9692-456A-AA24-38B5D6BCF4C5}' - name: grpc_zookeeper build: all language: c - public_headers: [include/grpc/grpc_zookeeper.h] - headers: [src/core/client_config/resolvers/zookeeper_resolver.h] - src: [src/core/client_config/resolvers/zookeeper_resolver.c] - deps: [gpr, grpc] - external_deps: [zookeeper] - platforms: [linux] + public_headers: + - include/grpc/grpc_zookeeper.h + headers: + - src/core/client_config/resolvers/zookeeper_resolver.h + src: + - src/core/client_config/resolvers/zookeeper_resolver.c + deps: + - gpr + - grpc + external_deps: + - zookeeper + platforms: + - linux secure: false - name: reconnect_server build: private language: c - headers: [test/core/util/reconnect_server.h] - src: [test/core/util/reconnect_server.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + headers: + - test/core/util/reconnect_server.h + src: + - test/core/util/reconnect_server.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: grpc++ build: all language: c++ - headers: [src/cpp/client/secure_credentials.h, src/cpp/common/secure_auth_context.h, - src/cpp/server/secure_server_credentials.h] - src: [src/cpp/client/secure_channel_arguments.cc, src/cpp/client/secure_credentials.cc, - src/cpp/common/auth_property_iterator.cc, src/cpp/common/secure_auth_context.cc, - src/cpp/common/secure_create_auth_context.cc, src/cpp/server/secure_server_credentials.cc] - deps: [gpr, grpc] + headers: + - src/cpp/client/secure_credentials.h + - src/cpp/common/secure_auth_context.h + - src/cpp/server/secure_server_credentials.h + src: + - src/cpp/client/secure_channel_arguments.cc + - src/cpp/client/secure_credentials.cc + - src/cpp/common/auth_property_iterator.cc + - src/cpp/common/secure_auth_context.cc + - src/cpp/common/secure_create_auth_context.cc + - src/cpp/server/secure_server_credentials.cc + deps: + - gpr + - grpc baselib: true dll: true - filegroups: [grpc++_base] + filegroups: + - grpc++_base secure: check vs_project_guid: '{C187A093-A0FE-489D-A40A-6E33DE0F9FEB}' - name: grpc++_test_config build: private language: c++ - headers: [test/cpp/util/test_config.h] - src: [test/cpp/util/test_config.cc] + headers: + - test/cpp/util/test_config.h + src: + - test/cpp/util/test_config.cc - name: grpc++_test_util build: private language: c++ - headers: [test/cpp/util/cli_call.h, test/cpp/util/create_test_channel.h, test/cpp/util/string_ref_helper.h, - test/cpp/util/subprocess.h] - src: [test/cpp/util/messages.proto, test/cpp/util/echo.proto, test/cpp/util/echo_duplicate.proto, - test/cpp/util/cli_call.cc, test/cpp/util/create_test_channel.cc, test/cpp/util/string_ref_helper.cc, - test/cpp/util/subprocess.cc] - deps: [grpc++, grpc_test_util] + headers: + - test/cpp/util/cli_call.h + - test/cpp/util/create_test_channel.h + - test/cpp/util/string_ref_helper.h + - test/cpp/util/subprocess.h + src: + - test/cpp/util/messages.proto + - test/cpp/util/echo.proto + - test/cpp/util/echo_duplicate.proto + - test/cpp/util/cli_call.cc + - test/cpp/util/create_test_channel.cc + - test/cpp/util/string_ref_helper.cc + - test/cpp/util/subprocess.cc + deps: + - grpc++ + - grpc_test_util - name: grpc++_unsecure build: all language: c++ - src: [src/cpp/common/insecure_create_auth_context.cc] - deps: [gpr, grpc_unsecure] + src: + - src/cpp/common/insecure_create_auth_context.cc + deps: + - gpr + - grpc_unsecure baselib: true dll: true - filegroups: [grpc++_base] + filegroups: + - grpc++_base secure: false vs_project_guid: '{6EE56155-DF7C-4F6E-BFC4-F6F776BEB211}' - name: grpc_plugin_support build: protoc language: c++ - headers: [include/grpc++/support/config.h, include/grpc++/support/config_protobuf.h, - src/compiler/config.h, src/compiler/cpp_generator.h, src/compiler/cpp_generator_helpers.h, - src/compiler/csharp_generator.h, src/compiler/csharp_generator_helpers.h, src/compiler/generator_helpers.h, - src/compiler/objective_c_generator.h, src/compiler/objective_c_generator_helpers.h, - src/compiler/python_generator.h, src/compiler/ruby_generator.h, src/compiler/ruby_generator_helpers-inl.h, - src/compiler/ruby_generator_map-inl.h, src/compiler/ruby_generator_string-inl.h] - src: [src/compiler/cpp_generator.cc, src/compiler/csharp_generator.cc, src/compiler/objective_c_generator.cc, - src/compiler/python_generator.cc, src/compiler/ruby_generator.cc] + headers: + - include/grpc++/support/config.h + - include/grpc++/support/config_protobuf.h + - src/compiler/config.h + - src/compiler/cpp_generator.h + - src/compiler/cpp_generator_helpers.h + - src/compiler/csharp_generator.h + - src/compiler/csharp_generator_helpers.h + - src/compiler/generator_helpers.h + - src/compiler/objective_c_generator.h + - src/compiler/objective_c_generator_helpers.h + - src/compiler/python_generator.h + - src/compiler/ruby_generator.h + - src/compiler/ruby_generator_helpers-inl.h + - src/compiler/ruby_generator_map-inl.h + - src/compiler/ruby_generator_string-inl.h + src: + - src/compiler/cpp_generator.cc + - src/compiler/csharp_generator.cc + - src/compiler/objective_c_generator.cc + - src/compiler/python_generator.cc + - src/compiler/ruby_generator.cc deps: [] secure: false vs_project_guid: '{B6E81D84-2ACB-41B8-8781-493A944C7817}' - name: interop_client_helper build: private language: c++ - headers: [test/cpp/interop/client_helper.h] - src: [test/proto/messages.proto, test/cpp/interop/client_helper.cc] - deps: [grpc++_test_util, grpc_test_util, grpc++, grpc, gpr] + headers: + - test/cpp/interop/client_helper.h + src: + - test/proto/messages.proto + - test/cpp/interop/client_helper.cc + deps: + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr - name: interop_client_main build: private language: c++ - headers: [test/cpp/interop/interop_client.h] - src: [test/proto/empty.proto, test/proto/messages.proto, test/proto/test.proto, - test/cpp/interop/client.cc, test/cpp/interop/interop_client.cc] - deps: [interop_client_helper, grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, - gpr, grpc++_test_config] + headers: + - test/cpp/interop/interop_client.h + src: + - test/proto/empty.proto + - test/proto/messages.proto + - test/proto/test.proto + - test/cpp/interop/client.cc + - test/cpp/interop/interop_client.cc + deps: + - interop_client_helper + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + - grpc++_test_config - name: interop_server_helper build: private language: c++ - headers: [test/cpp/interop/server_helper.h] - src: [test/cpp/interop/server_helper.cc] - deps: [grpc_test_util, grpc++, grpc, gpr] + headers: + - test/cpp/interop/server_helper.h + src: + - test/cpp/interop/server_helper.cc + deps: + - grpc_test_util + - grpc++ + - grpc + - gpr - name: interop_server_main build: private language: c++ - src: [test/proto/empty.proto, test/proto/messages.proto, test/proto/test.proto, - test/cpp/interop/server.cc] - deps: [interop_server_helper, grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, - gpr, grpc++_test_config] + src: + - test/proto/empty.proto + - test/proto/messages.proto + - test/proto/test.proto + - test/cpp/interop/server.cc + deps: + - interop_server_helper + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + - grpc++_test_config - name: qps build: private language: c++ - headers: [test/cpp/qps/client.h, test/cpp/qps/driver.h, test/cpp/qps/histogram.h, - test/cpp/qps/interarrival.h, test/cpp/qps/perf_db_client.h, test/cpp/qps/qps_worker.h, - test/cpp/qps/report.h, test/cpp/qps/server.h, test/cpp/qps/stats.h, test/cpp/qps/timer.h, - test/cpp/util/benchmark_config.h] - src: [test/cpp/qps/qpstest.proto, test/cpp/qps/perf_db.proto, test/cpp/qps/client_async.cc, - test/cpp/qps/client_sync.cc, test/cpp/qps/driver.cc, test/cpp/qps/perf_db_client.cc, - test/cpp/qps/qps_worker.cc, test/cpp/qps/report.cc, test/cpp/qps/server_async.cc, - test/cpp/qps/server_sync.cc, test/cpp/qps/timer.cc, test/cpp/util/benchmark_config.cc] - deps: [grpc_test_util, grpc++_test_util, grpc++] + headers: + - test/cpp/qps/client.h + - test/cpp/qps/driver.h + - test/cpp/qps/histogram.h + - test/cpp/qps/interarrival.h + - test/cpp/qps/perf_db_client.h + - test/cpp/qps/qps_worker.h + - test/cpp/qps/report.h + - test/cpp/qps/server.h + - test/cpp/qps/stats.h + - test/cpp/qps/timer.h + - test/cpp/util/benchmark_config.h + src: + - test/cpp/qps/qpstest.proto + - test/cpp/qps/perf_db.proto + - test/cpp/qps/client_async.cc + - test/cpp/qps/client_sync.cc + - test/cpp/qps/driver.cc + - test/cpp/qps/perf_db_client.cc + - test/cpp/qps/qps_worker.cc + - test/cpp/qps/report.cc + - test/cpp/qps/server_async.cc + - test/cpp/qps/server_sync.cc + - test/cpp/qps/timer.cc + - test/cpp/util/benchmark_config.cc + deps: + - grpc_test_util + - grpc++_test_util + - grpc++ - name: grpc_csharp_ext build: all language: csharp - src: [src/csharp/ext/grpc_csharp_ext.c] - deps: [gpr, grpc] + src: + - src/csharp/ext/grpc_csharp_ext.c + deps: + - gpr + - grpc dll: only vs_config_type: DynamicLibrary - vs_packages: [grpc.dependencies.openssl, grpc.dependencies.zlib] + vs_packages: + - grpc.dependencies.openssl + - grpc.dependencies.zlib vs_project_guid: '{D64C6D63-4458-4A88-AB38-35678384A7E4}' - vs_props: [zlib, openssl, winsock, global] + vs_props: + - zlib + - openssl + - winsock + - global targets: - name: alarm_heap_test build: test language: c - src: [test/core/iomgr/alarm_heap_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/iomgr/alarm_heap_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: alarm_list_test build: test language: c - src: [test/core/iomgr/alarm_list_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/iomgr/alarm_list_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: alpn_test build: test language: c - src: [test/core/transport/chttp2/alpn_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/transport/chttp2/alpn_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: bin_encoder_test build: test language: c - src: [test/core/transport/chttp2/bin_encoder_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/transport/chttp2/bin_encoder_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: chttp2_status_conversion_test build: test language: c - src: [test/core/transport/chttp2/status_conversion_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/transport/chttp2/status_conversion_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: chttp2_stream_encoder_test build: test language: c - src: [test/core/transport/chttp2/stream_encoder_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/transport/chttp2/stream_encoder_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: chttp2_stream_map_test build: test language: c - src: [test/core/transport/chttp2/stream_map_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/transport/chttp2/stream_map_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: compression_test build: test language: c - src: [test/core/compression/compression_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/compression/compression_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: dualstack_socket_test build: test language: c - src: [test/core/end2end/dualstack_socket_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] - platforms: [mac, linux, posix] + src: + - test/core/end2end/dualstack_socket_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr + platforms: + - mac + - linux + - posix - name: endpoint_pair_test build: test language: c - src: [test/core/iomgr/endpoint_pair_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/iomgr/endpoint_pair_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: fd_conservation_posix_test build: test language: c - src: [test/core/iomgr/fd_conservation_posix_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] - platforms: [mac, linux, posix] + src: + - test/core/iomgr/fd_conservation_posix_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr + platforms: + - mac + - linux + - posix - name: fd_posix_test build: test language: c - src: [test/core/iomgr/fd_posix_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] - platforms: [mac, linux, posix] + src: + - test/core/iomgr/fd_posix_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr + platforms: + - mac + - linux + - posix - name: fling_client build: test run: false language: c - src: [test/core/fling/client.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/fling/client.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: fling_server build: test run: false language: c - src: [test/core/fling/server.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/fling/server.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: fling_stream_test build: test language: c - src: [test/core/fling/fling_stream_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] - platforms: [mac, linux, posix] + src: + - test/core/fling/fling_stream_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr + platforms: + - mac + - linux + - posix - name: fling_test build: test language: c - src: [test/core/fling/fling_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] - platforms: [mac, linux, posix] + src: + - test/core/fling/fling_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr + platforms: + - mac + - linux + - posix - name: gen_hpack_tables build: tool language: c - src: [tools/codegen/core/gen_hpack_tables.c] - deps: [gpr, grpc] + src: + - tools/codegen/core/gen_hpack_tables.c + deps: + - gpr + - grpc - name: gen_legal_metadata_characters build: tool language: c - src: [tools/codegen/core/gen_legal_metadata_characters.c] + src: + - tools/codegen/core/gen_legal_metadata_characters.c deps: [] - name: gpr_cmdline_test build: test language: c - src: [test/core/support/cmdline_test.c] - deps: [gpr_test_util, gpr] + src: + - test/core/support/cmdline_test.c + deps: + - gpr_test_util + - gpr - name: gpr_env_test build: test language: c - src: [test/core/support/env_test.c] - deps: [gpr_test_util, gpr] + src: + - test/core/support/env_test.c + deps: + - gpr_test_util + - gpr - name: gpr_file_test build: test language: c - src: [test/core/support/file_test.c] - deps: [gpr_test_util, gpr] + src: + - test/core/support/file_test.c + deps: + - gpr_test_util + - gpr - name: gpr_histogram_test build: test language: c - src: [test/core/support/histogram_test.c] - deps: [gpr_test_util, gpr] + src: + - test/core/support/histogram_test.c + deps: + - gpr_test_util + - gpr - name: gpr_host_port_test build: test language: c - src: [test/core/support/host_port_test.c] - deps: [gpr_test_util, gpr] + src: + - test/core/support/host_port_test.c + deps: + - gpr_test_util + - gpr - name: gpr_log_test build: test language: c - src: [test/core/support/log_test.c] - deps: [gpr_test_util, gpr] + src: + - test/core/support/log_test.c + deps: + - gpr_test_util + - gpr - name: gpr_slice_buffer_test build: test language: c - src: [test/core/support/slice_buffer_test.c] - deps: [gpr_test_util, gpr] + src: + - test/core/support/slice_buffer_test.c + deps: + - gpr_test_util + - gpr - name: gpr_slice_test build: test language: c - src: [test/core/support/slice_test.c] - deps: [gpr_test_util, gpr] + src: + - test/core/support/slice_test.c + deps: + - gpr_test_util + - gpr - name: gpr_stack_lockfree_test build: test language: c - src: [test/core/support/stack_lockfree_test.c] - deps: [gpr_test_util, gpr] + src: + - test/core/support/stack_lockfree_test.c + deps: + - gpr_test_util + - gpr - name: gpr_string_test build: test language: c - src: [test/core/support/string_test.c] - deps: [gpr_test_util, gpr] + src: + - test/core/support/string_test.c + deps: + - gpr_test_util + - gpr - name: gpr_sync_test build: test language: c - src: [test/core/support/sync_test.c] - deps: [gpr_test_util, gpr] + src: + - test/core/support/sync_test.c + deps: + - gpr_test_util + - gpr - name: gpr_thd_test build: test language: c - src: [test/core/support/thd_test.c] - deps: [gpr_test_util, gpr] + src: + - test/core/support/thd_test.c + deps: + - gpr_test_util + - gpr - name: gpr_time_test build: test language: c - src: [test/core/support/time_test.c] - deps: [gpr_test_util, gpr] + src: + - test/core/support/time_test.c + deps: + - gpr_test_util + - gpr - name: gpr_tls_test build: test language: c - src: [test/core/support/tls_test.c] - deps: [gpr_test_util, gpr] + src: + - test/core/support/tls_test.c + deps: + - gpr_test_util + - gpr - name: gpr_useful_test build: test language: c - src: [test/core/support/useful_test.c] - deps: [gpr_test_util, gpr] + src: + - test/core/support/useful_test.c + deps: + - gpr_test_util + - gpr - name: grpc_auth_context_test build: test language: c - src: [test/core/security/auth_context_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/security/auth_context_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: grpc_base64_test build: test language: c - src: [test/core/security/base64_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/security/base64_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: grpc_byte_buffer_reader_test build: test language: c - src: [test/core/surface/byte_buffer_reader_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/surface/byte_buffer_reader_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: grpc_channel_args_test build: test language: c - src: [test/core/channel/channel_args_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/channel/channel_args_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: grpc_channel_stack_test build: test language: c - src: [test/core/channel/channel_stack_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/channel/channel_stack_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: grpc_completion_queue_test build: test language: c - src: [test/core/surface/completion_queue_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/surface/completion_queue_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: grpc_create_jwt build: tool language: c - src: [test/core/security/create_jwt.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/security/create_jwt.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: grpc_credentials_test build: test language: c - src: [test/core/security/credentials_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/security/credentials_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: grpc_fetch_oauth2 build: tool language: c - src: [test/core/security/fetch_oauth2.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/security/fetch_oauth2.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: grpc_json_token_test build: test language: c - src: [test/core/security/json_token_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] - platforms: [linux, posix, mac] + src: + - test/core/security/json_token_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr + platforms: + - linux + - posix + - mac - name: grpc_jwt_verifier_test build: test language: c - src: [test/core/security/jwt_verifier_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/security/jwt_verifier_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: grpc_print_google_default_creds_token build: tool language: c - src: [test/core/security/print_google_default_creds_token.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/security/print_google_default_creds_token.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: grpc_security_connector_test build: test language: c - src: [test/core/security/security_connector_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/security/security_connector_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: grpc_stream_op_test build: test language: c - src: [test/core/transport/stream_op_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/transport/stream_op_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: grpc_verify_jwt build: tool language: c - src: [test/core/security/verify_jwt.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/security/verify_jwt.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: hpack_parser_test build: test language: c - src: [test/core/transport/chttp2/hpack_parser_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/transport/chttp2/hpack_parser_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: hpack_table_test build: test language: c - src: [test/core/transport/chttp2/hpack_table_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/transport/chttp2/hpack_table_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: httpcli_format_request_test build: test language: c - src: [test/core/httpcli/format_request_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/httpcli/format_request_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: httpcli_parser_test build: test language: c - src: [test/core/httpcli/parser_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/httpcli/parser_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: httpcli_test build: test language: c - src: [test/core/httpcli/httpcli_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] - platforms: [mac, linux, posix] + src: + - test/core/httpcli/httpcli_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr + platforms: + - mac + - linux + - posix - name: json_rewrite build: test run: false language: c - src: [test/core/json/json_rewrite.c] - deps: [grpc, gpr] + src: + - test/core/json/json_rewrite.c + deps: + - grpc + - gpr - name: json_rewrite_test build: test language: c - src: [test/core/json/json_rewrite_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/json/json_rewrite_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: json_test build: test language: c - src: [test/core/json/json_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/json/json_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: lame_client_test build: test language: c - src: [test/core/surface/lame_client_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/surface/lame_client_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: lb_policies_test build: test language: c - src: [test/core/client_config/lb_policies_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/client_config/lb_policies_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: low_level_ping_pong_benchmark build: benchmark language: c - src: [test/core/network_benchmarks/low_level_ping_pong.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] - platforms: [mac, linux, posix] + src: + - test/core/network_benchmarks/low_level_ping_pong.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr + platforms: + - mac + - linux + - posix - name: message_compress_test build: test language: c - src: [test/core/compression/message_compress_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/compression/message_compress_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: multi_init_test build: test language: c - src: [test/core/surface/multi_init_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/surface/multi_init_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: multiple_server_queues_test build: test language: c - src: [test/core/end2end/multiple_server_queues_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/end2end/multiple_server_queues_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: murmur_hash_test build: test language: c - src: [test/core/support/murmur_hash_test.c] - deps: [gpr_test_util, gpr] + src: + - test/core/support/murmur_hash_test.c + deps: + - gpr_test_util + - gpr - name: no_server_test build: test language: c - src: [test/core/end2end/no_server_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/end2end/no_server_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: resolve_address_test build: test language: c - src: [test/core/iomgr/resolve_address_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/iomgr/resolve_address_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: secure_endpoint_test build: test language: c - src: [test/core/security/secure_endpoint_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/security/secure_endpoint_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: sockaddr_utils_test build: test language: c - src: [test/core/iomgr/sockaddr_utils_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/iomgr/sockaddr_utils_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: tcp_client_posix_test build: test language: c - src: [test/core/iomgr/tcp_client_posix_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] - platforms: [mac, linux, posix] + src: + - test/core/iomgr/tcp_client_posix_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr + platforms: + - mac + - linux + - posix - name: tcp_posix_test build: test language: c - src: [test/core/iomgr/tcp_posix_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] - platforms: [mac, linux, posix] + src: + - test/core/iomgr/tcp_posix_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr + platforms: + - mac + - linux + - posix - name: tcp_server_posix_test build: test language: c - src: [test/core/iomgr/tcp_server_posix_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] - platforms: [mac, linux, posix] + src: + - test/core/iomgr/tcp_server_posix_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr + platforms: + - mac + - linux + - posix - name: time_averaged_stats_test build: test language: c - src: [test/core/iomgr/time_averaged_stats_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/iomgr/time_averaged_stats_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: timeout_encoding_test build: test language: c - src: [test/core/transport/chttp2/timeout_encoding_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/transport/chttp2/timeout_encoding_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: timers_test build: test language: c - src: [test/core/profiling/timers_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/profiling/timers_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: transport_metadata_test build: test language: c - src: [test/core/transport/metadata_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/transport/metadata_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: transport_security_test build: test language: c - src: [test/core/tsi/transport_security_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] - platforms: [linux, posix, mac] + src: + - test/core/tsi/transport_security_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr + platforms: + - linux + - posix + - mac - name: udp_server_test build: test language: c - src: [test/core/iomgr/udp_server_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] - platforms: [mac, linux, posix] + src: + - test/core/iomgr/udp_server_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr + platforms: + - mac + - linux + - posix - name: uri_parser_test build: test language: c - src: [test/core/client_config/uri_parser_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] + src: + - test/core/client_config/uri_parser_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: workqueue_test build: test language: c - src: [test/core/iomgr/workqueue_test.c] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] - platforms: [mac, linux, posix] + src: + - test/core/iomgr/workqueue_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr + platforms: + - mac + - linux + - posix - name: async_end2end_test build: test language: c++ - src: [test/cpp/end2end/async_end2end_test.cc] - deps: [grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr] + src: + - test/cpp/end2end/async_end2end_test.cc + deps: + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr - name: async_streaming_ping_pong_test build: test language: c++ - src: [test/cpp/qps/async_streaming_ping_pong_test.cc] - deps: [qps, grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr] - platforms: [mac, linux, posix] + src: + - test/cpp/qps/async_streaming_ping_pong_test.cc + deps: + - qps + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + platforms: + - mac + - linux + - posix - name: async_unary_ping_pong_test build: test language: c++ - src: [test/cpp/qps/async_unary_ping_pong_test.cc] - deps: [qps, grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr] - platforms: [mac, linux, posix] + src: + - test/cpp/qps/async_unary_ping_pong_test.cc + deps: + - qps + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + platforms: + - mac + - linux + - posix - name: auth_property_iterator_test build: test language: c++ - src: [test/cpp/common/auth_property_iterator_test.cc] - deps: [grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr] + src: + - test/cpp/common/auth_property_iterator_test.cc + deps: + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr - name: channel_arguments_test build: test language: c++ - src: [test/cpp/client/channel_arguments_test.cc] - deps: [grpc++, grpc, gpr] + src: + - test/cpp/client/channel_arguments_test.cc + deps: + - grpc++ + - grpc + - gpr - name: cli_call_test build: test language: c++ - src: [test/cpp/util/cli_call_test.cc] - deps: [grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr] + src: + - test/cpp/util/cli_call_test.cc + deps: + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr - name: client_crash_test build: test language: c++ - src: [test/cpp/end2end/client_crash_test.cc] - deps: [grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr] - platforms: [mac, linux, posix] + src: + - test/cpp/end2end/client_crash_test.cc + deps: + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + platforms: + - mac + - linux + - posix - name: client_crash_test_server build: test run: false language: c++ - src: [test/cpp/end2end/client_crash_test_server.cc] - deps: [grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr] + src: + - test/cpp/end2end/client_crash_test_server.cc + deps: + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr - name: credentials_test build: test language: c++ - src: [test/cpp/client/credentials_test.cc] - deps: [grpc++, grpc, gpr] + src: + - test/cpp/client/credentials_test.cc + deps: + - grpc++ + - grpc + - gpr - name: cxx_byte_buffer_test build: test language: c++ - src: [test/cpp/util/byte_buffer_test.cc] - deps: [grpc_test_util, grpc++, grpc, gpr_test_util, gpr] + src: + - test/cpp/util/byte_buffer_test.cc + deps: + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr - name: cxx_slice_test build: test language: c++ - src: [test/cpp/util/slice_test.cc] - deps: [grpc_test_util, grpc++, grpc, gpr_test_util, gpr] + src: + - test/cpp/util/slice_test.cc + deps: + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr - name: cxx_string_ref_test build: test language: c++ - src: [test/cpp/util/string_ref_test.cc] - deps: [grpc++] + src: + - test/cpp/util/string_ref_test.cc + deps: + - grpc++ - name: cxx_time_test build: test language: c++ - src: [test/cpp/util/time_test.cc] - deps: [grpc_test_util, grpc++, grpc, gpr_test_util, gpr] + src: + - test/cpp/util/time_test.cc + deps: + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr - name: end2end_test build: test language: c++ - src: [test/cpp/end2end/end2end_test.cc] - deps: [grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr] + src: + - test/cpp/end2end/end2end_test.cc + deps: + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr - name: generic_end2end_test build: test language: c++ - src: [test/cpp/end2end/generic_end2end_test.cc] - deps: [grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr] + src: + - test/cpp/end2end/generic_end2end_test.cc + deps: + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr - name: grpc_cli build: test run: false language: c++ - src: [test/cpp/util/grpc_cli.cc] - deps: [grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr, grpc++_test_config] + src: + - test/cpp/util/grpc_cli.cc + deps: + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + - grpc++_test_config - name: grpc_cpp_plugin build: protoc language: c++ - src: [src/compiler/cpp_plugin.cc] - deps: [grpc_plugin_support] + src: + - src/compiler/cpp_plugin.cc + deps: + - grpc_plugin_support secure: false vs_config_type: Application vs_project_guid: '{7E51A25F-AC59-488F-906C-C60FAAE706AA}' - name: grpc_csharp_plugin build: protoc language: c++ - src: [src/compiler/csharp_plugin.cc] - deps: [grpc_plugin_support] + src: + - src/compiler/csharp_plugin.cc + deps: + - grpc_plugin_support secure: false vs_config_type: Application vs_project_guid: '{3C813052-A49A-4662-B90A-1ADBEC7EE453}' - name: grpc_objective_c_plugin build: protoc language: c++ - src: [src/compiler/objective_c_plugin.cc] - deps: [grpc_plugin_support] + src: + - src/compiler/objective_c_plugin.cc + deps: + - grpc_plugin_support secure: false vs_config_type: Application vs_project_guid: '{19564640-CEE6-4921-ABA5-676ED79A36F6}' - name: grpc_python_plugin build: protoc language: c++ - src: [src/compiler/python_plugin.cc] - deps: [grpc_plugin_support] + src: + - src/compiler/python_plugin.cc + deps: + - grpc_plugin_support secure: false vs_config_type: Application vs_project_guid: '{DF52D501-A6CF-4E6F-BA38-6EBE2E8DAFB2}' - name: grpc_ruby_plugin build: protoc language: c++ - src: [src/compiler/ruby_plugin.cc] - deps: [grpc_plugin_support] + src: + - src/compiler/ruby_plugin.cc + deps: + - grpc_plugin_support secure: false vs_config_type: Application vs_project_guid: '{069E9D05-B78B-4751-9252-D21EBAE7DE8E}' @@ -884,139 +1828,346 @@ targets: run: false language: c++ src: [] - deps: [interop_client_main, interop_client_helper, grpc++_test_util, grpc_test_util, - grpc++, grpc, gpr_test_util, gpr, grpc++_test_config] - platforms: [mac, linux, posix] + deps: + - interop_client_main + - interop_client_helper + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + - grpc++_test_config + platforms: + - mac + - linux + - posix - name: interop_server build: test run: false language: c++ src: [] - deps: [interop_server_main, interop_server_helper, grpc++_test_util, grpc_test_util, - grpc++, grpc, gpr_test_util, gpr, grpc++_test_config] - platforms: [mac, linux, posix] + deps: + - interop_server_main + - interop_server_helper + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + - grpc++_test_config + platforms: + - mac + - linux + - posix - name: interop_test build: test language: c++ - src: [test/cpp/interop/interop_test.cc] - deps: [grpc_test_util, grpc, gpr_test_util, gpr] - platforms: [mac, linux, posix] + src: + - test/cpp/interop/interop_test.cc + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr + platforms: + - mac + - linux + - posix - name: mock_test build: test language: c++ - src: [test/cpp/end2end/mock_test.cc] - deps: [grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr] + src: + - test/cpp/end2end/mock_test.cc + deps: + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr - name: qps_driver build: benchmark language: c++ - src: [test/cpp/qps/qps_driver.cc] - deps: [qps, grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr, - grpc++_test_config] + src: + - test/cpp/qps/qps_driver.cc + deps: + - qps + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + - grpc++_test_config - name: qps_interarrival_test build: test run: false language: c++ - src: [test/cpp/qps/qps_interarrival_test.cc] - deps: [qps, grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr] - platforms: [mac, linux, posix] + src: + - test/cpp/qps/qps_interarrival_test.cc + deps: + - qps + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + platforms: + - mac + - linux + - posix - name: qps_openloop_test build: test language: c++ - src: [test/cpp/qps/qps_openloop_test.cc] - deps: [qps, grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr, - grpc++_test_config] - platforms: [mac, linux, posix] + src: + - test/cpp/qps/qps_openloop_test.cc + deps: + - qps + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + - grpc++_test_config + platforms: + - mac + - linux + - posix - name: qps_test build: test language: c++ - src: [test/cpp/qps/qps_test.cc] - deps: [qps, grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr, - grpc++_test_config] - exclude_configs: [tsan] - platforms: [mac, linux, posix] + src: + - test/cpp/qps/qps_test.cc + deps: + - qps + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + - grpc++_test_config + exclude_configs: + - tsan + platforms: + - mac + - linux + - posix - name: qps_worker build: benchmark language: c++ - headers: [test/cpp/qps/client.h, test/cpp/qps/server.h] - src: [test/cpp/qps/worker.cc] - deps: [qps, grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr, - grpc++_test_config] + headers: + - test/cpp/qps/client.h + - test/cpp/qps/server.h + src: + - test/cpp/qps/worker.cc + deps: + - qps + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + - grpc++_test_config - name: reconnect_interop_client build: test run: false language: c++ - src: [test/proto/empty.proto, test/proto/messages.proto, test/proto/test.proto, - test/cpp/interop/reconnect_interop_client.cc] - deps: [grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr, grpc++_test_config] + src: + - test/proto/empty.proto + - test/proto/messages.proto + - test/proto/test.proto + - test/cpp/interop/reconnect_interop_client.cc + deps: + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + - grpc++_test_config - name: reconnect_interop_server build: test run: false language: c++ - src: [test/proto/empty.proto, test/proto/messages.proto, test/proto/test.proto, - test/cpp/interop/reconnect_interop_server.cc] - deps: [reconnect_server, grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, - gpr, grpc++_test_config] + src: + - test/proto/empty.proto + - test/proto/messages.proto + - test/proto/test.proto + - test/cpp/interop/reconnect_interop_server.cc + deps: + - reconnect_server + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + - grpc++_test_config - name: secure_auth_context_test build: test language: c++ - src: [test/cpp/common/secure_auth_context_test.cc] - deps: [grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr] + src: + - test/cpp/common/secure_auth_context_test.cc + deps: + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr - name: server_crash_test build: test language: c++ - src: [test/cpp/end2end/server_crash_test.cc] - deps: [grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr] - platforms: [mac, linux, posix] + src: + - test/cpp/end2end/server_crash_test.cc + deps: + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + platforms: + - mac + - linux + - posix - name: server_crash_test_client build: test run: false language: c++ - src: [test/cpp/end2end/server_crash_test_client.cc] - deps: [grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr] + src: + - test/cpp/end2end/server_crash_test_client.cc + deps: + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr - name: shutdown_test build: test language: c++ - src: [test/cpp/end2end/shutdown_test.cc] - deps: [grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr] + src: + - test/cpp/end2end/shutdown_test.cc + deps: + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr - name: status_test build: test language: c++ - src: [test/cpp/util/status_test.cc] - deps: [grpc_test_util, grpc++, grpc, gpr_test_util, gpr] + src: + - test/cpp/util/status_test.cc + deps: + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr - name: streaming_throughput_test build: test language: c++ - src: [test/cpp/end2end/streaming_throughput_test.cc] - deps: [grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr] - platforms: [mac, linux, posix] + src: + - test/cpp/end2end/streaming_throughput_test.cc + deps: + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + platforms: + - mac + - linux + - posix - name: sync_streaming_ping_pong_test build: test language: c++ - src: [test/cpp/qps/sync_streaming_ping_pong_test.cc] - deps: [qps, grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr] - platforms: [mac, linux, posix] + src: + - test/cpp/qps/sync_streaming_ping_pong_test.cc + deps: + - qps + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + platforms: + - mac + - linux + - posix - name: sync_unary_ping_pong_test build: test language: c++ - src: [test/cpp/qps/sync_unary_ping_pong_test.cc] - deps: [qps, grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr] - platforms: [mac, linux, posix] + src: + - test/cpp/qps/sync_unary_ping_pong_test.cc + deps: + - qps + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + platforms: + - mac + - linux + - posix - name: thread_stress_test build: test language: c++ - src: [test/cpp/end2end/thread_stress_test.cc] - deps: [grpc++_test_util, grpc_test_util, grpc++, grpc, gpr_test_util, gpr] + src: + - test/cpp/end2end/thread_stress_test.cc + deps: + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr - name: zookeeper_test build: test run: false language: c++ - src: [test/cpp/end2end/zookeeper_test.cc] - deps: [grpc++_test_util, grpc_test_util, grpc++, grpc_zookeeper, grpc, gpr_test_util, - gpr] - external_deps: [zookeeper] - platforms: [linux] + src: + - test/cpp/end2end/zookeeper_test.cc + deps: + - grpc++_test_util + - grpc_test_util + - grpc++ + - grpc_zookeeper + - grpc + - gpr_test_util + - gpr + external_deps: + - zookeeper + platforms: + - linux vspackages: -- {linkage: static, name: grpc.dependencies.zlib, props: false, redist: true, version: 1.2.8.9} -- {name: grpc.dependencies.openssl, props: true, redist: true, version: 1.0.2.3} -- {name: gflags, props: false, redist: false, version: 2.1.2.1} -- {name: gtest, props: false, redist: false, version: 1.7.0.1} +- linkage: static + name: grpc.dependencies.zlib + props: false + redist: true + version: 1.2.8.9 +- name: grpc.dependencies.openssl + props: true + redist: true + version: 1.0.2.3 +- name: gflags + props: false + redist: false + version: 2.1.2.1 +- name: gtest + props: false + redist: false + version: 1.7.0.1 diff --git a/tools/buildgen/build-cleaner.py b/tools/buildgen/build-cleaner.py index 6c9c6b3749e..8288a8998d2 100755 --- a/tools/buildgen/build-cleaner.py +++ b/tools/buildgen/build-cleaner.py @@ -88,7 +88,7 @@ for filename in sys.argv[1:]: if grp not in js: continue js[grp] = sorted([clean_elem(x) for x in js[grp]], key=lambda x: (x.get('language', '_'), x['name'])) - output = yaml.dump(js, indent=2, width=80) + output = yaml.dump(js, indent=2, width=80, default_flow_style=False) # massage out trailing whitespace lines = [] for line in output.splitlines(): From 6d19724a9008919eb1ef0a01790a2fa6299ac9d8 Mon Sep 17 00:00:00 2001 From: Nate Kibler Date: Fri, 25 Sep 2015 10:02:20 -0700 Subject: [PATCH 11/48] Adds class factory method to generated ProtoService classes --- src/compiler/objective_c_generator.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/compiler/objective_c_generator.cc b/src/compiler/objective_c_generator.cc index a3157db0fb4..9eed78c9618 100644 --- a/src/compiler/objective_c_generator.cc +++ b/src/compiler/objective_c_generator.cc @@ -203,6 +203,7 @@ void PrintMethodImplementations(Printer *printer, printer.Print( "- (instancetype)initWithHost:(NSString *)host" " NS_DESIGNATED_INITIALIZER;\n"); + printer.Print("+ (instancetype)serviceWithHost:(NSString *)host;\n"); printer.Print("@end\n"); } return output; @@ -239,6 +240,10 @@ void PrintMethodImplementations(Printer *printer, printer.Print(" packageName:(NSString *)packageName\n"); printer.Print(" serviceName:(NSString *)serviceName {\n"); printer.Print(" return [self initWithHost:host];\n"); + printer.Print("}\n\n"); + printer.Print("// Class factory method\n"); + printer.Print("+ (instancetype)serviceWithHost:(NSString *)host {\n"); + printer.Print(" return [[self alloc] initWithHost:host];\n"); printer.Print("}\n\n\n"); for (int i = 0; i < service->method_count(); i++) { From d7e09c31e66c80e2d67179441f1e9ec43301645c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 25 Sep 2015 11:33:39 -0700 Subject: [PATCH 12/48] Purple flakes --- tools/run_tests/jobset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index e696a0e9698..6ddc4d29bc5 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -72,6 +72,7 @@ _COLORS = { 'yellow': [ 33, 0 ], 'lightgray': [ 37, 0], 'gray': [ 30, 1 ], + 'purple': [ 35, 0 ], } @@ -81,7 +82,7 @@ _CLEAR_LINE = '\x1b[2K' _TAG_COLOR = { 'FAILED': 'red', - 'FLAKE': 'red', + 'FLAKE': 'purple', 'WARNING': 'yellow', 'TIMEOUT': 'red', 'PASSED': 'green', From 3dc1e4f909abc98a59ab1df091b5d656b3e7bcc2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 25 Sep 2015 11:46:56 -0700 Subject: [PATCH 13/48] Add support for flaky timeouts --- tools/run_tests/jobset.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index 6ddc4d29bc5..f6fdc805953 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -83,6 +83,7 @@ _CLEAR_LINE = '\x1b[2K' _TAG_COLOR = { 'FAILED': 'red', 'FLAKE': 'purple', + 'TIMEOUT_FLAKE': 'purple', 'WARNING': 'yellow', 'TIMEOUT': 'red', 'PASSED': 'green', @@ -231,11 +232,17 @@ class Job(object): self._tempfile.seek(0) stdout = self._tempfile.read() filtered_stdout = filter(lambda x: x in string.printable, stdout.decode(errors='ignore')) - message('TIMEOUT', self._spec.shortname, stdout, do_newline=True) - self.kill() - if self._xml_test is not None: - ET.SubElement(self._xml_test, 'system-out').text = filtered_stdout - ET.SubElement(self._xml_test, 'error', message='Timeout') + if self._retries < self._spec.flake_retries: + message('TIMEOUT_FLAKE', self._spec.shortname, stdout, do_newline=True) + self._retries += 1 + self._process.terminate() + self.start() + else: + message('TIMEOUT', self._spec.shortname, stdout, do_newline=True) + self.kill() + if self._xml_test is not None: + ET.SubElement(self._xml_test, 'system-out').text = filtered_stdout + ET.SubElement(self._xml_test, 'error', message='Timeout') return self._state def kill(self): From 2d32771a1b82400877d1aefe01c82705a3e10a7b Mon Sep 17 00:00:00 2001 From: Nate Kibler Date: Fri, 25 Sep 2015 11:56:05 -0700 Subject: [PATCH 14/48] Removed comment from code generation and updated tests --- src/compiler/objective_c_generator.cc | 1 - src/objective-c/tests/InteropTests.m | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/compiler/objective_c_generator.cc b/src/compiler/objective_c_generator.cc index 9eed78c9618..ff092053ad0 100644 --- a/src/compiler/objective_c_generator.cc +++ b/src/compiler/objective_c_generator.cc @@ -241,7 +241,6 @@ void PrintMethodImplementations(Printer *printer, printer.Print(" serviceName:(NSString *)serviceName {\n"); printer.Print(" return [self initWithHost:host];\n"); printer.Print("}\n\n"); - printer.Print("// Class factory method\n"); printer.Print("+ (instancetype)serviceWithHost:(NSString *)host {\n"); printer.Print(" return [[self alloc] initWithHost:host];\n"); printer.Print("}\n\n\n"); diff --git a/src/objective-c/tests/InteropTests.m b/src/objective-c/tests/InteropTests.m index 1b63fe2059c..af58e2bd048 100644 --- a/src/objective-c/tests/InteropTests.m +++ b/src/objective-c/tests/InteropTests.m @@ -89,7 +89,7 @@ static NSString * const kRemoteSSLHost = @"grpc-test.sandbox.google.com"; } - (void)setUp { - _service = [[RMTTestService alloc] initWithHost:self.class.host]; + _service = [RMTTestService serviceWithHost:self.class.host]; } - (void)testEmptyUnaryRPC { @@ -274,17 +274,17 @@ static NSString * const kRemoteSSLHost = @"grpc-test.sandbox.google.com"; - (void)testCancelAfterFirstResponseRPC { __weak XCTestExpectation *expectation = [self expectationWithDescription:@"CancelAfterFirstResponse"]; - + // A buffered pipe to which we write a single value but never close GRXBufferedPipe *requestsBuffer = [[GRXBufferedPipe alloc] init]; - + __block BOOL receivedResponse = NO; - + id request = [RMTStreamingOutputCallRequest messageWithPayloadSize:@21782 requestedResponseSize:@31415]; - + [requestsBuffer writeValue:request]; - + __block ProtoRPC *call = [_service RPCToFullDuplexCallWithRequestsWriter:requestsBuffer eventHandler:^(BOOL done, From 31fdaa4aa569eb4470c5e65d834165c3e4ab48c2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 25 Sep 2015 13:09:59 -0700 Subject: [PATCH 15/48] Retry checking port_server on socket timeout errors --- tools/run_tests/run_tests.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index d271137ca12..3c2b2a8b893 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -736,6 +736,10 @@ def _start_port_server(port_server_port): urllib2.urlopen('http://localhost:%d/get' % port_server_port, timeout=1).read() break + except socket.timeout: + print "waiting for port_server" + time.sleep(0.5) + waits += 1 except urllib2.URLError: print "waiting for port_server" time.sleep(0.5) From 8287523dafd1697d29d0abf942b097fd3ca0c56f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 25 Sep 2015 13:57:34 -0700 Subject: [PATCH 16/48] Add missing import --- tools/run_tests/run_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 3c2b2a8b893..52d753729f4 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -40,6 +40,7 @@ import os import platform import random import re +import socket import subprocess import sys import time From 4dbdd6a5c2adca0e91f5a71a6f23385a2991ae1e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 25 Sep 2015 15:12:16 -0700 Subject: [PATCH 17/48] Remove metadata canonicalization, and fix some bugs --- src/core/census/grpc_filter.c | 2 +- src/core/channel/compress_filter.c | 10 ++--- src/core/channel/http_client_filter.c | 4 +- src/core/channel/http_server_filter.c | 6 +-- src/core/security/client_auth_filter.c | 10 ++--- src/core/surface/call.c | 6 +-- src/core/surface/channel.c | 25 ++++++------ src/core/surface/server.c | 9 ++--- src/core/transport/chttp2/stream_encoder.c | 4 +- src/core/transport/chttp2_transport.c | 2 +- src/core/transport/metadata.c | 44 +++------------------- src/core/transport/metadata.h | 6 +-- test/core/end2end/tests/metadata.c | 4 +- test/core/transport/metadata_test.c | 16 ++++---- 14 files changed, 55 insertions(+), 93 deletions(-) diff --git a/src/core/census/grpc_filter.c b/src/core/census/grpc_filter.c index 1830af3feba..3545307b666 100644 --- a/src/core/census/grpc_filter.c +++ b/src/core/census/grpc_filter.c @@ -168,7 +168,7 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx, int is_first, int is_last) { channel_data *chand = elem->channel_data; GPR_ASSERT(chand != NULL); - chand->path_str = grpc_mdstr_from_string(mdctx, ":path", 0); + chand->path_str = grpc_mdstr_from_string(mdctx, ":path"); } static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, diff --git a/src/core/channel/compress_filter.c b/src/core/channel/compress_filter.c index f8dbe8c8176..2aa826fcd87 100644 --- a/src/core/channel/compress_filter.c +++ b/src/core/channel/compress_filter.c @@ -331,13 +331,13 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx, channeld->default_compression_algorithm; channeld->mdstr_request_compression_algorithm_key = - grpc_mdstr_from_string(mdctx, GRPC_COMPRESS_REQUEST_ALGORITHM_KEY, 0); + grpc_mdstr_from_string(mdctx, GRPC_COMPRESS_REQUEST_ALGORITHM_KEY); channeld->mdstr_outgoing_compression_algorithm_key = - grpc_mdstr_from_string(mdctx, "grpc-encoding", 0); + grpc_mdstr_from_string(mdctx, "grpc-encoding"); channeld->mdstr_compression_capabilities_key = - grpc_mdstr_from_string(mdctx, "grpc-accept-encoding", 0); + grpc_mdstr_from_string(mdctx, "grpc-accept-encoding"); for (algo_idx = 0; algo_idx < GRPC_COMPRESS_ALGORITHMS_COUNT; ++algo_idx) { char *algorithm_name; @@ -351,7 +351,7 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx, grpc_mdelem_from_metadata_strings( mdctx, GRPC_MDSTR_REF(channeld->mdstr_outgoing_compression_algorithm_key), - grpc_mdstr_from_string(mdctx, algorithm_name, 0)); + grpc_mdstr_from_string(mdctx, algorithm_name)); if (algo_idx > 0) { supported_algorithms_names[supported_algorithms_idx++] = algorithm_name; } @@ -365,7 +365,7 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx, channeld->mdelem_accept_encoding = grpc_mdelem_from_metadata_strings( mdctx, GRPC_MDSTR_REF(channeld->mdstr_compression_capabilities_key), - grpc_mdstr_from_string(mdctx, accept_encoding_str, 0)); + grpc_mdstr_from_string(mdctx, accept_encoding_str)); gpr_free(accept_encoding_str); GPR_ASSERT(!is_last); diff --git a/src/core/channel/http_client_filter.c b/src/core/channel/http_client_filter.c index da33c956f75..d67dc37ad22 100644 --- a/src/core/channel/http_client_filter.c +++ b/src/core/channel/http_client_filter.c @@ -239,7 +239,7 @@ static grpc_mdstr *user_agent_from_args(grpc_mdctx *mdctx, tmp = gpr_strvec_flatten(&v, NULL); gpr_strvec_destroy(&v); - result = grpc_mdstr_from_string(mdctx, tmp, 0); + result = grpc_mdstr_from_string(mdctx, tmp); gpr_free(tmp); return result; @@ -267,7 +267,7 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx, grpc_mdelem_from_strings(mdctx, "content-type", "application/grpc"); channeld->status = grpc_mdelem_from_strings(mdctx, ":status", "200"); channeld->user_agent = grpc_mdelem_from_metadata_strings( - mdctx, grpc_mdstr_from_string(mdctx, "user-agent", 0), + mdctx, grpc_mdstr_from_string(mdctx, "user-agent"), user_agent_from_args(mdctx, channel_args)); } diff --git a/src/core/channel/http_server_filter.c b/src/core/channel/http_server_filter.c index 549c1da580b..5e6d684a528 100644 --- a/src/core/channel/http_server_filter.c +++ b/src/core/channel/http_server_filter.c @@ -273,9 +273,9 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx, channeld->http_scheme = grpc_mdelem_from_strings(mdctx, ":scheme", "http"); channeld->https_scheme = grpc_mdelem_from_strings(mdctx, ":scheme", "https"); channeld->grpc_scheme = grpc_mdelem_from_strings(mdctx, ":scheme", "grpc"); - channeld->path_key = grpc_mdstr_from_string(mdctx, ":path", 0); - channeld->authority_key = grpc_mdstr_from_string(mdctx, ":authority", 0); - channeld->host_key = grpc_mdstr_from_string(mdctx, "host", 0); + channeld->path_key = grpc_mdstr_from_string(mdctx, ":path"); + channeld->authority_key = grpc_mdstr_from_string(mdctx, ":authority"); + channeld->host_key = grpc_mdstr_from_string(mdctx, "host"); channeld->content_type = grpc_mdelem_from_strings(mdctx, "content-type", "application/grpc"); diff --git a/src/core/security/client_auth_filter.c b/src/core/security/client_auth_filter.c index c152a06498d..3f4c084ffcd 100644 --- a/src/core/security/client_auth_filter.c +++ b/src/core/security/client_auth_filter.c @@ -326,12 +326,10 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx, (grpc_channel_security_connector *)GRPC_SECURITY_CONNECTOR_REF( sc, "client_auth_filter"); chand->md_ctx = metadata_context; - chand->authority_string = - grpc_mdstr_from_string(chand->md_ctx, ":authority", 0); - chand->path_string = grpc_mdstr_from_string(chand->md_ctx, ":path", 0); - chand->error_msg_key = - grpc_mdstr_from_string(chand->md_ctx, "grpc-message", 0); - chand->status_key = grpc_mdstr_from_string(chand->md_ctx, "grpc-status", 0); + chand->authority_string = grpc_mdstr_from_string(chand->md_ctx, ":authority"); + chand->path_string = grpc_mdstr_from_string(chand->md_ctx, ":path"); + chand->error_msg_key = grpc_mdstr_from_string(chand->md_ctx, "grpc-message"); + chand->status_key = grpc_mdstr_from_string(chand->md_ctx, "grpc-status"); } /* Destructor for channel data */ diff --git a/src/core/surface/call.c b/src/core/surface/call.c index 51dcbeb1a15..0b917f15611 100644 --- a/src/core/surface/call.c +++ b/src/core/surface/call.c @@ -1030,7 +1030,7 @@ static int prepare_application_metadata(grpc_call *call, size_t count, GPR_ASSERT(sizeof(grpc_linked_mdelem) == sizeof(md->internal_data)); l->md = grpc_mdelem_from_string_and_buffer(call->metadata_context, md->key, (const gpr_uint8 *)md->value, - md->value_length, 1); + md->value_length); if (!grpc_mdstr_is_legal_header(l->md->key)) { gpr_log(GPR_ERROR, "attempt to send invalid metadata key: %s", grpc_mdstr_as_c_string(l->md->key)); @@ -1330,7 +1330,7 @@ grpc_call_error grpc_call_cancel_with_status(grpc_call *c, static grpc_call_error cancel_with_status(grpc_call *c, grpc_status_code status, const char *description) { grpc_mdstr *details = - description ? grpc_mdstr_from_string(c->metadata_context, description, 0) + description ? grpc_mdstr_from_string(c->metadata_context, description) : NULL; GPR_ASSERT(status != GRPC_STATUS_OK); @@ -1689,7 +1689,7 @@ grpc_call_error grpc_call_start_batch(grpc_call *call, const grpc_op *ops, op->data.send_status_from_server.status_details != NULL ? grpc_mdstr_from_string( call->metadata_context, - op->data.send_status_from_server.status_details, 0) + op->data.send_status_from_server.status_details) : NULL; req = &reqs[out++]; if (out > GRPC_IOREQ_OP_COUNT) { diff --git a/src/core/surface/channel.c b/src/core/surface/channel.c index 08d003daca6..aed7a79e4ef 100644 --- a/src/core/surface/channel.c +++ b/src/core/surface/channel.c @@ -104,22 +104,21 @@ grpc_channel *grpc_channel_create_from_filters( /* decremented by grpc_channel_destroy */ gpr_ref_init(&channel->refs, 1); channel->metadata_context = mdctx; - channel->grpc_status_string = grpc_mdstr_from_string(mdctx, "grpc-status", 0); + channel->grpc_status_string = grpc_mdstr_from_string(mdctx, "grpc-status"); channel->grpc_compression_algorithm_string = - grpc_mdstr_from_string(mdctx, "grpc-encoding", 0); + grpc_mdstr_from_string(mdctx, "grpc-encoding"); channel->grpc_encodings_accepted_by_peer_string = - grpc_mdstr_from_string(mdctx, "grpc-accept-encoding", 0); - channel->grpc_message_string = - grpc_mdstr_from_string(mdctx, "grpc-message", 0); + grpc_mdstr_from_string(mdctx, "grpc-accept-encoding"); + channel->grpc_message_string = grpc_mdstr_from_string(mdctx, "grpc-message"); for (i = 0; i < NUM_CACHED_STATUS_ELEMS; i++) { char buf[GPR_LTOA_MIN_BUFSIZE]; gpr_ltoa((long)i, buf); channel->grpc_status_elem[i] = grpc_mdelem_from_metadata_strings( mdctx, GRPC_MDSTR_REF(channel->grpc_status_string), - grpc_mdstr_from_string(mdctx, buf, 0)); + grpc_mdstr_from_string(mdctx, buf)); } - channel->path_string = grpc_mdstr_from_string(mdctx, ":path", 0); - channel->authority_string = grpc_mdstr_from_string(mdctx, ":authority", 0); + channel->path_string = grpc_mdstr_from_string(mdctx, ":path"); + channel->authority_string = grpc_mdstr_from_string(mdctx, ":authority"); gpr_mu_init(&channel->registered_call_mu); channel->registered_calls = NULL; @@ -219,11 +218,11 @@ grpc_call *grpc_channel_create_call(grpc_channel *channel, channel, parent_call, propagation_mask, cq, grpc_mdelem_from_metadata_strings( channel->metadata_context, GRPC_MDSTR_REF(channel->path_string), - grpc_mdstr_from_string(channel->metadata_context, method, 0)), + grpc_mdstr_from_string(channel->metadata_context, method)), host ? grpc_mdelem_from_metadata_strings( channel->metadata_context, GRPC_MDSTR_REF(channel->authority_string), - grpc_mdstr_from_string(channel->metadata_context, host, 0)) + grpc_mdstr_from_string(channel->metadata_context, host)) : NULL, deadline); } @@ -234,12 +233,12 @@ void *grpc_channel_register_call(grpc_channel *channel, const char *method, GPR_ASSERT(!reserved); rc->path = grpc_mdelem_from_metadata_strings( channel->metadata_context, GRPC_MDSTR_REF(channel->path_string), - grpc_mdstr_from_string(channel->metadata_context, method, 0)); + grpc_mdstr_from_string(channel->metadata_context, method)); rc->authority = host ? grpc_mdelem_from_metadata_strings( channel->metadata_context, GRPC_MDSTR_REF(channel->authority_string), - grpc_mdstr_from_string(channel->metadata_context, host, 0)) + grpc_mdstr_from_string(channel->metadata_context, host)) : NULL; gpr_mu_lock(&channel->registered_call_mu); rc->next = channel->registered_calls; @@ -358,7 +357,7 @@ grpc_mdelem *grpc_channel_get_reffed_status_elem(grpc_channel *channel, int i) { gpr_ltoa(i, tmp); return grpc_mdelem_from_metadata_strings( channel->metadata_context, GRPC_MDSTR_REF(channel->grpc_status_string), - grpc_mdstr_from_string(channel->metadata_context, tmp, 0)); + grpc_mdstr_from_string(channel->metadata_context, tmp)); } } diff --git a/src/core/surface/server.c b/src/core/surface/server.c index 27fc0945f9f..41191ebf26b 100644 --- a/src/core/surface/server.c +++ b/src/core/surface/server.c @@ -729,9 +729,8 @@ static void init_channel_elem(grpc_exec_ctx *exec_ctx, GPR_ASSERT(!is_last); chand->server = NULL; chand->channel = NULL; - chand->path_key = grpc_mdstr_from_string(metadata_context, ":path", 0); - chand->authority_key = - grpc_mdstr_from_string(metadata_context, ":authority", 0); + chand->path_key = grpc_mdstr_from_string(metadata_context, ":path"); + chand->authority_key = grpc_mdstr_from_string(metadata_context, ":authority"); chand->next = chand->prev = chand; chand->registered_methods = NULL; chand->connectivity_state = GRPC_CHANNEL_IDLE; @@ -959,8 +958,8 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s, chand->registered_methods = gpr_malloc(alloc); memset(chand->registered_methods, 0, alloc); for (rm = s->registered_methods; rm; rm = rm->next) { - host = rm->host ? grpc_mdstr_from_string(mdctx, rm->host, 0) : NULL; - method = grpc_mdstr_from_string(mdctx, rm->method, 0); + host = rm->host ? grpc_mdstr_from_string(mdctx, rm->host) : NULL; + method = grpc_mdstr_from_string(mdctx, rm->method); hash = GRPC_MDSTR_KV_HASH(host ? host->hash : 0, method->hash); for (probes = 0; chand->registered_methods[(hash + probes) % slots] .server_registered_method != NULL; diff --git a/src/core/transport/chttp2/stream_encoder.c b/src/core/transport/chttp2/stream_encoder.c index eb02ccdec37..b1f1db05d26 100644 --- a/src/core/transport/chttp2/stream_encoder.c +++ b/src/core/transport/chttp2/stream_encoder.c @@ -466,7 +466,7 @@ static void deadline_enc(grpc_chttp2_hpack_compressor *c, gpr_timespec deadline, gpr_time_sub(deadline, gpr_now(deadline.clock_type)), timeout_str); mdelem = grpc_mdelem_from_metadata_strings( c->mdctx, GRPC_MDSTR_REF(c->timeout_key_str), - grpc_mdstr_from_string(c->mdctx, timeout_str, 0)); + grpc_mdstr_from_string(c->mdctx, timeout_str)); mdelem = hpack_enc(c, mdelem, st); if (mdelem) GRPC_MDELEM_UNREF(mdelem); } @@ -481,7 +481,7 @@ void grpc_chttp2_hpack_compressor_init(grpc_chttp2_hpack_compressor *c, grpc_mdctx *ctx) { memset(c, 0, sizeof(*c)); c->mdctx = ctx; - c->timeout_key_str = grpc_mdstr_from_string(ctx, "grpc-timeout", 0); + c->timeout_key_str = grpc_mdstr_from_string(ctx, "grpc-timeout"); } void grpc_chttp2_hpack_compressor_destroy(grpc_chttp2_hpack_compressor *c) { diff --git a/src/core/transport/chttp2_transport.c b/src/core/transport/chttp2_transport.c index 1679e4345d8..0202d53abdb 100644 --- a/src/core/transport/chttp2_transport.c +++ b/src/core/transport/chttp2_transport.c @@ -243,7 +243,7 @@ static void init_transport(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->global.pings.next = t->global.pings.prev = &t->global.pings; t->parsing.is_client = is_client; t->parsing.str_grpc_timeout = - grpc_mdstr_from_string(t->metadata_context, "grpc-timeout", 0); + grpc_mdstr_from_string(t->metadata_context, "grpc-timeout"); t->parsing.deframe_state = is_client ? GRPC_DTS_FH_0 : GRPC_DTS_CLIENT_PREFIX_0; t->writing.is_client = is_client; diff --git a/src/core/transport/metadata.c b/src/core/transport/metadata.c index 9d135f43560..3dbb9f0b539 100644 --- a/src/core/transport/metadata.c +++ b/src/core/transport/metadata.c @@ -312,38 +312,7 @@ static void slice_unref(void *p) { unlock(ctx); } -grpc_mdstr *grpc_mdstr_from_string(grpc_mdctx *ctx, const char *str, - int canonicalize_key) { - if (canonicalize_key) { - size_t len; - size_t i; - int canonical = 1; - - for (i = 0; str[i]; i++) { - if (str[i] >= 'A' && str[i] <= 'Z') { - canonical = 0; - /* Keep going in loop just to get string length */ - } - } - len = i; - - if (canonical) { - return grpc_mdstr_from_buffer(ctx, (const gpr_uint8 *)str, len); - } else { - char *copy = gpr_malloc(len); - grpc_mdstr *ret; - for (i = 0; i < len; i++) { - if (str[i] >= 'A' && str[i] <= 'Z') { - copy[i] = (char)(str[i] - 'A' + 'a'); - } else { - copy[i] = str[i]; - } - } - ret = grpc_mdstr_from_buffer(ctx, (const gpr_uint8 *)copy, len); - gpr_free(copy); - return ret; - } - } +grpc_mdstr *grpc_mdstr_from_string(grpc_mdctx *ctx, const char *str) { return grpc_mdstr_from_buffer(ctx, (const gpr_uint8 *)str, strlen(str)); } @@ -524,9 +493,9 @@ grpc_mdelem *grpc_mdelem_from_metadata_strings(grpc_mdctx *ctx, grpc_mdelem *grpc_mdelem_from_strings(grpc_mdctx *ctx, const char *key, const char *value) { - return grpc_mdelem_from_metadata_strings( - ctx, grpc_mdstr_from_string(ctx, key, 0), - grpc_mdstr_from_string(ctx, value, 0)); + return grpc_mdelem_from_metadata_strings(ctx, + grpc_mdstr_from_string(ctx, key), + grpc_mdstr_from_string(ctx, value)); } grpc_mdelem *grpc_mdelem_from_slices(grpc_mdctx *ctx, gpr_slice key, @@ -538,10 +507,9 @@ grpc_mdelem *grpc_mdelem_from_slices(grpc_mdctx *ctx, gpr_slice key, grpc_mdelem *grpc_mdelem_from_string_and_buffer(grpc_mdctx *ctx, const char *key, const gpr_uint8 *value, - size_t value_length, - int canonicalize_key) { + size_t value_length) { return grpc_mdelem_from_metadata_strings( - ctx, grpc_mdstr_from_string(ctx, key, canonicalize_key), + ctx, grpc_mdstr_from_string(ctx, key), grpc_mdstr_from_buffer(ctx, value, value_length)); } diff --git a/src/core/transport/metadata.h b/src/core/transport/metadata.h index eb17747be74..136a65f288e 100644 --- a/src/core/transport/metadata.h +++ b/src/core/transport/metadata.h @@ -95,8 +95,7 @@ size_t grpc_mdctx_get_mdtab_free_test_only(grpc_mdctx *mdctx); /* Constructors for grpc_mdstr instances; take a variety of data types that clients may have handy */ -grpc_mdstr *grpc_mdstr_from_string(grpc_mdctx *ctx, const char *str, - int perform_key_canonicalization); +grpc_mdstr *grpc_mdstr_from_string(grpc_mdctx *ctx, const char *str); /* Unrefs the slice. */ grpc_mdstr *grpc_mdstr_from_slice(grpc_mdctx *ctx, gpr_slice slice); grpc_mdstr *grpc_mdstr_from_buffer(grpc_mdctx *ctx, const gpr_uint8 *str, @@ -118,8 +117,7 @@ grpc_mdelem *grpc_mdelem_from_slices(grpc_mdctx *ctx, gpr_slice key, grpc_mdelem *grpc_mdelem_from_string_and_buffer(grpc_mdctx *ctx, const char *key, const gpr_uint8 *value, - size_t value_length, - int canonicalize_key); + size_t value_length); /* Mutator and accessor for grpc_mdelem user data. The destructor function is used as a type tag and is checked during user_data fetch. */ diff --git a/test/core/end2end/tests/metadata.c b/test/core/end2end/tests/metadata.c index d0604cc6b6f..be7a4f12c2d 100644 --- a/test/core/end2end/tests/metadata.c +++ b/test/core/end2end/tests/metadata.c @@ -113,8 +113,8 @@ static void test_request_response_with_metadata_and_payload( {"key1", "val1", 4, 0, {{NULL, NULL, NULL, NULL}}}, {"key2", "val2", 4, 0, {{NULL, NULL, NULL, NULL}}}}; grpc_metadata meta_s[2] = { - {"KeY3", "val3", 4, 0, {{NULL, NULL, NULL, NULL}}}, - {"KeY4", "val4", 4, 0, {{NULL, NULL, NULL, NULL}}}}; + {"key3", "val3", 4, 0, {{NULL, NULL, NULL, NULL}}}, + {"key4", "val4", 4, 0, {{NULL, NULL, NULL, NULL}}}}; grpc_end2end_test_fixture f = begin_test( config, "test_request_response_with_metadata_and_payload", NULL, NULL); cq_verifier *cqv = cq_verifier_create(f.cq); diff --git a/test/core/transport/metadata_test.c b/test/core/transport/metadata_test.c index 1861442bb1d..080e86cb633 100644 --- a/test/core/transport/metadata_test.c +++ b/test/core/transport/metadata_test.c @@ -63,9 +63,9 @@ static void test_create_string(void) { LOG_TEST("test_create_string"); ctx = grpc_mdctx_create(); - s1 = grpc_mdstr_from_string(ctx, "hello", 0); - s2 = grpc_mdstr_from_string(ctx, "hello", 0); - s3 = grpc_mdstr_from_string(ctx, "very much not hello", 0); + s1 = grpc_mdstr_from_string(ctx, "hello"); + s2 = grpc_mdstr_from_string(ctx, "hello"); + s3 = grpc_mdstr_from_string(ctx, "very much not hello"); GPR_ASSERT(s1 == s2); GPR_ASSERT(s3 != s1); GPR_ASSERT(gpr_slice_str_cmp(s1->slice, "hello") == 0); @@ -190,7 +190,7 @@ static void test_things_stick_around(void) { for (i = 0; i < nstrs; i++) { gpr_asprintf(&buffer, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx%dx", i); - strs[i] = grpc_mdstr_from_string(ctx, buffer, 0); + strs[i] = grpc_mdstr_from_string(ctx, buffer); shuf[i] = i; gpr_free(buffer); } @@ -212,7 +212,7 @@ static void test_things_stick_around(void) { GRPC_MDSTR_UNREF(strs[shuf[i]]); for (j = i + 1; j < nstrs; j++) { gpr_asprintf(&buffer, "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx%dx", shuf[j]); - test = grpc_mdstr_from_string(ctx, buffer, 0); + test = grpc_mdstr_from_string(ctx, buffer); GPR_ASSERT(test == strs[shuf[j]]); GRPC_MDSTR_UNREF(test); gpr_free(buffer); @@ -235,13 +235,13 @@ static void test_slices_work(void) { ctx = grpc_mdctx_create(); str = grpc_mdstr_from_string( - ctx, "123456789012345678901234567890123456789012345678901234567890", 0); + ctx, "123456789012345678901234567890123456789012345678901234567890"); slice = gpr_slice_ref(str->slice); GRPC_MDSTR_UNREF(str); gpr_slice_unref(slice); str = grpc_mdstr_from_string( - ctx, "123456789012345678901234567890123456789012345678901234567890", 0); + ctx, "123456789012345678901234567890123456789012345678901234567890"); slice = gpr_slice_ref(str->slice); gpr_slice_unref(slice); GRPC_MDSTR_UNREF(str); @@ -258,7 +258,7 @@ static void test_base64_and_huffman_works(void) { LOG_TEST("test_base64_and_huffman_works"); ctx = grpc_mdctx_create(); - str = grpc_mdstr_from_string(ctx, "abcdefg", 0); + str = grpc_mdstr_from_string(ctx, "abcdefg"); slice1 = grpc_mdstr_as_base64_encoded_and_huffman_compressed(str); slice2 = grpc_chttp2_base64_encode_and_huffman_compress(str->slice); GPR_ASSERT(0 == gpr_slice_cmp(slice1, slice2)); From 557d26f6438e40a482f65ea5f5e9d2990a59fa6e Mon Sep 17 00:00:00 2001 From: Tim Emiola Date: Fri, 25 Sep 2015 16:55:19 -0700 Subject: [PATCH 18/48] Remove a bad flaky rpc server test --- src/ruby/spec/generic/rpc_server_spec.rb | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/ruby/spec/generic/rpc_server_spec.rb b/src/ruby/spec/generic/rpc_server_spec.rb index e484a9ea505..efe07f734e6 100644 --- a/src/ruby/spec/generic/rpc_server_spec.rb +++ b/src/ruby/spec/generic/rpc_server_spec.rb @@ -405,22 +405,6 @@ describe GRPC::RpcServer do t.join end - it 'should not receive metadata if the client times out', server: true do - service = SlowService.new - @srv.handle(service) - t = Thread.new { @srv.run } - @srv.wait_till_running - req = EchoMsg.new - stub = SlowStub.new(@host, **client_opts) - timeout = 0.1 # too short for SlowService to respond - blk = proc { stub.an_rpc(req, timeout: timeout, k1: 'v1', k2: 'v2') } - expect(&blk).to raise_error GRPC::BadStatus - wanted_md = [] - expect(service.received_md).to eq(wanted_md) - @srv.stop - t.join - end - it 'should handle cancellation correctly', server: true do service = SlowService.new @srv.handle(service) From 8c7665e06d16b4978c2d09203f3ab75e0d0b698b Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Fri, 25 Sep 2015 21:40:19 -0700 Subject: [PATCH 19/48] Add some TODO comments for possible poll optimization --- src/core/iomgr/pollset_multipoller_with_epoll.c | 2 ++ src/core/iomgr/pollset_multipoller_with_poll_posix.c | 2 ++ src/core/iomgr/pollset_posix.c | 2 ++ 3 files changed, 6 insertions(+) diff --git a/src/core/iomgr/pollset_multipoller_with_epoll.c b/src/core/iomgr/pollset_multipoller_with_epoll.c index a4293eb4a40..b609e83c118 100644 --- a/src/core/iomgr/pollset_multipoller_with_epoll.c +++ b/src/core/iomgr/pollset_multipoller_with_epoll.c @@ -180,6 +180,8 @@ static void multipoll_with_epoll_pollset_maybe_work_and_unlock( pfds[1].events = POLLIN; pfds[1].revents = 0; + /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid + even going into the blocking annotation if possible */ GRPC_SCHEDULING_START_BLOCKING_REGION; poll_rv = grpc_poll_function(pfds, 2, timeout_ms); GRPC_SCHEDULING_END_BLOCKING_REGION; diff --git a/src/core/iomgr/pollset_multipoller_with_poll_posix.c b/src/core/iomgr/pollset_multipoller_with_poll_posix.c index 44031b8ef66..63e0b9edb9a 100644 --- a/src/core/iomgr/pollset_multipoller_with_poll_posix.c +++ b/src/core/iomgr/pollset_multipoller_with_poll_posix.c @@ -148,6 +148,8 @@ static void multipoll_with_poll_pollset_maybe_work_and_unlock( POLLOUT, &watchers[i]); } + /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid + even going into the blocking annotation if possible */ GRPC_SCHEDULING_START_BLOCKING_REGION; r = grpc_poll_function(pfds, pfd_count, timeout); GRPC_SCHEDULING_END_BLOCKING_REGION; diff --git a/src/core/iomgr/pollset_posix.c b/src/core/iomgr/pollset_posix.c index 10bab134d7f..283aae85f10 100644 --- a/src/core/iomgr/pollset_posix.c +++ b/src/core/iomgr/pollset_posix.c @@ -467,6 +467,8 @@ static void basic_pollset_maybe_work_and_unlock(grpc_exec_ctx *exec_ctx, gpr_mu_unlock(&pollset->mu); } + /* TODO(vpai): Consider first doing a 0 timeout poll here to avoid + even going into the blocking annotation if possible */ /* poll fd count (argument 2) is shortened by one if we have no events to poll on - such that it only includes the kicker */ GRPC_SCHEDULING_START_BLOCKING_REGION; From 499fb052a0fa8dd9845a274d8807f0c66e058b7e Mon Sep 17 00:00:00 2001 From: "Nicolas \"Pixel\" Noble" Date: Sat, 26 Sep 2015 07:07:06 +0200 Subject: [PATCH 20/48] Restoring sanity. --- include/grpc/support/port_platform.h | 6 ++++-- src/core/iomgr/udp_server.h | 2 +- src/core/support/block_annotate.h | 8 ++++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/include/grpc/support/port_platform.h b/include/grpc/support/port_platform.h index e5d45590aec..4373e885774 100644 --- a/include/grpc/support/port_platform.h +++ b/include/grpc/support/port_platform.h @@ -47,10 +47,12 @@ #endif /* NOMINMAX */ #ifndef _WIN32_WINNT -#error "Please compile grpc with _WIN32_WINNT of at least 0x600 (aka Windows Vista)" +#error \ + "Please compile grpc with _WIN32_WINNT of at least 0x600 (aka Windows Vista)" #else /* !defined(_WIN32_WINNT) */ #if (_WIN32_WINNT < 0x0600) -#error "Please compile grpc with _WIN32_WINNT of at least 0x600 (aka Windows Vista)" +#error \ + "Please compile grpc with _WIN32_WINNT of at least 0x600 (aka Windows Vista)" #endif /* _WIN32_WINNT < 0x0600 */ #endif /* defined(_WIN32_WINNT) */ diff --git a/src/core/iomgr/udp_server.h b/src/core/iomgr/udp_server.h index b6cf3c8ed9d..8e3abae864b 100644 --- a/src/core/iomgr/udp_server.h +++ b/src/core/iomgr/udp_server.h @@ -43,7 +43,7 @@ typedef struct grpc_server grpc_server; typedef struct grpc_udp_server grpc_udp_server; /* Called when data is available to read from the socket. */ -typedef void (*grpc_udp_server_read_cb)(int fd, grpc_server* server); +typedef void (*grpc_udp_server_read_cb)(int fd, grpc_server *server); /* Create a server, initially not bound to any ports */ grpc_udp_server *grpc_udp_server_create(void); diff --git a/src/core/support/block_annotate.h b/src/core/support/block_annotate.h index bf2c17f8595..3cd8eee272d 100644 --- a/src/core/support/block_annotate.h +++ b/src/core/support/block_annotate.h @@ -38,7 +38,11 @@ the code may block for reasons other than synchronization functions. These include poll, epoll, and getaddrinfo. */ -#define GRPC_SCHEDULING_START_BLOCKING_REGION do {} while (0) -#define GRPC_SCHEDULING_END_BLOCKING_REGION do {} while (0) +#define GRPC_SCHEDULING_START_BLOCKING_REGION \ + do { \ + } while (0) +#define GRPC_SCHEDULING_END_BLOCKING_REGION \ + do { \ + } while (0) #endif /* GRPC_INTERNAL_CORE_SUPPORT_BLOCK_ANNOTATE_H */ From d4509a16b38dc6e4a0dacbdad074accfe7894696 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 28 Sep 2015 09:18:40 -0700 Subject: [PATCH 21/48] Default to no-flakes, allow enabling them --- tools/run_tests/jobset.py | 2 +- tools/run_tests/run_tests.py | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index 6ddc4d29bc5..fb3632e4adf 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -133,7 +133,7 @@ class JobSpec(object): """Specifies what to run for a job.""" def __init__(self, cmdline, shortname=None, environ=None, hash_targets=None, - cwd=None, shell=False, timeout_seconds=5*60, flake_retries=5): + cwd=None, shell=False, timeout_seconds=5*60, flake_retries=0): """ Arguments: cmdline: a list of arguments to pass as the command line diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index ac5eef663e7..281901350fb 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -99,7 +99,8 @@ class SimpleConfig(object): environ=actual_environ, timeout_seconds=self.timeout_seconds, hash_targets=hash_targets - if self.allow_hashing else None) + if self.allow_hashing else None, + flake_retries=5 if args.allow_flakes else 0) # ValgrindConfig: compile with some CONFIG=config, but use valgrind to run @@ -117,7 +118,8 @@ class ValgrindConfig(object): return jobset.JobSpec(cmdline=['valgrind', '--tool=%s' % self.tool] + self.args + cmdline, shortname='valgrind %s' % cmdline[0], - hash_targets=None) + hash_targets=None, + flake_retries=5 if args.allow_flakes else 0) def get_c_tests(travis, test_lang) : @@ -555,6 +557,11 @@ argp.add_argument('--use_docker', help="Run all the tests under docker. That provides " + "additional isolation and prevents the need to installs " + "language specific prerequisites. Only available on Linux.") +argp.add_argument('--allow_flakes', + default=False, + action='store_const', + const=True, + help="Allow flaky tests to show as passing (re-runs failed tests up to five times)") argp.add_argument('-a', '--antagonists', default=0, type=int) argp.add_argument('-x', '--xml_report', default=None, type=str, help='Generates a JUnit-compatible XML report') From 2bc37297c8e0f268115b46604ed5f85371e5f679 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 28 Sep 2015 10:08:21 -0700 Subject: [PATCH 22/48] Enable backtraces on C tests --- Makefile | 8 ++--- include/grpc/support/port_platform.h | 2 ++ templates/Makefile.template | 8 ++--- test/core/util/test_config.c | 44 ++++++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 70918490d70..e0a7b804721 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ CXX_opt = $(DEFAULT_CXX) LD_opt = $(DEFAULT_CC) LDXX_opt = $(DEFAULT_CXX) CPPFLAGS_opt = -O2 -LDFLAGS_opt = +LDFLAGS_opt = -rdynamic DEFINES_opt = NDEBUG VALID_CONFIG_basicprof = 1 @@ -119,7 +119,7 @@ CXX_dbg = $(DEFAULT_CXX) LD_dbg = $(DEFAULT_CC) LDXX_dbg = $(DEFAULT_CXX) CPPFLAGS_dbg = -O0 -LDFLAGS_dbg = +LDFLAGS_dbg = -rdynamic DEFINES_dbg = _DEBUG DEBUG VALID_CONFIG_mutrace = 1 @@ -139,7 +139,7 @@ LD_valgrind = $(DEFAULT_CC) LDXX_valgrind = $(DEFAULT_CXX) CPPFLAGS_valgrind = -O0 OPENSSL_CFLAGS_valgrind = -DPURIFY -LDFLAGS_valgrind = +LDFLAGS_valgrind = -rdynamic DEFINES_valgrind = _DEBUG DEBUG GRPC_TEST_SLOWDOWN_BUILD_FACTOR=20 VALID_CONFIG_tsan = 1 @@ -190,7 +190,7 @@ CXX_gcov = g++ LD_gcov = gcc LDXX_gcov = g++ CPPFLAGS_gcov = -O0 -fprofile-arcs -ftest-coverage -LDFLAGS_gcov = -fprofile-arcs -ftest-coverage +LDFLAGS_gcov = -fprofile-arcs -ftest-coverage -rdynamic DEFINES_gcov = _DEBUG DEBUG diff --git a/include/grpc/support/port_platform.h b/include/grpc/support/port_platform.h index 93291a174da..0798b6eb092 100644 --- a/include/grpc/support/port_platform.h +++ b/include/grpc/support/port_platform.h @@ -121,6 +121,7 @@ #define GPR_GETPID_IN_UNISTD_H 1 #define GPR_HAVE_MSG_NOSIGNAL 1 #elif defined(__linux__) +#define GPR_POSIX_CRASH_HANDLER 1 #define GPR_PLATFORM_STRING "linux" #ifndef _BSD_SOURCE #define _BSD_SOURCE @@ -188,6 +189,7 @@ #define GPR_PLATFORM_STRING "osx" #define GPR_CPU_POSIX 1 #define GPR_GCC_TLS 1 +#define GPR_POSIX_CRASH_HANDLER 1 #endif #define GPR_GCC_ATOMIC 1 #define GPR_POSIX_LOG 1 diff --git a/templates/Makefile.template b/templates/Makefile.template index 8f79fdb8e4a..115d8136b53 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -108,7 +108,7 @@ LD_opt = $(DEFAULT_CC) LDXX_opt = $(DEFAULT_CXX) CPPFLAGS_opt = -O2 - LDFLAGS_opt = + LDFLAGS_opt = -rdynamic DEFINES_opt = NDEBUG VALID_CONFIG_basicprof = 1 @@ -135,7 +135,7 @@ LD_dbg = $(DEFAULT_CC) LDXX_dbg = $(DEFAULT_CXX) CPPFLAGS_dbg = -O0 - LDFLAGS_dbg = + LDFLAGS_dbg = -rdynamic DEFINES_dbg = _DEBUG DEBUG VALID_CONFIG_mutrace = 1 @@ -155,7 +155,7 @@ LDXX_valgrind = $(DEFAULT_CXX) CPPFLAGS_valgrind = -O0 OPENSSL_CFLAGS_valgrind = -DPURIFY - LDFLAGS_valgrind = + LDFLAGS_valgrind = -rdynamic DEFINES_valgrind = _DEBUG DEBUG GRPC_TEST_SLOWDOWN_BUILD_FACTOR=20 VALID_CONFIG_tsan = 1 @@ -206,7 +206,7 @@ LD_gcov = gcc LDXX_gcov = g++ CPPFLAGS_gcov = -O0 -fprofile-arcs -ftest-coverage - LDFLAGS_gcov = -fprofile-arcs -ftest-coverage + LDFLAGS_gcov = -fprofile-arcs -ftest-coverage -rdynamic DEFINES_gcov = _DEBUG DEBUG diff --git a/test/core/util/test_config.c b/test/core/util/test_config.c index 168b98fe634..c3d356c1f23 100644 --- a/test/core/util/test_config.c +++ b/test/core/util/test_config.c @@ -83,6 +83,50 @@ static void install_crash_handler() { _set_abort_behavior(0, _CALL_REPORTFAULT); signal(SIGABRT, abort_handler); } +#elif GPR_POSIX_CRASH_HANDLER +#include +#include +#include +#include + +static char g_alt_stack[8192]; + +#define MAX_FRAMES 32 + +static void crash_handler(int signum, siginfo_t *info, void *data) { + void *addrlist[MAX_FRAMES + 1]; + int addrlen; + int i; + char **symlist; + + fprintf(stderr, "Caught signal %d\n", signum); + addrlen = backtrace(addrlist, GPR_ARRAY_SIZE(addrlist)); + + symlist = backtrace_symbols(addrlist, addrlen); + for (i = 0; i < addrlen; i++) { + fprintf(stderr, " %s\n", symlist[i]); + } + free(symlist); + + raise(signum); +} + +static void install_crash_handler() { + stack_t ss; + struct sigaction sa; + memset(&ss, 0, sizeof(ss)); + memset(&sa, 0, sizeof(sa)); + ss.ss_size = sizeof(g_alt_stack); + ss.ss_sp = g_alt_stack; + GPR_ASSERT(sigaltstack(&ss, NULL) == 0); + sa.sa_flags = (int)(SA_SIGINFO | SA_ONSTACK | SA_RESETHAND); + sa.sa_sigaction = crash_handler; + GPR_ASSERT(sigaction(SIGILL, &sa, NULL) == 0); + GPR_ASSERT(sigaction(SIGABRT, &sa, NULL) == 0); + GPR_ASSERT(sigaction(SIGBUS, &sa, NULL) == 0); + GPR_ASSERT(sigaction(SIGSEGV, &sa, NULL) == 0); + GPR_ASSERT(sigaction(SIGSTKFLT, &sa, NULL) == 0); +} #else static void install_crash_handler() {} #endif From ecc4f9c6945da9382ff5201a3c426ce203fd3c09 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 28 Sep 2015 11:48:46 -0700 Subject: [PATCH 23/48] Fix mac build --- test/core/util/test_config.c | 1 - 1 file changed, 1 deletion(-) diff --git a/test/core/util/test_config.c b/test/core/util/test_config.c index c3d356c1f23..76b2cbd4d4d 100644 --- a/test/core/util/test_config.c +++ b/test/core/util/test_config.c @@ -125,7 +125,6 @@ static void install_crash_handler() { GPR_ASSERT(sigaction(SIGABRT, &sa, NULL) == 0); GPR_ASSERT(sigaction(SIGBUS, &sa, NULL) == 0); GPR_ASSERT(sigaction(SIGSEGV, &sa, NULL) == 0); - GPR_ASSERT(sigaction(SIGSTKFLT, &sa, NULL) == 0); } #else static void install_crash_handler() {} From 6ec5d6f63a177a66decd963de3cd5523f6b2eca6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 28 Sep 2015 12:53:05 -0700 Subject: [PATCH 24/48] Make run_jenkins script forward arguments to run_tests.py --- tools/jenkins/run_jenkins.sh | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tools/jenkins/run_jenkins.sh b/tools/jenkins/run_jenkins.sh index a55f1de9960..c4a01a7d663 100755 --- a/tools/jenkins/run_jenkins.sh +++ b/tools/jenkins/run_jenkins.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash # Copyright 2015, Google Inc. # All rights reserved. # @@ -31,8 +31,6 @@ # This script is invoked by Jenkins and triggers a test run based on # env variable settings. # -# Bootstrap into bash -[ -z $1 ] && exec bash $0 bootstrapped # Setting up rvm environment BEFORE we set -ex. [[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh # To prevent cygwin bash complaining about empty lines ending with \r @@ -56,11 +54,11 @@ if [ "$platform" == "linux" ] then echo "building $language on Linux" - ./tools/run_tests/run_tests.py --use_docker -t -l $language -c $config -x report.xml || true + ./tools/run_tests/run_tests.py --use_docker -t -l $language -c $config -x report.xml $@ || true elif [ "$platform" == "interop" ] then - python tools/run_tests/run_interops.py --language=$language + python tools/run_tests/run_interops.py --language=$language $@ elif [ "$platform" == "windows" ] then echo "building $language on Windows" @@ -72,19 +70,19 @@ then /cygdrive/c/nuget/nuget.exe restore vsprojects/grpc.sln /cygdrive/c/nuget/nuget.exe restore src/csharp/Grpc.sln - python tools/run_tests/run_tests.py -t -l $language -x report.xml || true + python tools/run_tests/run_tests.py -t -l $language -x report.xml $@ || true elif [ "$platform" == "macos" ] then echo "building $language on MacOS" - ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml || true + ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml $@ || true elif [ "$platform" == "freebsd" ] then echo "building $language on FreeBSD" - MAKE=gmake ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml || true + MAKE=gmake ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml $@ || true else echo "Unknown platform $platform" exit 1 From 73c4e876701eb274210352af36bef90c81d1dd87 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 28 Sep 2015 13:21:54 -0700 Subject: [PATCH 25/48] Fix mac build --- test/core/util/test_config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core/util/test_config.c b/test/core/util/test_config.c index 76b2cbd4d4d..30caf4b1ef8 100644 --- a/test/core/util/test_config.c +++ b/test/core/util/test_config.c @@ -89,7 +89,7 @@ static void install_crash_handler() { #include #include -static char g_alt_stack[8192]; +static char g_alt_stack[MINSIGSTKSZ]; #define MAX_FRAMES 32 From 95cc07b1691f5b076a280a6cd4d2299d8c164e73 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 28 Sep 2015 13:41:30 -0700 Subject: [PATCH 26/48] Split timeout and failure retry policy --- tools/run_tests/jobset.py | 14 +++++++++----- tools/run_tests/run_tests.py | 3 ++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index 068deb0b172..e62ed60c9fa 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -134,7 +134,8 @@ class JobSpec(object): """Specifies what to run for a job.""" def __init__(self, cmdline, shortname=None, environ=None, hash_targets=None, - cwd=None, shell=False, timeout_seconds=5*60, flake_retries=0): + cwd=None, shell=False, timeout_seconds=5*60, flake_retries=0, + timeout_retries=0): """ Arguments: cmdline: a list of arguments to pass as the command line @@ -154,6 +155,7 @@ class JobSpec(object): self.shell = shell self.timeout_seconds = timeout_seconds self.flake_retries = flake_retries + self.timeout_retries = 0 def identity(self): return '%r %r %r' % (self.cmdline, self.environ, self.hash_targets) @@ -177,6 +179,7 @@ class Job(object): self._xml_test = ET.SubElement(xml_report, 'testcase', name=self._spec.shortname) if xml_report is not None else None self._retries = 0 + self._timeout_retries = 0 message('START', spec.shortname, do_newline=self._travis) self.start() @@ -224,17 +227,18 @@ class Job(object): ET.SubElement(self._xml_test, 'failure', message='Failure').text else: self._state = _SUCCESS - message('PASSED', '%s [time=%.1fsec; retries=%d]' % (self._spec.shortname, elapsed, self._retries), - do_newline=self._newline_on_success or self._travis) + message('PASSED', '%s [time=%.1fsec; retries=%d;%d]' % ( + self._spec.shortname, elapsed, self._retries, self._timeout_retries), + do_newline=self._newline_on_success or self._travis) if self._bin_hash: update_cache.finished(self._spec.identity(), self._bin_hash) elif self._state == _RUNNING and time.time() - self._start > self._spec.timeout_seconds: self._tempfile.seek(0) stdout = self._tempfile.read() filtered_stdout = filter(lambda x: x in string.printable, stdout.decode(errors='ignore')) - if self._retries < self._spec.flake_retries: + if self._timeout_retries < self._spec.timeout_retries: message('TIMEOUT_FLAKE', self._spec.shortname, stdout, do_newline=True) - self._retries += 1 + self._timeout_retries += 1 self._process.terminate() self.start() else: diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index cbbb58d7e72..73b2b3640b3 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -120,7 +120,8 @@ class ValgrindConfig(object): self.args + cmdline, shortname='valgrind %s' % cmdline[0], hash_targets=None, - flake_retries=5 if args.allow_flakes else 0) + flake_retries=5 if args.allow_flakes else 0, + timeout_retries=2 if args.allow_flakes else 0) def get_c_tests(travis, test_lang) : From 926c0e99f8d44c58a8f3d3321f51b007623ae495 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 28 Sep 2015 14:25:38 -0700 Subject: [PATCH 27/48] Initialize/destroy TLS members - to fix iOS build --- src/core/iomgr/pollset_posix.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/iomgr/pollset_posix.c b/src/core/iomgr/pollset_posix.c index 10bab134d7f..e80963e0ea8 100644 --- a/src/core/iomgr/pollset_posix.c +++ b/src/core/iomgr/pollset_posix.c @@ -119,11 +119,13 @@ void grpc_pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) { void grpc_pollset_global_init(void) { gpr_tls_init(&g_current_thread_poller); + gpr_tls_init(&g_current_thread_worker); grpc_wakeup_fd_global_init(); } void grpc_pollset_global_shutdown(void) { gpr_tls_destroy(&g_current_thread_poller); + gpr_tls_destroy(&g_current_thread_worker); grpc_wakeup_fd_global_destroy(); } From bfc8a06a7b37cf8368938b853cd51296ed4add37 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 28 Sep 2015 14:40:21 -0700 Subject: [PATCH 28/48] Fix typo --- tools/run_tests/jobset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/jobset.py b/tools/run_tests/jobset.py index e62ed60c9fa..b9ec020f9ef 100755 --- a/tools/run_tests/jobset.py +++ b/tools/run_tests/jobset.py @@ -155,7 +155,7 @@ class JobSpec(object): self.shell = shell self.timeout_seconds = timeout_seconds self.flake_retries = flake_retries - self.timeout_retries = 0 + self.timeout_retries = timeout_retries def identity(self): return '%r %r %r' % (self.cmdline, self.environ, self.hash_targets) From 320bd614994a04dd36d829ecfe498584b3bb6872 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 15 Sep 2015 12:44:35 -0700 Subject: [PATCH 29/48] simplify running of interop tests on jenkins --- tools/jenkins/docker_run_interop_tests.sh | 82 +++++++++++++++++++ tools/jenkins/docker_run_tests.sh | 1 + tools/jenkins/run_jenkins.sh | 1 + tools/jenkins/run_local.sh | 59 +++++++++++++ ...n_interops_test.sh => run_interop_test.sh} | 20 +++-- tools/run_tests/run_interop_tests.py | 78 ++++++++++++++++++ tools/run_tests/run_interops.py | 37 --------- tools/run_tests/run_interops_build.sh | 75 ----------------- 8 files changed, 233 insertions(+), 120 deletions(-) create mode 100755 tools/jenkins/docker_run_interop_tests.sh create mode 100755 tools/jenkins/run_local.sh rename tools/run_tests/{run_interops_test.sh => run_interop_test.sh} (56%) create mode 100755 tools/run_tests/run_interop_tests.py delete mode 100755 tools/run_tests/run_interops.py delete mode 100755 tools/run_tests/run_interops_build.sh diff --git a/tools/jenkins/docker_run_interop_tests.sh b/tools/jenkins/docker_run_interop_tests.sh new file mode 100755 index 00000000000..bd6fa4424e1 --- /dev/null +++ b/tools/jenkins/docker_run_interop_tests.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# This script is invoked by run_jekins.sh. It contains the test logic +# that should run inside a docker container. +set -e + +mkdir -p /var/local/git +git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc + +cd /var/local/git/grpc +nvm use 0.12 +rvm use ruby-2.1 + +# TODO(jtattermusch): use cleaner way to install root certs +mkdir -p /usr/local/share/grpc +cp etc/roots.pem /usr/local/share/grpc/ + +# build C++ interop client & server +make interop_client interop_server + +# build C# interop client & server +make install_grpc_csharp_ext +(cd src/csharp && mono /var/local/NuGet.exe restore Grpc.sln) +(cd src/csharp && xbuild Grpc.sln) + +# build Node interop client & server +npm install -g node-gyp +make install_c -C /var/local/git/grpc +(cd src/node && npm install && node-gyp rebuild) + +# build Ruby interop client and server +(cd src/ruby && gem update bundler && bundle && rake compile:grpc) + +# TODO(jtattermusch): add python + +# build PHP interop client +# TODO(jtattermusch): make php work +# TODO(jtattermusch): prerequisites should be installed sooner than here. +# Install composer +#curl -sS https://getcomposer.org/installer | php +#mv composer.phar /usr/local/bin/composer +# Download the patched PHP protobuf so that PHP gRPC clients can be generated +# from proto3 schemas. +#git clone https://github.com/stanley-cheung/Protobuf-PHP.git /var/local/git/protobuf-php +#(cd src/php/ext/grpc && phpize && ./configure && make) +#rvm all do gem install ronn rake +#(cd /var/local/git/protobuf-php \ +# && rvm all do rake pear:package version=1.0 \ +# && pear install Protobuf-1.0.tgz) +#(cd src/php && composer install) +#(cd src/php && protoc-gen-php -i tests/interop/ -o tests/interop/ tests/interop/test.proto) + +# run the cloud-to-prod interop tests +tools/run_tests/run_interop_tests.py -l $language diff --git a/tools/jenkins/docker_run_tests.sh b/tools/jenkins/docker_run_tests.sh index 781bff26b93..7877ad2459e 100755 --- a/tools/jenkins/docker_run_tests.sh +++ b/tools/jenkins/docker_run_tests.sh @@ -30,6 +30,7 @@ # # This script is invoked by build_docker_and_run_tests.py inside a docker # container. You should never need to call this script on your own. + set -e export CONFIG=$config diff --git a/tools/jenkins/run_jenkins.sh b/tools/jenkins/run_jenkins.sh index c4a01a7d663..66364b3093d 100755 --- a/tools/jenkins/run_jenkins.sh +++ b/tools/jenkins/run_jenkins.sh @@ -59,6 +59,7 @@ then elif [ "$platform" == "interop" ] then python tools/run_tests/run_interops.py --language=$language $@ + elif [ "$platform" == "windows" ] then echo "building $language on Windows" diff --git a/tools/jenkins/run_local.sh b/tools/jenkins/run_local.sh new file mode 100755 index 00000000000..f9d8d26e9a6 --- /dev/null +++ b/tools/jenkins/run_local.sh @@ -0,0 +1,59 @@ +#!/bin/sh +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# This script can be used to run dockerized tests that normally run +# on Jenkins on your local machine using the working copy that +# is currently checked out locally. + +# IMPORTANT: The changes to be tested need to be committed locally, +# otherwise they won't be cloned inside the docker container. +set -e + +cd `dirname $0`/../.. + +#TODO(jtattermusch): provide way to tunnel run_tests cmdline options to run_tests. +#TODO(jtattermusch): provide way to grab the docker image built by run_jenkins + +# config: opt or dbg +export config=opt + +# platform: +# -- use linux to run tests under docker +# -- use interop to run dockerized interop tests +export platform=interop + +# language: one of languages supported by run_tests.py +export language=all + +# architecture +export arch=`uname -m` + +# test run configuration is done through environment variables above +tools/jenkins/run_jenkins.sh diff --git a/tools/run_tests/run_interops_test.sh b/tools/run_tests/run_interop_test.sh similarity index 56% rename from tools/run_tests/run_interops_test.sh rename to tools/run_tests/run_interop_test.sh index 9be253af460..255760ba849 100755 --- a/tools/run_tests/run_interops_test.sh +++ b/tools/run_tests/run_interop_test.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Copyright 2015, Google Inc. # All rights reserved. @@ -32,23 +32,27 @@ language=$1 test_case=$2 +# change dir gRPC repo root +cd $(dirname $0)/../.. + set -e if [ "$language" = "c++" ] then - sudo docker run grpc/cxx /var/local/git/grpc/bins/opt/interop_client --enable_ssl --use_prod_roots --server_host_override=grpc-test.sandbox.google.com --server_host=grpc-test.sandbox.google.com --server_port=443 --test_case=$test_case + bins/opt/interop_client --enable_ssl --use_prod_roots --server_host_override=grpc-test.sandbox.google.com --server_host=grpc-test.sandbox.google.com --server_port=443 --test_case=$test_case elif [ "$language" = "node" ] then - sudo docker run grpc/node /usr/bin/nodejs /var/local/git/grpc/src/node/interop/interop_client.js --use_tls=true --use_test_ca=true --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com --test_case=$test_case + SSL_CERT_FILE=/usr/local/share/grpc/roots.pem node src/node/interop/interop_client.js --use_tls=true --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com --test_case=$test_case elif [ "$language" = "ruby" ] then - cmd_prefix="SSL_CERT_FILE=/cacerts/roots.pem ruby /var/local/git/grpc/src/ruby/bin/interop/interop_client.rb --use_tls --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com " - cmd="$cmd_prefix --test_case=$test_case" - sudo docker run grpc/ruby bin/bash -l -c '$cmd' + SSL_CERT_FILE=/usr/local/share/grpc/roots.pem ruby src/ruby/bin/interop/interop_client.rb --use_tls --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com --test_case=$test_case +elif [ "$language" = "csharp" ] +then + (cd src/csharp/Grpc.IntegrationTesting.Client/bin/Debug && SSL_CERT_FILE=/usr/local/share/grpc/roots.pem mono Grpc.IntegrationTesting.Client.exe --use_tls --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com --test_case=$test_case) elif [ "$language" = "php" ] then - sudo docker run -e SSL_CERT_FILE=/cacerts/roots.pem grpc/php /var/local/git/grpc/src/php/bin/interop_client.sh --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com --test_case=$test_case + SSL_CERT_FILE=/usr/local/share/grpc/roots.pem src/php/bin/interop_client.sh --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com --test_case=$test_case else - echo "interop testss not added for $language" + echo "interop tests not added for $language" exit 1 fi diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py new file mode 100755 index 00000000000..24410778dcd --- /dev/null +++ b/tools/run_tests/run_interop_tests.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Run interop (cross-language) tests in parallel.""" + +import argparse +import itertools +import xml.etree.cElementTree as ET +import jobset + +# TODO(jtattermusch): add php and python once we get them working +_LANGUAGES = ['c++', 'node', 'csharp', 'ruby'] + +# TODO(jtattermusch): add empty_stream once C++ start supporting it. +# TODO(jtattermusch): add support for auth tests. +_TEST_CASES = ['large_unary', 'empty_unary', 'ping_pong', + 'client_streaming', 'server_streaming', + 'cancel_after_begin', 'cancel_after_first_response', + 'timeout_on_sleeping_server'] + +argp = argparse.ArgumentParser(description='Run interop tests.') +argp.add_argument('-l', '--language', + choices=['all'] + sorted(_LANGUAGES), + nargs='+', + default=['all']) +args = argp.parse_args() + +languages = [l for l in itertools.chain.from_iterable( + iter(_LANGUAGES) if x == 'all' else [x] + for x in args.language)] + +jobs = [] +jobNumber = 0 +for language in languages: + for test in _TEST_CASES: + test_job = jobset.JobSpec( + cmdline=['tools/run_tests/run_interop_test.sh', '%s' % language, '%s' % test], + shortname="cloud_to_prod:%s:%s" % (language, test), + timeout_seconds=60) + jobs.append(test_job) + jobNumber+=1 + +root = ET.Element('testsuites') +testsuite = ET.SubElement(root, 'testsuite', id='1', package='grpc', name='tests') + +jobset.run(jobs, maxjobs=jobNumber, xml_report=testsuite) + +tree = ET.ElementTree(root) +tree.write('report.xml', encoding='UTF-8') + + diff --git a/tools/run_tests/run_interops.py b/tools/run_tests/run_interops.py deleted file mode 100755 index 17083975d8c..00000000000 --- a/tools/run_tests/run_interops.py +++ /dev/null @@ -1,37 +0,0 @@ -import argparse -import xml.etree.cElementTree as ET -import jobset - -argp = argparse.ArgumentParser(description='Run interop tests.') -argp.add_argument('-l', '--language', - default='c++') -args = argp.parse_args() - -# build job -build_job = jobset.JobSpec(cmdline=['tools/run_tests/run_interops_build.sh', '%s' % args.language], - shortname='build', - timeout_seconds=30*60) - -# test jobs, each test is a separate job to run in parallel -_TESTS = ['large_unary', 'empty_unary', 'ping_pong', 'client_streaming', 'server_streaming'] -jobs = [] -jobNumber = 0 -for test in _TESTS: - test_job = jobset.JobSpec( - cmdline=['tools/run_tests/run_interops_test.sh', '%s' % args.language, '%s' % test], - shortname=test, - timeout_seconds=15*60) - jobs.append(test_job) - jobNumber+=1 - -root = ET.Element('testsuites') -testsuite = ET.SubElement(root, 'testsuite', id='1', package='grpc', name='tests') - -# always do the build of docker first, and then all the tests can run in parallel -jobset.run([build_job], maxjobs=1, xml_report=testsuite) -jobset.run(jobs, maxjobs=jobNumber, xml_report=testsuite) - -tree = ET.ElementTree(root) -tree.write('report.xml', encoding='UTF-8') - - diff --git a/tools/run_tests/run_interops_build.sh b/tools/run_tests/run_interops_build.sh deleted file mode 100755 index ff1a26cf899..00000000000 --- a/tools/run_tests/run_interops_build.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/sh - -# Copyright 2015, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -language=$1 - -set -e - -#clean up any old docker files and start mirroring repository if not started already -sudo docker rmi -f grpc/cxx || true -sudo docker rmi -f grpc/base || true -sudo docker rmi -f 0.0.0.0:5000/grpc/base || true -sudo docker run -d -e GCS_BUCKET=docker-interop-images -e STORAGE_PATH=/admin/docker_images -p 5000:5000 google/docker-registry || true - -#prepare building by pulling down base images and necessary files -sudo docker pull 0.0.0.0:5000/grpc/base -sudo docker tag -f 0.0.0.0:5000/grpc/base grpc/base - -if [ "$language" = "c++" ] -then - gsutil cp -R gs://docker-interop-images/admin/service_account tools/dockerfile/grpc_cxx - gsutil cp -R gs://docker-interop-images/admin/cacerts tools/dockerfile/grpc_cxx - sudo docker build --no-cache -t grpc/cxx tools/dockerfile/grpc_cxx -elif [ "$language" = "node" ] -then - sudo docker pull 0.0.0.0:5000/grpc/node_base - sudo docker tag -f 0.0.0.0:5000/grpc/node_base grpc/node_base - gsutil cp -R gs://docker-interop-images/admin/service_account tools/dockerfile/grpc_node - gsutil cp -R gs://docker-interop-images/admin/cacerts tools/dockerfile/grpc_node - sudo docker build --no-cache -t grpc/node tools/dockerfile/grpc_node -elif [ "$language" = "ruby" ] -then - sudo docker pull 0.0.0.0:5000/grpc/ruby_base - sudo docker tag -f 0.0.0.0:5000/grpc/ruby_base grpc/ruby_base - gsutil cp -R gs://docker-interop-images/admin/service_account tools/dockerfile/grpc_ruby - gsutil cp -R gs://docker-interop-images/admin/cacerts tools/dockerfile/grpc_ruby - sudo docker build --no-cache -t grpc/ruby tools/dockerfile/grpc_ruby -elif [ "$language" = "php" ] -then - sudo docker pull 0.0.0.0:5000/grpc/php_base - sudo docker tag -f 0.0.0.0:5000/grpc/php_base grpc/php_base - gsutil cp -R gs://docker-interop-images/admin/service_account tools/dockerfile/grpc_php - gsutil cp -R gs://docker-interop-images/admin/cacerts tools/dockerfile/grpc_php - sudo docker build --no-cache -t grpc/php tools/dockerfile/grpc_php -else - echo "interop testss not added for $language" - exit 1 -fi From f49936acef3cd54d758b7253404be4f1884f965a Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 16 Sep 2015 15:44:26 -0700 Subject: [PATCH 30/48] make python generate args for interop tests --- tools/jenkins/run_local.sh | 2 +- tools/run_tests/run_interop_test.sh | 58 -------------- tools/run_tests/run_interop_tests.py | 114 ++++++++++++++++++++++++--- 3 files changed, 106 insertions(+), 68 deletions(-) delete mode 100755 tools/run_tests/run_interop_test.sh diff --git a/tools/jenkins/run_local.sh b/tools/jenkins/run_local.sh index f9d8d26e9a6..4891637bbf9 100755 --- a/tools/jenkins/run_local.sh +++ b/tools/jenkins/run_local.sh @@ -49,7 +49,7 @@ export config=opt # -- use interop to run dockerized interop tests export platform=interop -# language: one of languages supported by run_tests.py +# language: one of languages supported by run_tests.py or run_interop_tests.py export language=all # architecture diff --git a/tools/run_tests/run_interop_test.sh b/tools/run_tests/run_interop_test.sh deleted file mode 100755 index 255760ba849..00000000000 --- a/tools/run_tests/run_interop_test.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -# Copyright 2015, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -language=$1 -test_case=$2 - -# change dir gRPC repo root -cd $(dirname $0)/../.. - -set -e -if [ "$language" = "c++" ] -then - bins/opt/interop_client --enable_ssl --use_prod_roots --server_host_override=grpc-test.sandbox.google.com --server_host=grpc-test.sandbox.google.com --server_port=443 --test_case=$test_case -elif [ "$language" = "node" ] -then - SSL_CERT_FILE=/usr/local/share/grpc/roots.pem node src/node/interop/interop_client.js --use_tls=true --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com --test_case=$test_case -elif [ "$language" = "ruby" ] -then - SSL_CERT_FILE=/usr/local/share/grpc/roots.pem ruby src/ruby/bin/interop/interop_client.rb --use_tls --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com --test_case=$test_case -elif [ "$language" = "csharp" ] -then - (cd src/csharp/Grpc.IntegrationTesting.Client/bin/Debug && SSL_CERT_FILE=/usr/local/share/grpc/roots.pem mono Grpc.IntegrationTesting.Client.exe --use_tls --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com --test_case=$test_case) -elif [ "$language" = "php" ] -then - SSL_CERT_FILE=/usr/local/share/grpc/roots.pem src/php/bin/interop_client.sh --server_port=443 --server_host=grpc-test.sandbox.google.com --server_host_override=grpc-test.sandbox.google.com --test_case=$test_case -else - echo "interop tests not added for $language" - exit 1 -fi - diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 24410778dcd..05fd0bb27bb 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -35,8 +35,93 @@ import itertools import xml.etree.cElementTree as ET import jobset + +_CLOUD_TO_PROD_BASE_ARGS = [ + '--server_host_override=grpc-test.sandbox.google.com', + '--server_host=grpc-test.sandbox.google.com', + '--server_port=443'] + +# TOOD(jtattermusch) wrapped languages use this variable for location +# of roots.pem. We might want to use GRPC_DEFAULT_SSL_ROOTS_FILE_PATH +# supported by C core SslCredentials instead. +_SSL_CERT_ENV = { 'SSL_CERT_FILE':'/usr/local/share/grpc/roots.pem' } + +# TODO(jtatttermusch) unify usage of --enable_ssl, --use_tls and --use_tls=true + +class CXXLanguage: + + def __init__(self): + self.client_cmdline_base = ['bins/opt/interop_client'] + self.client_cwd = None + + def cloud_to_prod_args(self): + return (self.client_cmdline_base + _CLOUD_TO_PROD_BASE_ARGS + + ['--enable_ssl','--use_prod_roots']) + + def cloud_to_prod_env(self): + return None + + def __str__(self): + return 'c++' + + +class CSharpLanguage: + + def __init__(self): + self.client_cmdline_base = ['mono', 'Grpc.IntegrationTesting.Client.exe'] + self.client_cwd = 'src/csharp/Grpc.IntegrationTesting.Client/bin/Debug' + + def cloud_to_prod_args(self): + return (self.client_cmdline_base + _CLOUD_TO_PROD_BASE_ARGS + + ['--use_tls']) + + def cloud_to_prod_env(self): + return _SSL_CERT_ENV + + def __str__(self): + return 'csharp' + + +class NodeLanguage: + + def __init__(self): + self.client_cmdline_base = ['node', 'src/node/interop/interop_client.js'] + self.client_cwd = None + + def cloud_to_prod_args(self): + return (self.client_cmdline_base + _CLOUD_TO_PROD_BASE_ARGS + + ['--use_tls=true']) + + def cloud_to_prod_env(self): + return _SSL_CERT_ENV + + def __str__(self): + return 'node' + +class RubyLanguage: + + def __init__(self): + self.client_cmdline_base = ['ruby', 'src/ruby/bin/interop/interop_client.rb'] + self.client_cwd = None + + def cloud_to_prod_args(self): + return (self.client_cmdline_base + _CLOUD_TO_PROD_BASE_ARGS + + ['--use_tls']) + + def cloud_to_prod_env(self): + return _SSL_CERT_ENV + + def __str__(self): + return 'ruby' + + # TODO(jtattermusch): add php and python once we get them working -_LANGUAGES = ['c++', 'node', 'csharp', 'ruby'] +_LANGUAGES = { + 'c++' : CXXLanguage(), + 'csharp' : CSharpLanguage(), + 'node' : NodeLanguage(), + 'ruby' : RubyLanguage(), +} # TODO(jtattermusch): add empty_stream once C++ start supporting it. # TODO(jtattermusch): add support for auth tests. @@ -45,6 +130,17 @@ _TEST_CASES = ['large_unary', 'empty_unary', 'ping_pong', 'cancel_after_begin', 'cancel_after_first_response', 'timeout_on_sleeping_server'] +def cloud_to_prod_jobspec(language, test_case): + """Creates jobspec for cloud-to-prod interop test""" + cmdline = language.cloud_to_prod_args() + ['--test_case=%s' % test_case] + test_job = jobset.JobSpec( + cmdline=cmdline, + cwd=language.client_cwd, + shortname="cloud_to_prod:%s:%s" % (language, test_case), + environ=language.cloud_to_prod_env(), + timeout_seconds=60) + return test_job + argp = argparse.ArgumentParser(description='Run interop tests.') argp.add_argument('-l', '--language', choices=['all'] + sorted(_LANGUAGES), @@ -52,18 +148,18 @@ argp.add_argument('-l', '--language', default=['all']) args = argp.parse_args() -languages = [l for l in itertools.chain.from_iterable( - iter(_LANGUAGES) if x == 'all' else [x] - for x in args.language)] +languages = set(_LANGUAGES[l] + for l in itertools.chain.from_iterable( + _LANGUAGES.iterkeys() if x == 'all' else [x] + for x in args.language)) +# TODO(jtattermusch): make python script generate cmdline params for interop +# tests. It's easier to manage than in a shell script. jobs = [] jobNumber = 0 for language in languages: - for test in _TEST_CASES: - test_job = jobset.JobSpec( - cmdline=['tools/run_tests/run_interop_test.sh', '%s' % language, '%s' % test], - shortname="cloud_to_prod:%s:%s" % (language, test), - timeout_seconds=60) + for test_case in _TEST_CASES: + test_job = cloud_to_prod_jobspec(language, test_case) jobs.append(test_job) jobNumber+=1 From f88f3e2c69df107a6414c32fbb7937375f81afd6 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 16 Sep 2015 17:50:45 -0700 Subject: [PATCH 31/48] enable php interop tests --- tools/jenkins/docker_run_interop_tests.sh | 24 +++++++++++------------ tools/run_tests/run_interop_tests.py | 19 ++++++++++++++++++ 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/tools/jenkins/docker_run_interop_tests.sh b/tools/jenkins/docker_run_interop_tests.sh index bd6fa4424e1..d363d1c68cc 100755 --- a/tools/jenkins/docker_run_interop_tests.sh +++ b/tools/jenkins/docker_run_interop_tests.sh @@ -62,21 +62,21 @@ make install_c -C /var/local/git/grpc # TODO(jtattermusch): add python # build PHP interop client -# TODO(jtattermusch): make php work -# TODO(jtattermusch): prerequisites should be installed sooner than here. +# TODO(jtattermusch): prerequisites for PHP should be installed sooner than here. # Install composer -#curl -sS https://getcomposer.org/installer | php -#mv composer.phar /usr/local/bin/composer +curl -sS https://getcomposer.org/installer | php +mv composer.phar /usr/local/bin/composer # Download the patched PHP protobuf so that PHP gRPC clients can be generated # from proto3 schemas. -#git clone https://github.com/stanley-cheung/Protobuf-PHP.git /var/local/git/protobuf-php -#(cd src/php/ext/grpc && phpize && ./configure && make) -#rvm all do gem install ronn rake -#(cd /var/local/git/protobuf-php \ -# && rvm all do rake pear:package version=1.0 \ -# && pear install Protobuf-1.0.tgz) -#(cd src/php && composer install) -#(cd src/php && protoc-gen-php -i tests/interop/ -o tests/interop/ tests/interop/test.proto) +git clone https://github.com/stanley-cheung/Protobuf-PHP.git /var/local/git/protobuf-php +(cd src/php/ext/grpc && phpize && ./configure && make) +rvm all do gem install ronn rake +(cd third_party/protobuf && make install) +(cd /var/local/git/protobuf-php \ + && rvm all do rake pear:package version=1.0 \ + && pear install Protobuf-1.0.tgz) +(cd src/php && composer install) +(cd src/php && protoc-gen-php -i tests/interop/ -o tests/interop/ tests/interop/test.proto) # run the cloud-to-prod interop tests tools/run_tests/run_interop_tests.py -l $language diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 05fd0bb27bb..e9c4ce0a0f9 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -98,6 +98,24 @@ class NodeLanguage: def __str__(self): return 'node' + +class PHPLanguage: + + def __init__(self): + self.client_cmdline_base = ['src/php/bin/interop_client.sh'] + self.client_cwd = None + + def cloud_to_prod_args(self): + return (self.client_cmdline_base + _CLOUD_TO_PROD_BASE_ARGS + + ['--use_tls']) + + def cloud_to_prod_env(self): + return _SSL_CERT_ENV + + def __str__(self): + return 'php' + + class RubyLanguage: def __init__(self): @@ -120,6 +138,7 @@ _LANGUAGES = { 'c++' : CXXLanguage(), 'csharp' : CSharpLanguage(), 'node' : NodeLanguage(), + 'php' : PHPLanguage(), 'ruby' : RubyLanguage(), } From 8266c67127912a7b53dac1b6a4f7823db4bcb67b Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 17 Sep 2015 09:18:03 -0700 Subject: [PATCH 32/48] enable cloud_to_cloud tests with servers in a separate container --- .../Grpc.IntegrationTesting/InteropClient.cs | 2 - .../build_docker_and_run_interop_tests.sh | 111 ++++++++++++ tools/jenkins/docker_prepare_interop_tests.sh | 79 +++++++++ ...local.sh => docker_run_interop_servers.sh} | 35 ++-- tools/jenkins/docker_run_interop_tests.sh | 49 +----- tools/jenkins/docker_run_tests.sh | 2 +- tools/jenkins/run_distribution.sh | 2 +- tools/jenkins/run_jenkins.sh | 10 +- tools/run_tests/run_interop_tests.py | 163 ++++++++++++++++-- tools/run_tests/run_tests.py | 2 +- 10 files changed, 365 insertions(+), 90 deletions(-) create mode 100755 tools/jenkins/build_docker_and_run_interop_tests.sh create mode 100755 tools/jenkins/docker_prepare_interop_tests.sh rename tools/jenkins/{run_local.sh => docker_run_interop_servers.sh} (63%) diff --git a/src/csharp/Grpc.IntegrationTesting/InteropClient.cs b/src/csharp/Grpc.IntegrationTesting/InteropClient.cs index 616093d4aeb..504d798b893 100644 --- a/src/csharp/Grpc.IntegrationTesting/InteropClient.cs +++ b/src/csharp/Grpc.IntegrationTesting/InteropClient.cs @@ -126,8 +126,6 @@ namespace Grpc.IntegrationTesting new ChannelOption(ChannelOptions.SslTargetNameOverride, options.ServerHostOverride) }; } - Console.WriteLine(options.ServerHost); - Console.WriteLine(options.ServerPort); var channel = new Channel(options.ServerHost, options.ServerPort, credentials, channelOptions); TestService.TestServiceClient client = new TestService.TestServiceClient(channel); await RunTestCaseAsync(client, options); diff --git a/tools/jenkins/build_docker_and_run_interop_tests.sh b/tools/jenkins/build_docker_and_run_interop_tests.sh new file mode 100755 index 00000000000..d2d56d947bf --- /dev/null +++ b/tools/jenkins/build_docker_and_run_interop_tests.sh @@ -0,0 +1,111 @@ +#!/bin/bash +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# This script is invoked by run_interop_tests.py to accommodate +# "interop tests under docker" scenario. You should never need to call this +# script on your own. + +set -ex + +cd `dirname $0`/../.. +git_root=`pwd` +cd - + +mkdir -p /tmp/ccache + +# Use image name based on Dockerfile checksum +DOCKER_IMAGE_NAME=grpc_jenkins_slave${docker_suffix}_`sha1sum tools/jenkins/grpc_jenkins_slave/Dockerfile | cut -f1 -d\ ` + +# Make sure docker image has been built. Should be instantaneous if so. +docker build -t $DOCKER_IMAGE_NAME tools/jenkins/grpc_jenkins_slave$docker_suffix + +# Create a local branch so the child Docker script won't complain +git branch -f jenkins-docker + +# Make sure the CID files are gone. +rm -f prepare.cid server.cid client.cid + +# Prepare image for interop tests +docker run \ + -e CCACHE_DIR=/tmp/ccache \ + -i $TTY_FLAG \ + -v "$git_root:/var/local/jenkins/grpc" \ + -v /tmp/ccache:/tmp/ccache \ + --cidfile=prepare.cid \ + $DOCKER_IMAGE_NAME \ + bash -l /var/local/jenkins/grpc/tools/jenkins/docker_prepare_interop_tests.sh || DOCKER_FAILED="true" + +PREPARE_CID=`cat prepare.cid` + +# Create image from the container, we will spawn one docker for clients +# and one for servers. +INTEROP_IMAGE=interop_`uuidgen` +docker commit $PREPARE_CID $INTEROP_IMAGE +# remove container, possibly killing it first +docker rm -f $PREPARE_CID || true +echo "Successfully built image $INTEROP_IMAGE" + +# run interop servers under docker in the background +docker run \ + -d -i \ + $SERVERS_DOCKER_EXTRA_ARGS \ + --cidfile=server.cid \ + $INTEROP_IMAGE bash -l /var/local/git/grpc/tools/jenkins/docker_run_interop_servers.sh + +SERVER_CID=`cat server.cid` + +SERVER_PORTS="" +for tuple in $SERVER_PORT_TUPLES +do + # lookup under which port docker exposes given internal port + exposed_port=`docker port $SERVER_CID ${tuple#*:} | awk -F ":" '{print $NF}'` + + # override the port for corresponding cloud_to_cloud server + SERVER_PORTS+=" --override_server ${tuple%:*}=localhost:$exposed_port" + echo "${tuple%:*} server is exposed under port $exposed_port" +done + +# run interop clients +docker run \ + -e "RUN_TESTS_COMMAND=$RUN_TESTS_COMMAND $SERVER_PORTS" \ + -w /var/local/git/grpc \ + -i $TTY_FLAG \ + --net=host \ + --cidfile=client.cid \ + $INTEROP_IMAGE bash -l /var/local/git/grpc/tools/jenkins/docker_run_interop_tests.sh || DOCKER_FAILED="true" + +CLIENT_CID=`cat client.cid` + +echo "killing and removing server container $SERVER_CID" +docker rm -f $SERVER_CID || true + +docker cp $CLIENT_CID:/var/local/git/grpc/report.xml $git_root +docker rm -f $CLIENT_CID || true +docker rmi -f $DOCKER_IMAGE_NAME || true diff --git a/tools/jenkins/docker_prepare_interop_tests.sh b/tools/jenkins/docker_prepare_interop_tests.sh new file mode 100755 index 00000000000..a1fe0b72d2f --- /dev/null +++ b/tools/jenkins/docker_prepare_interop_tests.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# Copyright 2015, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# This script is invoked by run_jekins.sh. It contains the test logic +# that should run inside a docker container. +set -e + +mkdir -p /var/local/git +git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc + +cd /var/local/git/grpc +nvm use 0.12 +rvm use ruby-2.1 + +# TODO(jtattermusch): use cleaner way to install root certs +mkdir -p /usr/local/share/grpc +cp etc/roots.pem /usr/local/share/grpc/ + +# build C++ interop client & server +make interop_client interop_server + +# build C# interop client & server +make install_grpc_csharp_ext +(cd src/csharp && mono /var/local/NuGet.exe restore Grpc.sln) +(cd src/csharp && xbuild Grpc.sln) + +# build Node interop client & server +npm install -g node-gyp +make install_c -C /var/local/git/grpc +(cd src/node && npm install && node-gyp rebuild) + +# build Ruby interop client and server +(cd src/ruby && gem update bundler && bundle && rake compile:grpc) + +# TODO(jtattermusch): add python + +# build PHP interop client +# TODO(jtattermusch): prerequisites for PHP should be installed sooner than here. +# Install composer +curl -sS https://getcomposer.org/installer | php +mv composer.phar /usr/local/bin/composer +# Download the patched PHP protobuf so that PHP gRPC clients can be generated +# from proto3 schemas. +git clone https://github.com/stanley-cheung/Protobuf-PHP.git /var/local/git/protobuf-php +(cd src/php/ext/grpc && phpize && ./configure && make) +rvm all do gem install ronn rake +(cd third_party/protobuf && make install) +(cd /var/local/git/protobuf-php \ + && rvm all do rake pear:package version=1.0 \ + && pear install Protobuf-1.0.tgz) +(cd src/php && composer install) +(cd src/php && protoc-gen-php -i tests/interop/ -o tests/interop/ tests/interop/test.proto) diff --git a/tools/jenkins/run_local.sh b/tools/jenkins/docker_run_interop_servers.sh similarity index 63% rename from tools/jenkins/run_local.sh rename to tools/jenkins/docker_run_interop_servers.sh index 4891637bbf9..9f29a65aaa4 100755 --- a/tools/jenkins/run_local.sh +++ b/tools/jenkins/docker_run_interop_servers.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # Copyright 2015, Google Inc. # All rights reserved. # @@ -28,32 +28,23 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -# This script can be used to run dockerized tests that normally run -# on Jenkins on your local machine using the working copy that -# is currently checked out locally. - -# IMPORTANT: The changes to be tested need to be committed locally, -# otherwise they won't be cloned inside the docker container. +# This script is invoked by run_jekins.sh. It contains the test logic +# that should run inside a docker container. set -e -cd `dirname $0`/../.. +cd /var/local/git/grpc +nvm use 0.12 +rvm use ruby-2.1 -#TODO(jtattermusch): provide way to tunnel run_tests cmdline options to run_tests. -#TODO(jtattermusch): provide way to grab the docker image built by run_jenkins +# If port env variable is set, run corresponding interop server on given port in background. +# TODO(jtattermusch): ideally, run_interop_tests.py would generate the commands to run servers. -# config: opt or dbg -export config=opt +[ -z "${SERVER_PORT_cxx}" ] || bins/opt/interop_server --enable_ssl --port=${SERVER_PORT_cxx} & -# platform: -# -- use linux to run tests under docker -# -- use interop to run dockerized interop tests -export platform=interop +[ -z "${SERVER_PORT_node}" ] || node src/node/interop/interop_server.js --use_tls=true --port=${SERVER_PORT_node} & -# language: one of languages supported by run_tests.py or run_interop_tests.py -export language=all +[ -z "${SERVER_PORT_ruby}" ] || ruby src/ruby/bin/interop/interop_server.rb --use_tls --port=${SERVER_PORT_ruby} & -# architecture -export arch=`uname -m` +[ -z "${SERVER_PORT_csharp}" ] || (cd src/csharp/Grpc.IntegrationTesting.Server/bin/Debug && mono Grpc.IntegrationTesting.Server.exe --use_tls --port=${SERVER_PORT_csharp}) & -# test run configuration is done through environment variables above -tools/jenkins/run_jenkins.sh +sleep infinity diff --git a/tools/jenkins/docker_run_interop_tests.sh b/tools/jenkins/docker_run_interop_tests.sh index d363d1c68cc..29970afb258 100755 --- a/tools/jenkins/docker_run_interop_tests.sh +++ b/tools/jenkins/docker_run_interop_tests.sh @@ -28,55 +28,12 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -# This script is invoked by run_jekins.sh. It contains the test logic -# that should run inside a docker container. +# This script is invoked by build_docker_and_run_interop_tests.sh inside +# a docker container. You should never need to call this script on your own. set -e -mkdir -p /var/local/git -git clone --recursive /var/local/jenkins/grpc /var/local/git/grpc - -cd /var/local/git/grpc nvm use 0.12 rvm use ruby-2.1 -# TODO(jtattermusch): use cleaner way to install root certs -mkdir -p /usr/local/share/grpc -cp etc/roots.pem /usr/local/share/grpc/ - -# build C++ interop client & server -make interop_client interop_server - -# build C# interop client & server -make install_grpc_csharp_ext -(cd src/csharp && mono /var/local/NuGet.exe restore Grpc.sln) -(cd src/csharp && xbuild Grpc.sln) - -# build Node interop client & server -npm install -g node-gyp -make install_c -C /var/local/git/grpc -(cd src/node && npm install && node-gyp rebuild) - -# build Ruby interop client and server -(cd src/ruby && gem update bundler && bundle && rake compile:grpc) - -# TODO(jtattermusch): add python - -# build PHP interop client -# TODO(jtattermusch): prerequisites for PHP should be installed sooner than here. -# Install composer -curl -sS https://getcomposer.org/installer | php -mv composer.phar /usr/local/bin/composer -# Download the patched PHP protobuf so that PHP gRPC clients can be generated -# from proto3 schemas. -git clone https://github.com/stanley-cheung/Protobuf-PHP.git /var/local/git/protobuf-php -(cd src/php/ext/grpc && phpize && ./configure && make) -rvm all do gem install ronn rake -(cd third_party/protobuf && make install) -(cd /var/local/git/protobuf-php \ - && rvm all do rake pear:package version=1.0 \ - && pear install Protobuf-1.0.tgz) -(cd src/php && composer install) -(cd src/php && protoc-gen-php -i tests/interop/ -o tests/interop/ tests/interop/test.proto) - # run the cloud-to-prod interop tests -tools/run_tests/run_interop_tests.py -l $language +$RUN_TESTS_COMMAND diff --git a/tools/jenkins/docker_run_tests.sh b/tools/jenkins/docker_run_tests.sh index 7877ad2459e..3595a95f5ce 100755 --- a/tools/jenkins/docker_run_tests.sh +++ b/tools/jenkins/docker_run_tests.sh @@ -28,7 +28,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # -# This script is invoked by build_docker_and_run_tests.py inside a docker +# This script is invoked by build_docker_and_run_tests.sh inside a docker # container. You should never need to call this script on your own. set -e diff --git a/tools/jenkins/run_distribution.sh b/tools/jenkins/run_distribution.sh index 49b7d306d16..64c60f15027 100755 --- a/tools/jenkins/run_distribution.sh +++ b/tools/jenkins/run_distribution.sh @@ -54,7 +54,7 @@ if [ "$platform" == "linux" ]; then docker build -t $DOCKER_IMAGE_NAME tools/jenkins/grpc_linuxbrew # run per-language homebrew installation script - docker run $DOCKER_IMAGE_NAME bash -l \ + docker run --rm=true $DOCKER_IMAGE_NAME bash -l \ -c "nvm use 0.12; \ npm set unsafe-perm true; \ rvm use ruby-2.1; \ diff --git a/tools/jenkins/run_jenkins.sh b/tools/jenkins/run_jenkins.sh index 66364b3093d..4bc18528236 100755 --- a/tools/jenkins/run_jenkins.sh +++ b/tools/jenkins/run_jenkins.sh @@ -56,10 +56,6 @@ then ./tools/run_tests/run_tests.py --use_docker -t -l $language -c $config -x report.xml $@ || true -elif [ "$platform" == "interop" ] -then - python tools/run_tests/run_interops.py --language=$language $@ - elif [ "$platform" == "windows" ] then echo "building $language on Windows" @@ -84,6 +80,12 @@ then echo "building $language on FreeBSD" MAKE=gmake ./tools/run_tests/run_tests.py -t -l $language -c $config -x report.xml $@ || true + +elif [ "$platform" == "interop" ] +then + echo "building interop tests for language $language" + + ./tools/run_tests/run_interop_tests.py --use_docker -t -l $language --cloud_to_prod --server all $@ || true else echo "Unknown platform $platform" exit 1 diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index e9c4ce0a0f9..a2fb1243cc4 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -34,6 +34,10 @@ import argparse import itertools import xml.etree.cElementTree as ET import jobset +import os +import subprocess +import sys +import time _CLOUD_TO_PROD_BASE_ARGS = [ @@ -41,6 +45,9 @@ _CLOUD_TO_PROD_BASE_ARGS = [ '--server_host=grpc-test.sandbox.google.com', '--server_port=443'] +_CLOUD_TO_CLOUD_BASE_ARGS = [ + '--server_host_override=foo.test.google.fr'] + # TOOD(jtattermusch) wrapped languages use this variable for location # of roots.pem. We might want to use GRPC_DEFAULT_SSL_ROOTS_FILE_PATH # supported by C core SslCredentials instead. @@ -48,6 +55,7 @@ _SSL_CERT_ENV = { 'SSL_CERT_FILE':'/usr/local/share/grpc/roots.pem' } # TODO(jtatttermusch) unify usage of --enable_ssl, --use_tls and --use_tls=true + class CXXLanguage: def __init__(self): @@ -58,6 +66,10 @@ class CXXLanguage: return (self.client_cmdline_base + _CLOUD_TO_PROD_BASE_ARGS + ['--enable_ssl','--use_prod_roots']) + def cloud_to_cloud_args(self): + return (self.client_cmdline_base + _CLOUD_TO_CLOUD_BASE_ARGS + + ['--enable_ssl']) + def cloud_to_prod_env(self): return None @@ -75,6 +87,10 @@ class CSharpLanguage: return (self.client_cmdline_base + _CLOUD_TO_PROD_BASE_ARGS + ['--use_tls']) + def cloud_to_cloud_args(self): + return (self.client_cmdline_base + _CLOUD_TO_CLOUD_BASE_ARGS + + ['--use_tls', '--use_test_ca']) + def cloud_to_prod_env(self): return _SSL_CERT_ENV @@ -92,6 +108,10 @@ class NodeLanguage: return (self.client_cmdline_base + _CLOUD_TO_PROD_BASE_ARGS + ['--use_tls=true']) + def cloud_to_cloud_args(self): + return (self.client_cmdline_base + _CLOUD_TO_CLOUD_BASE_ARGS + + ['--use_tls=true', '--use_test_ca=true']) + def cloud_to_prod_env(self): return _SSL_CERT_ENV @@ -109,6 +129,10 @@ class PHPLanguage: return (self.client_cmdline_base + _CLOUD_TO_PROD_BASE_ARGS + ['--use_tls']) + def cloud_to_cloud_args(self): + return (self.client_cmdline_base + _CLOUD_TO_CLOUD_BASE_ARGS + + ['--use_tls', '--use_test_ca']) + def cloud_to_prod_env(self): return _SSL_CERT_ENV @@ -126,6 +150,10 @@ class RubyLanguage: return (self.client_cmdline_base + _CLOUD_TO_PROD_BASE_ARGS + ['--use_tls']) + def cloud_to_cloud_args(self): + return (self.client_cmdline_base + _CLOUD_TO_CLOUD_BASE_ARGS + + ['--use_tls', '--use_test_ca']) + def cloud_to_prod_env(self): return _SSL_CERT_ENV @@ -142,6 +170,10 @@ _LANGUAGES = { 'ruby' : RubyLanguage(), } +# languages supported as cloud_to_cloud servers +# TODO(jtattermusch): enable other languages as servers as well +_SERVERS = { 'c++' : 8010, 'node' : 8040, 'csharp': 8070 } + # TODO(jtattermusch): add empty_stream once C++ start supporting it. # TODO(jtattermusch): add support for auth tests. _TEST_CASES = ['large_unary', 'empty_unary', 'ping_pong', @@ -149,6 +181,7 @@ _TEST_CASES = ['large_unary', 'empty_unary', 'ping_pong', 'cancel_after_begin', 'cancel_after_first_response', 'timeout_on_sleeping_server'] + def cloud_to_prod_jobspec(language, test_case): """Creates jobspec for cloud-to-prod interop test""" cmdline = language.cloud_to_prod_args() + ['--test_case=%s' % test_case] @@ -160,34 +193,138 @@ def cloud_to_prod_jobspec(language, test_case): timeout_seconds=60) return test_job + +def cloud_to_cloud_jobspec(language, test_case, server_name, server_host, + server_port): + """Creates jobspec for cloud-to-cloud interop test""" + cmdline = language.cloud_to_cloud_args() + ['--test_case=%s' % test_case, + '--server_host=%s' % server_host, + '--server_port=%s' % server_port ] + test_job = jobset.JobSpec( + cmdline=cmdline, + cwd=language.client_cwd, + shortname="cloud_to_cloud:%s:%s_server:%s" % (language, server_name, + test_case), + timeout_seconds=60) + return test_job + argp = argparse.ArgumentParser(description='Run interop tests.') argp.add_argument('-l', '--language', choices=['all'] + sorted(_LANGUAGES), nargs='+', - default=['all']) + default=['all'], + help='Clients to run.') +argp.add_argument('-j', '--jobs', default=24, type=int) +argp.add_argument('--cloud_to_prod', + default=False, + action='store_const', + const=True, + help='Run cloud_to_prod tests.') +argp.add_argument('-s', '--server', + choices=['all'] + sorted(_SERVERS), + action='append', + help='Run cloud_to_cloud servers in a separate docker ' + + 'image. Servers can only be started automatically if ' + + '--use_docker option is enabled.', + default=[]) +argp.add_argument('--override_server', + action='append', + type=lambda kv: kv.split("="), + help='Use servername=HOST:PORT to explicitly specify a server. E.g. csharp=localhost:50000', + default=[]) +argp.add_argument('-t', '--travis', + default=False, + action='store_const', + const=True) +argp.add_argument('--use_docker', + default=False, + action='store_const', + const=True, + help='Run all the interop tests under docker. That provides ' + + 'additional isolation and prevents the need to install ' + + 'language specific prerequisites. Only available on Linux.') args = argp.parse_args() +servers = set(s for s in itertools.chain.from_iterable(_SERVERS.iterkeys() + if x == 'all' else [x] + for x in args.server)) + +if args.use_docker: + if not args.travis: + print 'Seen --use_docker flag, will run interop tests under docker.' + print + print 'IMPORTANT: The changes you are testing need to be locally committed' + print 'because only the committed changes in the current branch will be' + print 'copied to the docker environment.' + time.sleep(5) + + child_argv = [ arg for arg in sys.argv if not arg == '--use_docker' ] + run_tests_cmd = ('tools/run_tests/run_interop_tests.py %s' % + " ".join(child_argv[1:])) + + # cmdline args to pass to the container running servers. + servers_extra_docker_args = '' + server_port_tuples = '' + for server in servers: + port = _SERVERS[server] + servers_extra_docker_args += ' -p %s' % port + servers_extra_docker_args += ' -e SERVER_PORT_%s=%s' % (server.replace("+", "x"), port) + server_port_tuples += ' %s:%s' % (server, port) + + env = os.environ.copy() + env['RUN_TESTS_COMMAND'] = run_tests_cmd + env['SERVERS_DOCKER_EXTRA_ARGS'] = servers_extra_docker_args + env['SERVER_PORT_TUPLES'] = server_port_tuples + if not args.travis: + env['TTY_FLAG'] = '-t' # enables Ctrl-C when not on Jenkins. + + subprocess.check_call(['tools/jenkins/build_docker_and_run_interop_tests.sh'], + shell=True, + env=env) + sys.exit(0) + languages = set(_LANGUAGES[l] for l in itertools.chain.from_iterable( _LANGUAGES.iterkeys() if x == 'all' else [x] for x in args.language)) -# TODO(jtattermusch): make python script generate cmdline params for interop -# tests. It's easier to manage than in a shell script. jobs = [] -jobNumber = 0 -for language in languages: - for test_case in _TEST_CASES: - test_job = cloud_to_prod_jobspec(language, test_case) - jobs.append(test_job) - jobNumber+=1 +if args.cloud_to_prod: + for language in languages: + for test_case in _TEST_CASES: + test_job = cloud_to_prod_jobspec(language, test_case) + jobs.append(test_job) + +# default servers to "localhost" and the default port +server_addresses = dict((s, ("localhost", _SERVERS[s])) for s in servers) + +for server in args.override_server: + server_name = server[0] + (server_host, server_port) = server[1].split(":") + server_addresses[server_name] = (server_host, server_port) + +for server_name, server_address in server_addresses.iteritems(): + (server_host, server_port) = server_address + for language in languages: + for test_case in _TEST_CASES: + test_job = cloud_to_cloud_jobspec(language, + test_case, + server_name, + server_host, + server_port) + jobs.append(test_job) + +if not jobs: + print "No jobs to run." + sys.exit(1) root = ET.Element('testsuites') testsuite = ET.SubElement(root, 'testsuite', id='1', package='grpc', name='tests') -jobset.run(jobs, maxjobs=jobNumber, xml_report=testsuite) +if jobset.run(jobs, newline_on_success=True, maxjobs=args.jobs, xml_report=testsuite): + jobset.message('SUCCESS', 'All tests passed', do_newline=True) +else: + jobset.message('FAILED', 'Some tests failed', do_newline=True) tree = ET.ElementTree(root) -tree.write('report.xml', encoding='UTF-8') - - +tree.write('report.xml', encoding='UTF-8') \ No newline at end of file diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index cbbb58d7e72..b2370537e5c 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -556,7 +556,7 @@ argp.add_argument('--use_docker', action='store_const', const=True, help="Run all the tests under docker. That provides " + - "additional isolation and prevents the need to installs " + + "additional isolation and prevents the need to install " + "language specific prerequisites. Only available on Linux.") argp.add_argument('--allow_flakes', default=False, From bd7682fe9954200e9806fd6714ca420fd6d35916 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 28 Sep 2015 15:07:48 -0700 Subject: [PATCH 33/48] dont propagate --allow_flakes to run_interop_tests.py --- tools/jenkins/run_jenkins.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/jenkins/run_jenkins.sh b/tools/jenkins/run_jenkins.sh index 4bc18528236..f79e739f6ae 100755 --- a/tools/jenkins/run_jenkins.sh +++ b/tools/jenkins/run_jenkins.sh @@ -85,7 +85,7 @@ elif [ "$platform" == "interop" ] then echo "building interop tests for language $language" - ./tools/run_tests/run_interop_tests.py --use_docker -t -l $language --cloud_to_prod --server all $@ || true + ./tools/run_tests/run_interop_tests.py --use_docker -t -l $language --cloud_to_prod --server all || true else echo "Unknown platform $platform" exit 1 From fa6e70ff852de3276af32b597bf0cd05691c0e9d Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Mon, 28 Sep 2015 15:56:07 -0700 Subject: [PATCH 34/48] Remove one pessimizing std::move --- test/cpp/end2end/streaming_throughput_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/end2end/streaming_throughput_test.cc b/test/cpp/end2end/streaming_throughput_test.cc index d64d96fe939..c1355b38f0f 100644 --- a/test/cpp/end2end/streaming_throughput_test.cc +++ b/test/cpp/end2end/streaming_throughput_test.cc @@ -145,7 +145,7 @@ class End2endTest : public ::testing::Test { void ResetStub() { std::shared_ptr channel = CreateChannel( server_address_.str(), InsecureCredentials()); - stub_ = std::move(grpc::cpp::test::util::TestService::NewStub(channel)); + stub_ = grpc::cpp::test::util::TestService::NewStub(channel); } std::unique_ptr stub_; From 114bda10906a602907fd9e469b9583a9e219fa47 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 29 Sep 2015 08:37:09 -0700 Subject: [PATCH 35/48] Fix rounding error so IOCP completes on time --- src/core/iomgr/iocp_windows.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/iomgr/iocp_windows.c b/src/core/iomgr/iocp_windows.c index af846f680f7..349440fa88c 100644 --- a/src/core/iomgr/iocp_windows.c +++ b/src/core/iomgr/iocp_windows.c @@ -68,7 +68,7 @@ static DWORD deadline_to_millis_timeout(gpr_timespec deadline, } timeout = gpr_time_sub(deadline, now); return gpr_time_to_millis(gpr_time_add( - timeout, gpr_time_from_nanos(GPR_NS_PER_SEC - 1, GPR_TIMESPAN))); + timeout, gpr_time_from_nanos(GPR_NS_PER_MS - 1, GPR_TIMESPAN))); } void grpc_iocp_work(grpc_exec_ctx *exec_ctx, gpr_timespec deadline) { From c3e577805d7538d1d5e1f2610387744651b0507f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 29 Sep 2015 08:37:32 -0700 Subject: [PATCH 36/48] Properly initialize variables, destroy variables in usage order --- src/core/iomgr/pollset_windows.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core/iomgr/pollset_windows.c b/src/core/iomgr/pollset_windows.c index cb0aad0e33c..acdef9796ce 100644 --- a/src/core/iomgr/pollset_windows.c +++ b/src/core/iomgr/pollset_windows.c @@ -137,6 +137,8 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, int added_worker = 0; worker->links[GRPC_POLLSET_WORKER_LINK_POLLSET].next = worker->links[GRPC_POLLSET_WORKER_LINK_POLLSET].prev = + worker->links[GRPC_POLLSET_WORKER_LINK_GLOBAL].next = + worker->links[GRPC_POLLSET_WORKER_LINK_GLOBAL].prev = NULL; gpr_cv_init(&worker->cv); if (grpc_alarm_check(exec_ctx, now, &deadline)) { @@ -182,11 +184,11 @@ done: grpc_exec_ctx_flush(exec_ctx); gpr_mu_lock(&pollset->mu); } - gpr_cv_destroy(&worker->cv); if (added_worker) { remove_worker(worker, GRPC_POLLSET_WORKER_LINK_GLOBAL); remove_worker(worker, GRPC_POLLSET_WORKER_LINK_POLLSET); } + gpr_cv_destroy(&worker->cv); } void grpc_pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) { From 3ccd9613dab3c4de1e679ef2fc8b01ac24d4fbc0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 29 Sep 2015 11:17:54 -0700 Subject: [PATCH 37/48] Ensure GrpcEnvironment setup for these (special) tests --- .../Grpc.Core.Tests/Internal/CompletionQueueSafeHandleTest.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/csharp/Grpc.Core.Tests/Internal/CompletionQueueSafeHandleTest.cs b/src/csharp/Grpc.Core.Tests/Internal/CompletionQueueSafeHandleTest.cs index a2ee1832724..c6843f10af2 100644 --- a/src/csharp/Grpc.Core.Tests/Internal/CompletionQueueSafeHandleTest.cs +++ b/src/csharp/Grpc.Core.Tests/Internal/CompletionQueueSafeHandleTest.cs @@ -45,17 +45,21 @@ namespace Grpc.Core.Internal.Tests [Test] public void CreateAndDestroy() { + GrpcEnvironment.AddRef(); var cq = CompletionQueueSafeHandle.Create(); cq.Dispose(); + GrpcEnvironment.Release(); } [Test] public void CreateAndShutdown() { + GrpcEnvironment.AddRef(); var cq = CompletionQueueSafeHandle.Create(); cq.Shutdown(); var ev = cq.Next(); cq.Dispose(); + GrpcEnvironment.Release(); Assert.AreEqual(GRPCCompletionType.Shutdown, ev.type); Assert.AreNotEqual(IntPtr.Zero, ev.success); Assert.AreEqual(IntPtr.Zero, ev.tag); From 084c8089a4ca3f9e1a5516751a9562b902926f3b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 29 Sep 2015 11:19:06 -0700 Subject: [PATCH 38/48] Threading fixes - single global mutex (simpler, easier to make correct for now) - properly flag kick state for workers to avoid missing wakeups (if signal is called before wait on the cv) --- src/core/iomgr/pollset_windows.c | 41 +++++++++++++++++--------------- src/core/iomgr/pollset_windows.h | 14 +++++++---- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/src/core/iomgr/pollset_windows.c b/src/core/iomgr/pollset_windows.c index acdef9796ce..dd5301f5be9 100644 --- a/src/core/iomgr/pollset_windows.c +++ b/src/core/iomgr/pollset_windows.c @@ -43,12 +43,12 @@ #include "src/core/iomgr/pollset.h" #include "src/core/iomgr/pollset_windows.h" -static gpr_mu g_polling_mu; +gpr_mu grpc_polling_mu; static grpc_pollset_worker *g_active_poller; static grpc_pollset_worker g_global_root_worker; void grpc_pollset_global_init() { - gpr_mu_init(&g_polling_mu); + gpr_mu_init(&grpc_polling_mu); g_active_poller = NULL; g_global_root_worker.links[GRPC_POLLSET_WORKER_LINK_GLOBAL].next = g_global_root_worker.links[GRPC_POLLSET_WORKER_LINK_GLOBAL].prev = @@ -56,7 +56,7 @@ void grpc_pollset_global_init() { } void grpc_pollset_global_shutdown() { - gpr_mu_destroy(&g_polling_mu); + gpr_mu_destroy(&grpc_polling_mu); } static void remove_worker(grpc_pollset_worker *worker, @@ -108,7 +108,6 @@ static void push_front_worker(grpc_pollset_worker *root, void grpc_pollset_init(grpc_pollset *pollset) { memset(pollset, 0, sizeof(*pollset)); - gpr_mu_init(&pollset->mu); pollset->root_worker.links[GRPC_POLLSET_WORKER_LINK_POLLSET].next = pollset->root_worker.links[GRPC_POLLSET_WORKER_LINK_POLLSET].prev = &pollset->root_worker; @@ -116,7 +115,7 @@ void grpc_pollset_init(grpc_pollset *pollset) { void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, grpc_closure *closure) { - gpr_mu_lock(&pollset->mu); + gpr_mu_lock(&grpc_polling_mu); pollset->shutting_down = 1; grpc_pollset_kick(pollset, GRPC_POLLSET_KICK_BROADCAST); if (!pollset->is_iocp_worker) { @@ -124,11 +123,10 @@ void grpc_pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, } else { pollset->on_shutdown = closure; } - gpr_mu_unlock(&pollset->mu); + gpr_mu_unlock(&grpc_polling_mu); } void grpc_pollset_destroy(grpc_pollset *pollset) { - gpr_mu_destroy(&pollset->mu); } void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, @@ -140,29 +138,31 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, worker->links[GRPC_POLLSET_WORKER_LINK_GLOBAL].next = worker->links[GRPC_POLLSET_WORKER_LINK_GLOBAL].prev = NULL; + worker->kicked = 0; + worker->pollset = pollset; gpr_cv_init(&worker->cv); if (grpc_alarm_check(exec_ctx, now, &deadline)) { goto done; } if (!pollset->kicked_without_pollers && !pollset->shutting_down) { - gpr_mu_lock(&g_polling_mu); if (g_active_poller == NULL) { grpc_pollset_worker *next_worker; /* become poller */ pollset->is_iocp_worker = 1; g_active_poller = worker; - gpr_mu_unlock(&g_polling_mu); - gpr_mu_unlock(&pollset->mu); + gpr_mu_unlock(&grpc_polling_mu); grpc_iocp_work(exec_ctx, deadline); - gpr_mu_lock(&pollset->mu); - gpr_mu_lock(&g_polling_mu); + gpr_mu_lock(&grpc_polling_mu); pollset->is_iocp_worker = 0; g_active_poller = NULL; + /* try to get a worker from this pollsets worker list */ + next_worker = pop_front_worker(&pollset->root_worker, GRPC_POLLSET_WORKER_LINK_POLLSET); + /* try to get a worker from the global list */ next_worker = pop_front_worker(&g_global_root_worker, GRPC_POLLSET_WORKER_LINK_GLOBAL); if (next_worker != NULL) { + next_worker->kicked = 1; gpr_cv_signal(&next_worker->cv); } - gpr_mu_unlock(&g_polling_mu); if (pollset->shutting_down && pollset->on_shutdown != NULL) { grpc_exec_ctx_enqueue(exec_ctx, pollset->on_shutdown, 1); @@ -171,18 +171,21 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, goto done; } push_front_worker(&g_global_root_worker, GRPC_POLLSET_WORKER_LINK_GLOBAL, worker); - gpr_mu_unlock(&g_polling_mu); push_front_worker(&pollset->root_worker, GRPC_POLLSET_WORKER_LINK_POLLSET, worker); added_worker = 1; - gpr_cv_wait(&worker->cv, &pollset->mu, deadline); + while (!worker->kicked) { + if (gpr_cv_wait(&worker->cv, &grpc_polling_mu, deadline)) { + break; + } + } } else { pollset->kicked_without_pollers = 0; } done: if (!grpc_closure_list_empty(exec_ctx->closure_list)) { - gpr_mu_unlock(&pollset->mu); + gpr_mu_unlock(&grpc_polling_mu); grpc_exec_ctx_flush(exec_ctx); - gpr_mu_lock(&pollset->mu); + gpr_mu_lock(&grpc_polling_mu); } if (added_worker) { remove_worker(worker, GRPC_POLLSET_WORKER_LINK_GLOBAL); @@ -197,6 +200,7 @@ void grpc_pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) { for (specific_worker = p->root_worker.links[GRPC_POLLSET_WORKER_LINK_POLLSET].next; specific_worker != &p->root_worker; specific_worker = specific_worker->links[GRPC_POLLSET_WORKER_LINK_POLLSET].next) { + specific_worker->kicked = 1; gpr_cv_signal(&specific_worker->cv); } p->kicked_without_pollers = 1; @@ -205,12 +209,11 @@ void grpc_pollset_kick(grpc_pollset *p, grpc_pollset_worker *specific_worker) { } } else { if (p->is_iocp_worker) { - gpr_mu_lock(&g_polling_mu); if (g_active_poller == specific_worker) { grpc_iocp_kick(); } - gpr_mu_unlock(&g_polling_mu); } else { + specific_worker->kicked = 1; gpr_cv_signal(&specific_worker->cv); } } diff --git a/src/core/iomgr/pollset_windows.h b/src/core/iomgr/pollset_windows.h index 55f87aca729..65ba80619b7 100644 --- a/src/core/iomgr/pollset_windows.h +++ b/src/core/iomgr/pollset_windows.h @@ -54,20 +54,26 @@ typedef struct grpc_pollset_worker_link { struct grpc_pollset_worker *prev; } grpc_pollset_worker_link; +struct grpc_pollset; +typedef struct grpc_pollset grpc_pollset; + typedef struct grpc_pollset_worker { gpr_cv cv; + int kicked; + struct grpc_pollset *pollset; grpc_pollset_worker_link links[GRPC_POLLSET_WORKER_LINK_TYPES]; } grpc_pollset_worker; -typedef struct grpc_pollset { - gpr_mu mu; +struct grpc_pollset { int shutting_down; int kicked_without_pollers; int is_iocp_worker; grpc_pollset_worker root_worker; grpc_closure *on_shutdown; -} grpc_pollset; +}; + +extern gpr_mu grpc_polling_mu; -#define GRPC_POLLSET_MU(pollset) (&(pollset)->mu) +#define GRPC_POLLSET_MU(pollset) (&grpc_polling_mu) #endif /* GRPC_INTERNAL_CORE_IOMGR_POLLSET_WINDOWS_H */ From a8e2e5263db11e7946e2b7fc249a94cca0f49089 Mon Sep 17 00:00:00 2001 From: Robbie Shade Date: Tue, 29 Sep 2015 14:19:45 -0400 Subject: [PATCH 39/48] Pass through grpc_fd pointer rather than int. --- src/core/iomgr/udp_server.c | 2 +- src/core/iomgr/udp_server.h | 2 +- test/core/iomgr/udp_server_test.c | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/iomgr/udp_server.c b/src/core/iomgr/udp_server.c index 9baaf1edc78..d884359aa43 100644 --- a/src/core/iomgr/udp_server.c +++ b/src/core/iomgr/udp_server.c @@ -278,7 +278,7 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, int success) { /* Tell the registered callback that data is available to read. */ GPR_ASSERT(sp->read_cb); - sp->read_cb(sp->fd, sp->server->grpc_server); + sp->read_cb(sp->emfd, sp->server->grpc_server); /* Re-arm the notification event so we get another chance to read. */ grpc_fd_notify_on_read(exec_ctx, sp->emfd, &sp->read_closure); diff --git a/src/core/iomgr/udp_server.h b/src/core/iomgr/udp_server.h index 8e3abae864b..dbbe0971093 100644 --- a/src/core/iomgr/udp_server.h +++ b/src/core/iomgr/udp_server.h @@ -43,7 +43,7 @@ typedef struct grpc_server grpc_server; typedef struct grpc_udp_server grpc_udp_server; /* Called when data is available to read from the socket. */ -typedef void (*grpc_udp_server_read_cb)(int fd, grpc_server *server); +typedef void (*grpc_udp_server_read_cb)(grpc_fd *emfd, grpc_server *server); /* Create a server, initially not bound to any ports */ grpc_udp_server *grpc_udp_server_create(void); diff --git a/test/core/iomgr/udp_server_test.c b/test/core/iomgr/udp_server_test.c index 6d3dfeeb57c..fc0026da4df 100644 --- a/test/core/iomgr/udp_server_test.c +++ b/test/core/iomgr/udp_server_test.c @@ -49,12 +49,12 @@ static grpc_pollset g_pollset; static int g_number_of_reads = 0; static int g_number_of_bytes_read = 0; -static void on_read(int fd, grpc_server *server) { +static void on_read(grpc_fd *emfd, grpc_server *server) { char read_buffer[512]; ssize_t byte_count; gpr_mu_lock(GRPC_POLLSET_MU(&g_pollset)); - byte_count = recv(fd, read_buffer, sizeof(read_buffer), 0); + byte_count = recv(emfd->fd, read_buffer, sizeof(read_buffer), 0); g_number_of_reads++; g_number_of_bytes_read += (int)byte_count; From 7d3ea5965d34323323bd60d3cfaef62e11790333 Mon Sep 17 00:00:00 2001 From: sreek Date: Tue, 29 Sep 2015 14:12:44 -0700 Subject: [PATCH 40/48] Fix float to bool conversion compilation error in mac --- src/core/support/histogram.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/support/histogram.c b/src/core/support/histogram.c index 78dbf98684a..8a1a9d92330 100644 --- a/src/core/support/histogram.c +++ b/src/core/support/histogram.c @@ -212,7 +212,7 @@ double gpr_histogram_percentile(gpr_histogram *h, double percentile) { } double gpr_histogram_mean(gpr_histogram *h) { - GPR_ASSERT(h->count); + GPR_ASSERT(h->count != 0); return h->sum / h->count; } From 7b080baf2abd5f826b519760cceb9b8b2174fd15 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 29 Sep 2015 22:22:36 +0000 Subject: [PATCH 41/48] C changes to avoid shadowed global declaration warnings in gcc4.4 --- src/core/iomgr/pollset_multipoller_with_epoll.c | 8 ++++---- src/core/iomgr/tcp_server_posix.c | 4 ++-- src/core/iomgr/udp_server.c | 4 ++-- src/core/support/time_posix.c | 14 +++++++------- src/core/transport/chttp2/hpack_table.c | 12 ++++++------ src/core/transport/chttp2/stream_encoder.c | 12 +++++++----- src/core/transport/stream_op.c | 6 +++--- 7 files changed, 31 insertions(+), 29 deletions(-) diff --git a/src/core/iomgr/pollset_multipoller_with_epoll.c b/src/core/iomgr/pollset_multipoller_with_epoll.c index b609e83c118..b22eaa62889 100644 --- a/src/core/iomgr/pollset_multipoller_with_epoll.c +++ b/src/core/iomgr/pollset_multipoller_with_epoll.c @@ -211,12 +211,12 @@ static void multipoll_with_epoll_pollset_maybe_work_and_unlock( /* TODO(klempner): We might want to consider making err and pri * separate events */ int cancel = ep_ev[i].events & (EPOLLERR | EPOLLHUP); - int read = ep_ev[i].events & (EPOLLIN | EPOLLPRI); - int write = ep_ev[i].events & EPOLLOUT; - if (read || cancel) { + int read_ev = ep_ev[i].events & (EPOLLIN | EPOLLPRI); + int write_ev = ep_ev[i].events & EPOLLOUT; + if (read_ev || cancel) { grpc_fd_become_readable(exec_ctx, fd); } - if (write || cancel) { + if (write_ev || cancel) { grpc_fd_become_writable(exec_ctx, fd); } } diff --git a/src/core/iomgr/tcp_server_posix.c b/src/core/iomgr/tcp_server_posix.c index a582f4a7c31..13bd67576f5 100644 --- a/src/core/iomgr/tcp_server_posix.c +++ b/src/core/iomgr/tcp_server_posix.c @@ -478,8 +478,8 @@ done: return allocated_port1 >= 0 ? allocated_port1 : allocated_port2; } -int grpc_tcp_server_get_fd(grpc_tcp_server *s, unsigned index) { - return (index < s->nports) ? s->ports[index].fd : -1; +int grpc_tcp_server_get_fd(grpc_tcp_server *s, unsigned port_index) { + return (port_index < s->nports) ? s->ports[port_index].fd : -1; } void grpc_tcp_server_start(grpc_exec_ctx *exec_ctx, grpc_tcp_server *s, diff --git a/src/core/iomgr/udp_server.c b/src/core/iomgr/udp_server.c index d884359aa43..a8d611c3f2d 100644 --- a/src/core/iomgr/udp_server.c +++ b/src/core/iomgr/udp_server.c @@ -399,8 +399,8 @@ done: return allocated_port1 >= 0 ? allocated_port1 : allocated_port2; } -int grpc_udp_server_get_fd(grpc_udp_server *s, unsigned index) { - return (index < s->nports) ? s->ports[index].fd : -1; +int grpc_udp_server_get_fd(grpc_udp_server *s, unsigned port_index) { + return (port_index < s->nports) ? s->ports[port_index].fd : -1; } void grpc_udp_server_start(grpc_exec_ctx *exec_ctx, grpc_udp_server *s, diff --git a/src/core/support/time_posix.c b/src/core/support/time_posix.c index eedfd0a0607..78f2c2bb77a 100644 --- a/src/core/support/time_posix.c +++ b/src/core/support/time_posix.c @@ -52,11 +52,11 @@ static struct timespec timespec_from_gpr(gpr_timespec gts) { #if _POSIX_TIMERS > 0 static gpr_timespec gpr_from_timespec(struct timespec ts, - gpr_clock_type clock) { + gpr_clock_type clock_type) { gpr_timespec rv; rv.tv_sec = ts.tv_sec; rv.tv_nsec = (int)ts.tv_nsec; - rv.clock_type = clock; + rv.clock_type = clock_type; return rv; } @@ -65,16 +65,16 @@ static clockid_t clockid_for_gpr_clock[] = {CLOCK_MONOTONIC, CLOCK_REALTIME}; void gpr_time_init(void) {} -gpr_timespec gpr_now(gpr_clock_type clock) { +gpr_timespec gpr_now(gpr_clock_type clock_type) { struct timespec now; - GPR_ASSERT(clock != GPR_TIMESPAN); - if (clock == GPR_CLOCK_PRECISE) { + GPR_ASSERT(clock_type != GPR_TIMESPAN); + if (clock_type == GPR_CLOCK_PRECISE) { gpr_timespec ret; gpr_precise_clock_now(&ret); return ret; } else { - clock_gettime(clockid_for_gpr_clock[clock], &now); - return gpr_from_timespec(now, clock); + clock_gettime(clockid_for_gpr_clock[clock_type], &now); + return gpr_from_timespec(now, clock_type); } } #else diff --git a/src/core/transport/chttp2/hpack_table.c b/src/core/transport/chttp2/hpack_table.c index d5cb752789f..c442c2c3413 100644 --- a/src/core/transport/chttp2/hpack_table.c +++ b/src/core/transport/chttp2/hpack_table.c @@ -193,15 +193,15 @@ void grpc_chttp2_hptbl_destroy(grpc_chttp2_hptbl *tbl) { } grpc_mdelem *grpc_chttp2_hptbl_lookup(const grpc_chttp2_hptbl *tbl, - gpr_uint32 index) { + gpr_uint32 tbl_index) { /* Static table comes first, just return an entry from it */ - if (index <= GRPC_CHTTP2_LAST_STATIC_ENTRY) { - return tbl->static_ents[index - 1]; + if (tbl_index <= GRPC_CHTTP2_LAST_STATIC_ENTRY) { + return tbl->static_ents[tbl_index - 1]; } /* Otherwise, find the value in the list of valid entries */ - index -= (GRPC_CHTTP2_LAST_STATIC_ENTRY + 1); - if (index < tbl->num_ents) { - gpr_uint32 offset = (tbl->num_ents - 1u - index + tbl->first_ent) % + tbl_index -= (GRPC_CHTTP2_LAST_STATIC_ENTRY + 1); + if (tbl_index < tbl->num_ents) { + gpr_uint32 offset = (tbl->num_ents - 1u - tbl_index + tbl->first_ent) % GRPC_CHTTP2_MAX_TABLE_COUNT; return tbl->ents[offset]; } diff --git a/src/core/transport/chttp2/stream_encoder.c b/src/core/transport/chttp2/stream_encoder.c index b1f1db05d26..ec97af3d5dd 100644 --- a/src/core/transport/chttp2/stream_encoder.c +++ b/src/core/transport/chttp2/stream_encoder.c @@ -274,10 +274,11 @@ static grpc_mdelem *add_elem(grpc_chttp2_hpack_compressor *c, return elem_to_unref; } -static void emit_indexed(grpc_chttp2_hpack_compressor *c, gpr_uint32 index, +static void emit_indexed(grpc_chttp2_hpack_compressor *c, gpr_uint32 elem_index, framer_state *st) { - gpr_uint32 len = GRPC_CHTTP2_VARINT_LENGTH(index, 1); - GRPC_CHTTP2_WRITE_VARINT(index, 1, 0x80, add_tiny_header_data(st, len), len); + gpr_uint32 len = GRPC_CHTTP2_VARINT_LENGTH(elem_index, 1); + GRPC_CHTTP2_WRITE_VARINT(elem_index, 1, 0x80, add_tiny_header_data(st, len), + len); } static gpr_slice get_wire_value(grpc_mdelem *elem, gpr_uint8 *huffman_prefix) { @@ -363,9 +364,10 @@ static void emit_lithdr_noidx_v(grpc_chttp2_hpack_compressor *c, add_header_data(st, gpr_slice_ref(value_slice)); } -static gpr_uint32 dynidx(grpc_chttp2_hpack_compressor *c, gpr_uint32 index) { +static gpr_uint32 dynidx(grpc_chttp2_hpack_compressor *c, + gpr_uint32 elem_index) { return 1 + GRPC_CHTTP2_LAST_STATIC_ENTRY + c->tail_remote_index + - c->table_elems - index; + c->table_elems - elem_index; } /* encode an mdelem; returns metadata element to unref */ diff --git a/src/core/transport/stream_op.c b/src/core/transport/stream_op.c index 038586d48e2..1cb2bd7c598 100644 --- a/src/core/transport/stream_op.c +++ b/src/core/transport/stream_op.c @@ -274,14 +274,14 @@ void grpc_metadata_batch_link_tail(grpc_metadata_batch *batch, } void grpc_metadata_batch_merge(grpc_metadata_batch *target, - grpc_metadata_batch *add) { + grpc_metadata_batch *to_add) { grpc_linked_mdelem *l; grpc_linked_mdelem *next; - for (l = add->list.head; l; l = next) { + for (l = to_add->list.head; l; l = next) { next = l->next; link_tail(&target->list, l); } - for (l = add->garbage.head; l; l = next) { + for (l = to_add->garbage.head; l; l = next) { next = l->next; link_tail(&target->garbage, l); } From e57abcfbdbaaff0fcf985bdef2da905e81f2b567 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 29 Sep 2015 22:55:34 +0000 Subject: [PATCH 42/48] C++ changes required to maintain gcc4.4 compatibility - reduce use of ambiguous nullptr, eliminate use of brace initializer lists --- src/cpp/client/secure_credentials.cc | 15 +++++++------ src/cpp/server/secure_server_credentials.cc | 24 +++++++++++---------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/cpp/client/secure_credentials.cc b/src/cpp/client/secure_credentials.cc index 1693cf740bd..1368a4c9141 100644 --- a/src/cpp/client/secure_credentials.cc +++ b/src/cpp/client/secure_credentials.cc @@ -154,10 +154,10 @@ void MetadataCredentialsPluginWrapper::Destroy(void* wrapper) { void MetadataCredentialsPluginWrapper::GetMetadata( void* wrapper, const char* service_url, grpc_credentials_plugin_metadata_cb cb, void* user_data) { - GPR_ASSERT(wrapper != nullptr); + GPR_ASSERT(!wrapper); MetadataCredentialsPluginWrapper* w = reinterpret_cast(wrapper); - if (w->plugin_ == nullptr) { + if (!w->plugin_) { cb(user_data, NULL, 0, GRPC_STATUS_OK, NULL); return; } @@ -177,11 +177,12 @@ void MetadataCredentialsPluginWrapper::InvokePlugin( Status status = plugin_->GetMetadata(service_url, &metadata); std::vector md; for (auto it = metadata.begin(); it != metadata.end(); ++it) { - md.push_back({it->first.c_str(), - it->second.data(), - it->second.size(), - 0, - {{nullptr, nullptr, nullptr, nullptr}}}); + grpc_metadata md_entry; + md_entry.key = it->first.c_str(); + md_entry.value = it->second.data(); + md_entry.value_length = it->second.size(); + md_entry.flags = 0; + md.push_back(md_entry); } cb(user_data, md.empty() ? nullptr : &md[0], md.size(), static_cast(status.error_code()), diff --git a/src/cpp/server/secure_server_credentials.cc b/src/cpp/server/secure_server_credentials.cc index 90afebfd2e2..7c828cb1257 100644 --- a/src/cpp/server/secure_server_credentials.cc +++ b/src/cpp/server/secure_server_credentials.cc @@ -52,7 +52,7 @@ void AuthMetadataProcessorAyncWrapper::Process( void* wrapper, grpc_auth_context* context, const grpc_metadata* md, size_t num_md, grpc_process_auth_metadata_done_cb cb, void* user_data) { auto* w = reinterpret_cast(wrapper); - if (w->processor_ == nullptr) { + if (!w->processor_) { // Early exit. cb(user_data, nullptr, 0, nullptr, 0, GRPC_STATUS_OK, nullptr); return; @@ -86,20 +86,22 @@ void AuthMetadataProcessorAyncWrapper::InvokeProcessor( std::vector consumed_md; for (auto it = consumed_metadata.begin(); it != consumed_metadata.end(); ++it) { - consumed_md.push_back({it->first.c_str(), - it->second.data(), - it->second.size(), - 0, - {{nullptr, nullptr, nullptr, nullptr}}}); + grpc_metadata md_entry; + md_entry.key = it->first.c_str(); + md_entry.value = it->second.data(); + md_entry.value_length = it->second.size(); + md_entry.flags = 0; + consumed_md.push_back(md_entry); } std::vector response_md; for (auto it = response_metadata.begin(); it != response_metadata.end(); ++it) { - response_md.push_back({it->first.c_str(), - it->second.data(), - it->second.size(), - 0, - {{nullptr, nullptr, nullptr, nullptr}}}); + grpc_metadata md_entry; + md_entry.key = it->first.c_str(); + md_entry.value = it->second.data(); + md_entry.value_length = it->second.size(); + md_entry.flags = 0; + response_md.push_back(md_entry); } auto consumed_md_data = consumed_md.empty() ? nullptr : &consumed_md[0]; auto response_md_data = response_md.empty() ? nullptr : &response_md[0]; From 8423203cbb316a251bcefb076c76c59a68b6bfea Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 29 Sep 2015 23:16:08 +0000 Subject: [PATCH 43/48] For compatibility with gcc-4.4, eliminate use of sleep_for and std::atomic --- test/cpp/end2end/streaming_throughput_test.cc | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/test/cpp/end2end/streaming_throughput_test.cc b/test/cpp/end2end/streaming_throughput_test.cc index c1355b38f0f..344bf507ce1 100644 --- a/test/cpp/end2end/streaming_throughput_test.cc +++ b/test/cpp/end2end/streaming_throughput_test.cc @@ -31,9 +31,9 @@ * */ -#include #include #include +#include #include #include @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -99,12 +100,17 @@ namespace testing { class TestServiceImpl : public ::grpc::cpp::test::util::TestService::Service { public: - static void BidiStream_Sender(ServerReaderWriter* stream, std::atomic* should_exit) { + static void BidiStream_Sender(ServerReaderWriter* stream, gpr_atm* should_exit) { EchoResponse response; response.set_message(kLargeString); - while (!should_exit->load()) { - // TODO(vpai): Decide if the below requires blocking annotation - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + while (gpr_atm_acq_load(should_exit) == static_cast(0)) { + struct timespec tv = {0, 1000000}; // 1 ms + struct timespec rem; + // TODO (vpai): Mark this blocking + while (nanosleep(&tv, &rem) != 0) { + tv = rem; + }; + stream->Write(response); } } @@ -114,14 +120,20 @@ class TestServiceImpl : public ::grpc::cpp::test::util::TestService::Service { ServerReaderWriter* stream) GRPC_OVERRIDE { EchoRequest request; - std::atomic should_exit(false); + gpr_atm should_exit; + gpr_atm_rel_store(&should_exit, static_cast(0)); + std::thread sender(std::bind(&TestServiceImpl::BidiStream_Sender, stream, &should_exit)); while (stream->Read(&request)) { - // TODO(vpai): Decide if the below requires blocking annotation - std::this_thread::sleep_for(std::chrono::milliseconds(3)); + struct timespec tv = {0, 3000000}; // 3 ms + struct timespec rem; + // TODO (vpai): Mark this blocking + while (nanosleep(&tv, &rem) != 0) { + tv = rem; + }; } - should_exit.store(true); + gpr_atm_rel_store(&should_exit, static_cast(1)); sender.join(); return Status::OK; } From e547bdf4d7abf46dc8bc7b9364a02c39fb597c9e Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Tue, 29 Sep 2015 23:29:38 +0000 Subject: [PATCH 44/48] Fix an assert --- src/cpp/client/secure_credentials.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/client/secure_credentials.cc b/src/cpp/client/secure_credentials.cc index 1368a4c9141..8299ebeb8a2 100644 --- a/src/cpp/client/secure_credentials.cc +++ b/src/cpp/client/secure_credentials.cc @@ -154,7 +154,7 @@ void MetadataCredentialsPluginWrapper::Destroy(void* wrapper) { void MetadataCredentialsPluginWrapper::GetMetadata( void* wrapper, const char* service_url, grpc_credentials_plugin_metadata_cb cb, void* user_data) { - GPR_ASSERT(!wrapper); + GPR_ASSERT(wrapper); MetadataCredentialsPluginWrapper* w = reinterpret_cast(wrapper); if (!w->plugin_) { From 4086474399be1811731fac8bb6755de730ddbc3f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 30 Sep 2015 08:35:31 -0700 Subject: [PATCH 45/48] Make grpc_exec_ctx_flush return a status indicating if work was performed --- src/core/iomgr/exec_ctx.c | 5 ++++- src/core/iomgr/exec_ctx.h | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/core/iomgr/exec_ctx.c b/src/core/iomgr/exec_ctx.c index a830a27b0ba..f2914d376ef 100644 --- a/src/core/iomgr/exec_ctx.c +++ b/src/core/iomgr/exec_ctx.c @@ -35,16 +35,19 @@ #include -void grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) { +int grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx) { + int did_something = 0; while (!grpc_closure_list_empty(exec_ctx->closure_list)) { grpc_closure *c = exec_ctx->closure_list.head; exec_ctx->closure_list.head = exec_ctx->closure_list.tail = NULL; while (c != NULL) { grpc_closure *next = c->next; + did_something = 1; c->cb(exec_ctx, c->cb_arg, c->success); c = next; } } + return did_something; } void grpc_exec_ctx_finish(grpc_exec_ctx *exec_ctx) { diff --git a/src/core/iomgr/exec_ctx.h b/src/core/iomgr/exec_ctx.h index f99aa038c59..aa0610cbeaf 100644 --- a/src/core/iomgr/exec_ctx.h +++ b/src/core/iomgr/exec_ctx.h @@ -61,8 +61,9 @@ struct grpc_exec_ctx { { GRPC_CLOSURE_LIST_INIT } /** Flush any work that has been enqueued onto this grpc_exec_ctx. - * Caller must guarantee that no interfering locks are held. */ -void grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx); + * Caller must guarantee that no interfering locks are held. + * Returns 1 if work was performed, 0 otherwise. */ +int grpc_exec_ctx_flush(grpc_exec_ctx *exec_ctx); /** Finish any pending work for a grpc_exec_ctx. Must be called before * the instance is destroyed, or work may be lost. */ void grpc_exec_ctx_finish(grpc_exec_ctx *exec_ctx); From 01be53d1a11c966baf1b1ce66baa60bf763bfc8b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 30 Sep 2015 08:36:03 -0700 Subject: [PATCH 46/48] Add a facility to flush iocp at iomgr shutdown --- src/core/iomgr/iocp_windows.c | 12 +++++++++--- src/core/iomgr/iocp_windows.h | 1 + src/core/iomgr/iomgr.c | 2 ++ src/core/iomgr/iomgr_internal.h | 3 +++ src/core/iomgr/iomgr_posix.c | 3 +++ src/core/iomgr/iomgr_windows.c | 4 ++++ 6 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/core/iomgr/iocp_windows.c b/src/core/iomgr/iocp_windows.c index 349440fa88c..791f2e39c84 100644 --- a/src/core/iomgr/iocp_windows.c +++ b/src/core/iomgr/iocp_windows.c @@ -119,9 +119,7 @@ void grpc_iocp_work(grpc_exec_ctx *exec_ctx, gpr_timespec deadline) { info->has_pending_iocp = 1; } gpr_mu_unlock(&socket->state_mu); - if (closure) { - closure->cb(exec_ctx, closure->cb_arg, 1); - } + grpc_exec_ctx_enqueue(exec_ctx, closure, 1); } void grpc_iocp_init(void) { @@ -139,6 +137,14 @@ void grpc_iocp_kick(void) { GPR_ASSERT(success); } +void grpc_iocp_flush(void) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + + do { + grpc_iocp_work(&exec_ctx, gpr_inf_future(GPR_CLOCK_MONOTONIC)); + } while (grpc_exec_ctx_flush(&exec_ctx)); +} + void grpc_iocp_shutdown(void) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; while (gpr_atm_acq_load(&g_custom_events)) { diff --git a/src/core/iomgr/iocp_windows.h b/src/core/iomgr/iocp_windows.h index 7e330e7ce2f..75f3ba84770 100644 --- a/src/core/iomgr/iocp_windows.h +++ b/src/core/iomgr/iocp_windows.h @@ -41,6 +41,7 @@ void grpc_iocp_work(grpc_exec_ctx *exec_ctx, gpr_timespec deadline); void grpc_iocp_init(void); void grpc_iocp_kick(void); +void grpc_iocp_flush(void); void grpc_iocp_shutdown(void); void grpc_iocp_add_socket(grpc_winsocket *); diff --git a/src/core/iomgr/iomgr.c b/src/core/iomgr/iomgr.c index 0c067e51877..a10399311fc 100644 --- a/src/core/iomgr/iomgr.c +++ b/src/core/iomgr/iomgr.c @@ -91,6 +91,8 @@ void grpc_iomgr_shutdown(void) { gpr_timespec last_warning_time = gpr_now(GPR_CLOCK_REALTIME); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_iomgr_platform_flush(); + gpr_mu_lock(&g_mu); g_shutdown = 1; while (g_root_object.next != &g_root_object) { diff --git a/src/core/iomgr/iomgr_internal.h b/src/core/iomgr/iomgr_internal.h index 1a0724b431c..e372c18e8a0 100644 --- a/src/core/iomgr/iomgr_internal.h +++ b/src/core/iomgr/iomgr_internal.h @@ -50,6 +50,9 @@ void grpc_iomgr_register_object(grpc_iomgr_object *obj, const char *name); void grpc_iomgr_unregister_object(grpc_iomgr_object *obj); void grpc_iomgr_platform_init(void); +/** flush any globally queued work from iomgr */ +void grpc_iomgr_platform_flush(void); +/** tear down all platform specific global iomgr structures */ void grpc_iomgr_platform_shutdown(void); #endif /* GRPC_INTERNAL_CORE_IOMGR_IOMGR_INTERNAL_H */ diff --git a/src/core/iomgr/iomgr_posix.c b/src/core/iomgr/iomgr_posix.c index db93d0a7561..f6474b7e6d4 100644 --- a/src/core/iomgr/iomgr_posix.c +++ b/src/core/iomgr/iomgr_posix.c @@ -45,6 +45,9 @@ void grpc_iomgr_platform_init(void) { grpc_register_tracer("tcp", &grpc_tcp_trace); } +void grpc_iomgr_platform_flush(void) { +} + void grpc_iomgr_platform_shutdown(void) { grpc_fd_global_shutdown(); } diff --git a/src/core/iomgr/iomgr_windows.c b/src/core/iomgr/iomgr_windows.c index b49cb87e97b..93bdc5ec16e 100644 --- a/src/core/iomgr/iomgr_windows.c +++ b/src/core/iomgr/iomgr_windows.c @@ -63,6 +63,10 @@ void grpc_iomgr_platform_init(void) { grpc_iocp_init(); } +void grpc_iomgr_platform_flush(void) { + grpc_iocp_flush(); +} + void grpc_iomgr_platform_shutdown(void) { grpc_iocp_shutdown(); winsock_shutdown(); From 0b6312e970a425819a68843e93f72c4d06e31543 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 30 Sep 2015 08:36:27 -0700 Subject: [PATCH 47/48] Flush iocp related work immediately --- src/core/iomgr/pollset_windows.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/iomgr/pollset_windows.c b/src/core/iomgr/pollset_windows.c index dd5301f5be9..798b6376358 100644 --- a/src/core/iomgr/pollset_windows.c +++ b/src/core/iomgr/pollset_windows.c @@ -152,6 +152,7 @@ void grpc_pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset, g_active_poller = worker; gpr_mu_unlock(&grpc_polling_mu); grpc_iocp_work(exec_ctx, deadline); + grpc_exec_ctx_flush(exec_ctx); gpr_mu_lock(&grpc_polling_mu); pollset->is_iocp_worker = 0; g_active_poller = NULL; From 1433791d87ec0c0b50163d8474e49a08d189287c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 30 Sep 2015 08:41:51 -0700 Subject: [PATCH 48/48] Don't wait forever for iocp to shutdown --- src/core/iomgr/iocp_windows.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/iomgr/iocp_windows.c b/src/core/iomgr/iocp_windows.c index 791f2e39c84..cf33d74366f 100644 --- a/src/core/iomgr/iocp_windows.c +++ b/src/core/iomgr/iocp_windows.c @@ -141,7 +141,7 @@ void grpc_iocp_flush(void) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; do { - grpc_iocp_work(&exec_ctx, gpr_inf_future(GPR_CLOCK_MONOTONIC)); + grpc_iocp_work(&exec_ctx, gpr_inf_past(GPR_CLOCK_MONOTONIC)); } while (grpc_exec_ctx_flush(&exec_ctx)); }