mirror of https://github.com/grpc/grpc.git
Merge branch 'master' of https://github.com/grpc/grpc
Noah Eisen
8 years ago
commit
7d5f822bec
35 changed files with 1174 additions and 92 deletions
@ -0,0 +1,118 @@ |
||||
/*
|
||||
* |
||||
* Copyright 2016, Google Inc. |
||||
* All rights reserved. |
||||
* |
||||
* Redistribution and use in source and binary forms, with or without |
||||
* modification, are permitted provided that the following conditions are |
||||
* met: |
||||
* |
||||
* * Redistributions of source code must retain the above copyright |
||||
* notice, this list of conditions and the following disclaimer. |
||||
* * Redistributions in binary form must reproduce the above |
||||
* copyright notice, this list of conditions and the following disclaimer |
||||
* in the documentation and/or other materials provided with the |
||||
* distribution. |
||||
* * Neither the name of Google Inc. nor the names of its |
||||
* contributors may be used to endorse or promote products derived from |
||||
* this software without specific prior written permission. |
||||
* |
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
* |
||||
*/ |
||||
|
||||
#include <grpc/support/port_platform.h> |
||||
|
||||
#ifdef GPR_POSIX_WAKEUP_FD |
||||
|
||||
#include "src/core/lib/iomgr/wakeup_fd_cv.h" |
||||
|
||||
#include <errno.h> |
||||
#include <string.h> |
||||
|
||||
#include <grpc/support/alloc.h> |
||||
#include <grpc/support/log.h> |
||||
#include <grpc/support/sync.h> |
||||
#include <grpc/support/thd.h> |
||||
#include <grpc/support/time.h> |
||||
#include <grpc/support/useful.h> |
||||
|
||||
#define MAX_TABLE_RESIZE 256 |
||||
|
||||
extern cv_fd_table g_cvfds; |
||||
|
||||
static grpc_error* cv_fd_init(grpc_wakeup_fd* fd_info) { |
||||
unsigned int i, newsize; |
||||
int idx; |
||||
gpr_mu_lock(&g_cvfds.mu); |
||||
if (!g_cvfds.free_fds) { |
||||
newsize = GPR_MIN(g_cvfds.size * 2, g_cvfds.size + MAX_TABLE_RESIZE); |
||||
g_cvfds.cvfds = gpr_realloc(g_cvfds.cvfds, sizeof(fd_node) * newsize); |
||||
for (i = g_cvfds.size; i < newsize; i++) { |
||||
g_cvfds.cvfds[i].is_set = 0; |
||||
g_cvfds.cvfds[i].cvs = NULL; |
||||
g_cvfds.cvfds[i].next_free = g_cvfds.free_fds; |
||||
g_cvfds.free_fds = &g_cvfds.cvfds[i]; |
||||
} |
||||
g_cvfds.size = newsize; |
||||
} |
||||
|
||||
idx = (int)(g_cvfds.free_fds - g_cvfds.cvfds); |
||||
g_cvfds.free_fds = g_cvfds.free_fds->next_free; |
||||
g_cvfds.cvfds[idx].cvs = NULL; |
||||
g_cvfds.cvfds[idx].is_set = 0; |
||||
fd_info->read_fd = IDX_TO_FD(idx); |
||||
fd_info->write_fd = -1; |
||||
gpr_mu_unlock(&g_cvfds.mu); |
||||
return GRPC_ERROR_NONE; |
||||
} |
||||
|
||||
static grpc_error* cv_fd_wakeup(grpc_wakeup_fd* fd_info) { |
||||
cv_node* cvn; |
||||
gpr_mu_lock(&g_cvfds.mu); |
||||
g_cvfds.cvfds[FD_TO_IDX(fd_info->read_fd)].is_set = 1; |
||||
cvn = g_cvfds.cvfds[FD_TO_IDX(fd_info->read_fd)].cvs; |
||||
while (cvn) { |
||||
gpr_cv_signal(cvn->cv); |
||||
cvn = cvn->next; |
||||
} |
||||
gpr_mu_unlock(&g_cvfds.mu); |
||||
return GRPC_ERROR_NONE; |
||||
} |
||||
|
||||
static grpc_error* cv_fd_consume(grpc_wakeup_fd* fd_info) { |
||||
gpr_mu_lock(&g_cvfds.mu); |
||||
g_cvfds.cvfds[FD_TO_IDX(fd_info->read_fd)].is_set = 0; |
||||
gpr_mu_unlock(&g_cvfds.mu); |
||||
return GRPC_ERROR_NONE; |
||||
} |
||||
|
||||
static void cv_fd_destroy(grpc_wakeup_fd* fd_info) { |
||||
if (fd_info->read_fd == 0) { |
||||
return; |
||||
} |
||||
gpr_mu_lock(&g_cvfds.mu); |
||||
// Assert that there are no active pollers
|
||||
GPR_ASSERT(!g_cvfds.cvfds[FD_TO_IDX(fd_info->read_fd)].cvs); |
||||
g_cvfds.cvfds[FD_TO_IDX(fd_info->read_fd)].next_free = g_cvfds.free_fds; |
||||
g_cvfds.free_fds = &g_cvfds.cvfds[FD_TO_IDX(fd_info->read_fd)]; |
||||
gpr_mu_unlock(&g_cvfds.mu); |
||||
} |
||||
|
||||
static int cv_check_availability(void) { return 1; } |
||||
|
||||
const grpc_wakeup_fd_vtable grpc_cv_wakeup_fd_vtable = { |
||||
cv_fd_init, cv_fd_consume, cv_fd_wakeup, cv_fd_destroy, |
||||
cv_check_availability}; |
||||
|
||||
#endif /* GPR_POSIX_WAKUP_FD */ |
||||
@ -0,0 +1,80 @@ |
||||
/*
|
||||
* |
||||
* Copyright 2016, Google Inc. |
||||
* All rights reserved. |
||||
* |
||||
* Redistribution and use in source and binary forms, with or without |
||||
* modification, are permitted provided that the following conditions are |
||||
* met: |
||||
* |
||||
* * Redistributions of source code must retain the above copyright |
||||
* notice, this list of conditions and the following disclaimer. |
||||
* * Redistributions in binary form must reproduce the above |
||||
* copyright notice, this list of conditions and the following disclaimer |
||||
* in the documentation and/or other materials provided with the |
||||
* distribution. |
||||
* * Neither the name of Google Inc. nor the names of its |
||||
* contributors may be used to endorse or promote products derived from |
||||
* this software without specific prior written permission. |
||||
* |
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
* |
||||
*/ |
||||
|
||||
/*
|
||||
* wakeup_fd_cv uses condition variables to implement wakeup fds. |
||||
* |
||||
* It is intended for use only in cases when eventfd() and pipe() are not |
||||
* available. It can only be used with the "poll" engine. |
||||
* |
||||
* Implementation: |
||||
* A global table of cv wakeup fds is mantained. A cv wakeup fd is a negative |
||||
* file descriptor. poll() is then run in a background thread with only the |
||||
* real socket fds while we wait on a condition variable trigged by either the |
||||
* poll() completion or a wakeup_fd() call. |
||||
* |
||||
*/ |
||||
|
||||
#ifndef GRPC_CORE_LIB_IOMGR_WAKEUP_FD_CV_H |
||||
#define GRPC_CORE_LIB_IOMGR_WAKEUP_FD_CV_H |
||||
|
||||
#include <grpc/support/sync.h> |
||||
|
||||
#include "src/core/lib/iomgr/ev_posix.h" |
||||
|
||||
#define FD_TO_IDX(fd) (-(fd)-1) |
||||
#define IDX_TO_FD(idx) (-(idx)-1) |
||||
|
||||
typedef struct cv_node { |
||||
gpr_cv* cv; |
||||
struct cv_node* next; |
||||
} cv_node; |
||||
|
||||
typedef struct fd_node { |
||||
int is_set; |
||||
cv_node* cvs; |
||||
struct fd_node* next_free; |
||||
} fd_node; |
||||
|
||||
typedef struct cv_fd_table { |
||||
gpr_mu mu; |
||||
int pollcount; |
||||
int shutdown; |
||||
gpr_cv shutdown_complete; |
||||
fd_node* cvfds; |
||||
fd_node* free_fds; |
||||
unsigned int size; |
||||
grpc_poll_function_type poll; |
||||
} cv_fd_table; |
||||
|
||||
#endif /* GRPC_CORE_LIB_IOMGR_WAKEUP_FD_CV_H */ |
||||
@ -0,0 +1,240 @@ |
||||
/*
|
||||
* |
||||
* Copyright 2016, Google Inc. |
||||
* All rights reserved. |
||||
* |
||||
* Redistribution and use in source and binary forms, with or without |
||||
* modification, are permitted provided that the following conditions are |
||||
* met: |
||||
* |
||||
* * Redistributions of source code must retain the above copyright |
||||
* notice, this list of conditions and the following disclaimer. |
||||
* * Redistributions in binary form must reproduce the above |
||||
* copyright notice, this list of conditions and the following disclaimer |
||||
* in the documentation and/or other materials provided with the |
||||
* distribution. |
||||
* * Neither the name of Google Inc. nor the names of its |
||||
* contributors may be used to endorse or promote products derived from |
||||
* this software without specific prior written permission. |
||||
* |
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
* |
||||
*/ |
||||
|
||||
#include <pthread.h> |
||||
|
||||
#include <grpc/support/log.h> |
||||
#include <grpc/support/thd.h> |
||||
#include <grpc/support/time.h> |
||||
#include <grpc/support/useful.h> |
||||
|
||||
#include "src/core/lib/iomgr/ev_posix.h" |
||||
#include "src/core/lib/iomgr/iomgr_posix.h" |
||||
#include "src/core/lib/support/env.h" |
||||
|
||||
typedef struct poll_args { |
||||
struct pollfd *fds; |
||||
nfds_t nfds; |
||||
int timeout; |
||||
int result; |
||||
} poll_args; |
||||
|
||||
gpr_cv poll_cv; |
||||
gpr_mu poll_mu; |
||||
static int socket_event = 0; |
||||
|
||||
// Trigger a "socket" POLLIN in mock_poll()
|
||||
void trigger_socket_event() { |
||||
gpr_mu_lock(&poll_mu); |
||||
socket_event = 1; |
||||
gpr_cv_broadcast(&poll_cv); |
||||
gpr_mu_unlock(&poll_mu); |
||||
} |
||||
|
||||
void reset_socket_event() { |
||||
gpr_mu_lock(&poll_mu); |
||||
socket_event = 0; |
||||
gpr_mu_unlock(&poll_mu); |
||||
} |
||||
|
||||
// Mocks posix poll() function
|
||||
int mock_poll(struct pollfd *fds, nfds_t nfds, int timeout) { |
||||
int res = 0; |
||||
gpr_timespec poll_time; |
||||
gpr_mu_lock(&poll_mu); |
||||
GPR_ASSERT(nfds == 3); |
||||
GPR_ASSERT(fds[0].fd == 20); |
||||
GPR_ASSERT(fds[1].fd == 30); |
||||
GPR_ASSERT(fds[2].fd == 50); |
||||
GPR_ASSERT(fds[0].events == (POLLIN | POLLHUP)); |
||||
GPR_ASSERT(fds[1].events == (POLLIN | POLLHUP)); |
||||
GPR_ASSERT(fds[2].events == POLLIN); |
||||
|
||||
if (timeout < 0) { |
||||
poll_time = gpr_inf_future(GPR_CLOCK_REALTIME); |
||||
} else { |
||||
poll_time = gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), |
||||
gpr_time_from_millis(timeout, GPR_TIMESPAN)); |
||||
} |
||||
|
||||
if (socket_event || !gpr_cv_wait(&poll_cv, &poll_mu, poll_time)) { |
||||
fds[0].revents = POLLIN; |
||||
res = 1; |
||||
} |
||||
gpr_mu_unlock(&poll_mu); |
||||
return res; |
||||
} |
||||
|
||||
void background_poll(void *args) { |
||||
poll_args *pargs = (poll_args *)args; |
||||
pargs->result = grpc_poll_function(pargs->fds, pargs->nfds, pargs->timeout); |
||||
} |
||||
|
||||
void test_many_fds(void) { |
||||
int i; |
||||
grpc_wakeup_fd fd[1000]; |
||||
for (i = 0; i < 1000; i++) { |
||||
GPR_ASSERT(grpc_wakeup_fd_init(&fd[i]) == GRPC_ERROR_NONE); |
||||
} |
||||
for (i = 0; i < 1000; i++) { |
||||
grpc_wakeup_fd_destroy(&fd[i]); |
||||
} |
||||
} |
||||
|
||||
void test_poll_cv_trigger(void) { |
||||
grpc_wakeup_fd cvfd1, cvfd2, cvfd3; |
||||
struct pollfd pfds[6]; |
||||
poll_args pargs; |
||||
gpr_thd_id t_id; |
||||
gpr_thd_options opt; |
||||
|
||||
GPR_ASSERT(grpc_wakeup_fd_init(&cvfd1) == GRPC_ERROR_NONE); |
||||
GPR_ASSERT(grpc_wakeup_fd_init(&cvfd2) == GRPC_ERROR_NONE); |
||||
GPR_ASSERT(grpc_wakeup_fd_init(&cvfd3) == GRPC_ERROR_NONE); |
||||
GPR_ASSERT(cvfd1.read_fd < 0); |
||||
GPR_ASSERT(cvfd2.read_fd < 0); |
||||
GPR_ASSERT(cvfd3.read_fd < 0); |
||||
GPR_ASSERT(cvfd1.read_fd != cvfd2.read_fd); |
||||
GPR_ASSERT(cvfd2.read_fd != cvfd3.read_fd); |
||||
GPR_ASSERT(cvfd1.read_fd != cvfd3.read_fd); |
||||
|
||||
pfds[0].fd = cvfd1.read_fd; |
||||
pfds[1].fd = cvfd2.read_fd; |
||||
pfds[2].fd = 20; |
||||
pfds[3].fd = 30; |
||||
pfds[4].fd = cvfd3.read_fd; |
||||
pfds[5].fd = 50; |
||||
|
||||
pfds[0].events = 0; |
||||
pfds[1].events = POLLIN; |
||||
pfds[2].events = POLLIN | POLLHUP; |
||||
pfds[3].events = POLLIN | POLLHUP; |
||||
pfds[4].events = POLLIN; |
||||
pfds[5].events = POLLIN; |
||||
|
||||
pargs.fds = pfds; |
||||
pargs.nfds = 6; |
||||
pargs.timeout = 1000; |
||||
pargs.result = -2; |
||||
|
||||
opt = gpr_thd_options_default(); |
||||
gpr_thd_options_set_joinable(&opt); |
||||
gpr_thd_new(&t_id, &background_poll, &pargs, &opt); |
||||
|
||||
// Wakeup wakeup_fd not listening for events
|
||||
GPR_ASSERT(grpc_wakeup_fd_wakeup(&cvfd1) == GRPC_ERROR_NONE); |
||||
gpr_thd_join(t_id); |
||||
GPR_ASSERT(pargs.result == 0); |
||||
GPR_ASSERT(pfds[0].revents == 0); |
||||
GPR_ASSERT(pfds[1].revents == 0); |
||||
GPR_ASSERT(pfds[2].revents == 0); |
||||
GPR_ASSERT(pfds[3].revents == 0); |
||||
GPR_ASSERT(pfds[4].revents == 0); |
||||
GPR_ASSERT(pfds[5].revents == 0); |
||||
|
||||
// Pollin on socket fd
|
||||
pargs.timeout = -1; |
||||
pargs.result = -2; |
||||
gpr_thd_new(&t_id, &background_poll, &pargs, &opt); |
||||
trigger_socket_event(); |
||||
gpr_thd_join(t_id); |
||||
GPR_ASSERT(pargs.result == 1); |
||||
GPR_ASSERT(pfds[0].revents == 0); |
||||
GPR_ASSERT(pfds[1].revents == 0); |
||||
GPR_ASSERT(pfds[2].revents == POLLIN); |
||||
GPR_ASSERT(pfds[3].revents == 0); |
||||
GPR_ASSERT(pfds[4].revents == 0); |
||||
GPR_ASSERT(pfds[5].revents == 0); |
||||
|
||||
// Pollin on wakeup fd
|
||||
reset_socket_event(); |
||||
pargs.result = -2; |
||||
gpr_thd_new(&t_id, &background_poll, &pargs, &opt); |
||||
GPR_ASSERT(grpc_wakeup_fd_wakeup(&cvfd2) == GRPC_ERROR_NONE); |
||||
gpr_thd_join(t_id); |
||||
|
||||
GPR_ASSERT(pargs.result == 1); |
||||
GPR_ASSERT(pfds[0].revents == 0); |
||||
GPR_ASSERT(pfds[1].revents == POLLIN); |
||||
GPR_ASSERT(pfds[2].revents == 0); |
||||
GPR_ASSERT(pfds[3].revents == 0); |
||||
GPR_ASSERT(pfds[4].revents == 0); |
||||
GPR_ASSERT(pfds[5].revents == 0); |
||||
|
||||
// Pollin on wakeup fd + socket fd
|
||||
trigger_socket_event(); |
||||
pargs.result = -2; |
||||
gpr_thd_new(&t_id, &background_poll, &pargs, &opt); |
||||
gpr_thd_join(t_id); |
||||
|
||||
GPR_ASSERT(pargs.result == 2); |
||||
GPR_ASSERT(pfds[0].revents == 0); |
||||
GPR_ASSERT(pfds[1].revents == POLLIN); |
||||
GPR_ASSERT(pfds[2].revents == POLLIN); |
||||
GPR_ASSERT(pfds[3].revents == 0); |
||||
GPR_ASSERT(pfds[4].revents == 0); |
||||
GPR_ASSERT(pfds[5].revents == 0); |
||||
|
||||
// No Events
|
||||
pargs.result = -2; |
||||
pargs.timeout = 1000; |
||||
reset_socket_event(); |
||||
GPR_ASSERT(grpc_wakeup_fd_consume_wakeup(&cvfd1) == GRPC_ERROR_NONE); |
||||
GPR_ASSERT(grpc_wakeup_fd_consume_wakeup(&cvfd2) == GRPC_ERROR_NONE); |
||||
gpr_thd_new(&t_id, &background_poll, &pargs, &opt); |
||||
gpr_thd_join(t_id); |
||||
|
||||
GPR_ASSERT(pargs.result == 0); |
||||
GPR_ASSERT(pfds[0].revents == 0); |
||||
GPR_ASSERT(pfds[1].revents == 0); |
||||
GPR_ASSERT(pfds[2].revents == 0); |
||||
GPR_ASSERT(pfds[3].revents == 0); |
||||
GPR_ASSERT(pfds[4].revents == 0); |
||||
GPR_ASSERT(pfds[5].revents == 0); |
||||
} |
||||
|
||||
int main(int argc, char **argv) { |
||||
gpr_setenv("GRPC_POLL_STRATEGY", "poll-cv"); |
||||
grpc_poll_function = &mock_poll; |
||||
gpr_mu_init(&poll_mu); |
||||
gpr_cv_init(&poll_cv); |
||||
|
||||
grpc_iomgr_platform_init(); |
||||
test_many_fds(); |
||||
grpc_iomgr_platform_shutdown(); |
||||
|
||||
grpc_iomgr_platform_init(); |
||||
test_poll_cv_trigger(); |
||||
grpc_iomgr_platform_shutdown(); |
||||
return 0; |
||||
} |
||||
@ -0,0 +1,184 @@ |
||||
#!/usr/bin/env python2.7 |
||||
# Copyright 2015, Google Inc. |
||||
# All rights reserved. |
||||
# |
||||
# Redistribution and use in source and binary forms, with or without |
||||
# modification, are permitted provided that the following conditions are |
||||
# met: |
||||
# |
||||
# * Redistributions of source code must retain the above copyright |
||||
# notice, this list of conditions and the following disclaimer. |
||||
# * Redistributions in binary form must reproduce the above |
||||
# copyright notice, this list of conditions and the following disclaimer |
||||
# in the documentation and/or other materials provided with the |
||||
# distribution. |
||||
# * Neither the name of Google Inc. nor the names of its |
||||
# contributors may be used to endorse or promote products derived from |
||||
# this software without specific prior written permission. |
||||
# |
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
||||
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
||||
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
||||
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
||||
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
||||
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
||||
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
||||
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
||||
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
||||
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
|
||||
"""Filter out tests based on file differences compared to merge target branch""" |
||||
|
||||
import re |
||||
from subprocess import call, check_output |
||||
|
||||
|
||||
class TestSuite: |
||||
""" |
||||
Contains tag to identify job as belonging to this test suite and |
||||
triggers to identify if changed files are relevant |
||||
""" |
||||
def __init__(self, tags): |
||||
""" |
||||
Build TestSuite to group tests by their tags |
||||
:param tag: string used to identify if a job belongs to this TestSuite |
||||
todo(mattkwong): Change the use of tag because do not want to depend on |
||||
job.shortname to identify what suite a test belongs to |
||||
""" |
||||
self.triggers = [] |
||||
self.tags = tags |
||||
|
||||
def add_trigger(self, trigger): |
||||
""" |
||||
Add a regex to list of triggers that determine if a changed file should run tests |
||||
:param trigger: regex matching file relevant to tests |
||||
""" |
||||
self.triggers.append(trigger) |
||||
|
||||
# Create test suites |
||||
_core_test_suite = TestSuite(['_c_']) |
||||
_cpp_test_suite = TestSuite(['_c++_']) |
||||
_csharp_test_suite = TestSuite(['_csharp_']) |
||||
_node_test_suite = TestSuite(['_node_']) |
||||
_objc_test_suite = TestSuite(['_objc_']) |
||||
_php_test_suite = TestSuite(['_php_', '_php7_']) |
||||
_python_test_suite = TestSuite(['_python_']) |
||||
_ruby_test_suite = TestSuite(['_ruby']) |
||||
_all_test_suites = [_core_test_suite, _cpp_test_suite, _csharp_test_suite, |
||||
_node_test_suite, _objc_test_suite, _php_test_suite, |
||||
_python_test_suite, _ruby_test_suite] |
||||
|
||||
# Dictionary of whitelistable files where the key is a regex matching changed files |
||||
# and the value is a list of tests that should be run. An empty list means that |
||||
# the changed files should not trigger any tests. Any changed file that does not |
||||
# match any of these regexes will trigger all tests |
||||
_WHITELIST_DICT = { |
||||
'^templates/.*': [], |
||||
'^doc/.*': [], |
||||
'^examples/.*': [], |
||||
'^summerofcode/.*': [], |
||||
'.*README.md$': [], |
||||
'.*LICENSE$': [], |
||||
'^src/cpp.*': [_cpp_test_suite], |
||||
'^src/csharp.*': [_csharp_test_suite], |
||||
'^src/node.*': [_node_test_suite], |
||||
'^src/objective-c.*': [_objc_test_suite], |
||||
'^src/php.*': [_php_test_suite], |
||||
'^src/python.*': [_python_test_suite], |
||||
'^src/ruby.*': [_ruby_test_suite], |
||||
'^test/core.*': [_core_test_suite], |
||||
'^test/cpp.*': [_cpp_test_suite], |
||||
'^test/distrib/cpp.*': [_cpp_test_suite], |
||||
'^test/distrib/csharp.*': [_csharp_test_suite], |
||||
'^test/distrib/node.*': [_node_test_suite], |
||||
'^test/distrib/php.*': [_php_test_suite], |
||||
'^test/distrib/python.*': [_python_test_suite], |
||||
'^test/distrib/ruby.*': [_ruby_test_suite] |
||||
} |
||||
# Add all triggers to their respective test suites |
||||
for trigger, test_suites in _WHITELIST_DICT.iteritems(): |
||||
for test_suite in test_suites: |
||||
test_suite.add_trigger(trigger) |
||||
|
||||
|
||||
def _get_changed_files(base_branch): |
||||
""" |
||||
Get list of changed files between current branch and base of target merge branch |
||||
""" |
||||
# git fetch might need to be called on Jenkins slave |
||||
# todo(mattkwong): remove or uncomment below after seeing if Jenkins needs this |
||||
# call(['git', 'fetch']) |
||||
|
||||
# Get file changes between branch and merge-base of specified branch |
||||
# Not combined to be Windows friendly |
||||
base_commit = check_output(["git", "merge-base", base_branch, "HEAD"]).rstrip() |
||||
return check_output(["git", "diff", base_commit, "--name-only"]).splitlines() |
||||
|
||||
|
||||
def _can_skip_tests(file_names, triggers): |
||||
""" |
||||
Determines if tests are skippable based on if all files do not match list of regexes |
||||
:param file_names: list of changed files generated by _get_changed_files() |
||||
:param triggers: list of regexes matching file name that indicates tests should be run |
||||
:return: safe to skip tests |
||||
""" |
||||
for file_name in file_names: |
||||
if any(re.match(trigger, file_name) for trigger in triggers): |
||||
return False |
||||
return True |
||||
|
||||
|
||||
def _remove_irrelevant_tests(tests, tag): |
||||
""" |
||||
Filters out tests by config or language - will not remove sanitizer tests |
||||
:param tests: list of all tests generated by run_tests_matrix.py |
||||
:param tag: string representing language or config to filter - "_(language)_" or "_(config)" |
||||
:return: list of relevant tests |
||||
""" |
||||
# todo(mattkwong): find a more reliable way to filter tests - don't use shortname |
||||
return [test for test in tests if tag not in test.shortname or |
||||
any(san_tag in test.shortname for san_tag in ['_asan', '_tsan', '_msan'])] |
||||
|
||||
|
||||
def _remove_sanitizer_tests(tests): |
||||
""" |
||||
Filters out sanitizer tests |
||||
:param tests: list of all tests generated by run_tests_matrix.py |
||||
:return: list of relevant tests |
||||
""" |
||||
# todo(mattkwong): find a more reliable way to filter tests - don't use shortname |
||||
return [test for test in tests if |
||||
all(san_tag not in test.shortname for san_tag in ['_asan', '_tsan', '_msan'])] |
||||
|
||||
|
||||
def filter_tests(tests, base_branch): |
||||
""" |
||||
Filters out tests that are safe to ignore |
||||
:param tests: list of all tests generated by run_tests_matrix.py |
||||
:return: list of relevant tests |
||||
""" |
||||
print("Finding file differences between %s repo and current branch...\n" % base_branch) |
||||
changed_files = _get_changed_files(base_branch) |
||||
for changed_file in changed_files: |
||||
print(changed_file) |
||||
print |
||||
|
||||
# Regex that combines all keys in _WHITELIST_DICT |
||||
all_triggers = "(" + ")|(".join(_WHITELIST_DICT.keys()) + ")" |
||||
# Check if all tests have to be run |
||||
for changed_file in changed_files: |
||||
if not re.match(all_triggers, changed_file): |
||||
return(tests) |
||||
# Filter out tests by language |
||||
for test_suite in _all_test_suites: |
||||
if _can_skip_tests(changed_files, test_suite.triggers): |
||||
for tag in test_suite.tags: |
||||
print(" Filtering %s tests" % tag) |
||||
tests = _remove_irrelevant_tests(tests, tag) |
||||
# Sanitizer tests skipped if core and c++ are skipped |
||||
if _can_skip_tests(changed_files, _cpp_test_suite.triggers + _core_test_suite.triggers): |
||||
print(" Filtering Sanitizer tests") |
||||
tests = _remove_sanitizer_tests(tests) |
||||
|
||||
return tests |
||||
Loading…
Reference in new issue