Merge github.com:grpc/grpc into trim-the-locks

pull/9906/head
Craig Tiller 8 years ago
commit 350ef9a293
  1. 42
      CMakeLists.txt
  2. 48
      Makefile
  3. 20
      build.yaml
  4. 2
      test/cpp/microbenchmarks/bm_metadata.cc
  5. 254
      test/cpp/microbenchmarks/bm_pollset.cc
  6. 4
      tools/profiling/microbenchmarks/bm_json.py
  7. 2
      tools/run_tests/artifacts/artifact_targets.py
  8. 2
      tools/run_tests/artifacts/distribtest_targets.py
  9. 2
      tools/run_tests/artifacts/package_targets.py
  10. 21
      tools/run_tests/generated/sources_and_headers.json
  11. 22
      tools/run_tests/generated/tests.json
  12. 10
      tools/run_tests/performance/bq_upload_result.py
  13. 2
      tools/run_tests/python_utils/antagonist.py
  14. 7
      tools/run_tests/python_utils/filter_pull_request_tests.py
  15. 2
      tools/run_tests/python_utils/port_server.py
  16. 3
      tools/run_tests/python_utils/report_utils.py
  17. 6
      tools/run_tests/run_build_statistics.py
  18. 13
      tools/run_tests/run_interop_tests.py
  19. 2
      tools/run_tests/run_microbenchmark.py
  20. 11
      tools/run_tests/run_performance_tests.py
  21. 16
      tools/run_tests/run_stress_tests.py
  22. 3
      tools/run_tests/run_tests.py
  23. 4
      tools/run_tests/run_tests_matrix.py
  24. 4
      tools/run_tests/sanity/check_sources_and_headers.py
  25. 4
      tools/run_tests/sanity/check_test_filtering.py
  26. 18
      tools/run_tests/sanity/check_version.py
  27. 6
      tools/run_tests/sanity/core_banned_functions.py
  28. 4
      tools/run_tests/start_port_server.py
  29. 2
      tools/run_tests/stress_test/print_summary.py
  30. 60
      tools/run_tests/stress_test/run_on_gke.py
  31. 2
      tools/run_tests/task_runner.py

@ -603,6 +603,9 @@ endif()
if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
add_dependencies(buildtests_cxx bm_metadata)
endif()
if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
add_dependencies(buildtests_cxx bm_pollset)
endif()
add_dependencies(buildtests_cxx channel_arguments_test)
add_dependencies(buildtests_cxx channel_filter_test)
add_dependencies(buildtests_cxx cli_call_test)
@ -8011,6 +8014,45 @@ target_link_libraries(bm_metadata
${_gRPC_GFLAGS_LIBRARIES}
)
endif()
endif (gRPC_BUILD_TESTS)
if (gRPC_BUILD_TESTS)
if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX)
add_executable(bm_pollset
test/cpp/microbenchmarks/bm_pollset.cc
third_party/googletest/src/gtest-all.cc
)
target_include_directories(bm_pollset
PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include
PRIVATE ${BORINGSSL_ROOT_DIR}/include
PRIVATE ${PROTOBUF_ROOT_DIR}/src
PRIVATE ${BENCHMARK_ROOT_DIR}/include
PRIVATE ${ZLIB_ROOT_DIR}
PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib
PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include
PRIVATE third_party/googletest/include
PRIVATE third_party/googletest
PRIVATE ${_gRPC_PROTO_GENS_DIR}
)
target_link_libraries(bm_pollset
${_gRPC_PROTOBUF_LIBRARIES}
${_gRPC_ALLTARGETS_LIBRARIES}
grpc_benchmark
benchmark
grpc++_test_util
grpc_test_util
grpc++
grpc
gpr_test_util
gpr
${_gRPC_GFLAGS_LIBRARIES}
)
endif()
endif (gRPC_BUILD_TESTS)
if (gRPC_BUILD_TESTS)

@ -1057,6 +1057,7 @@ bm_fullstack_streaming_pump: $(BINDIR)/$(CONFIG)/bm_fullstack_streaming_pump
bm_fullstack_trickle: $(BINDIR)/$(CONFIG)/bm_fullstack_trickle
bm_fullstack_unary_ping_pong: $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong
bm_metadata: $(BINDIR)/$(CONFIG)/bm_metadata
bm_pollset: $(BINDIR)/$(CONFIG)/bm_pollset
channel_arguments_test: $(BINDIR)/$(CONFIG)/channel_arguments_test
channel_filter_test: $(BINDIR)/$(CONFIG)/channel_filter_test
cli_call_test: $(BINDIR)/$(CONFIG)/cli_call_test
@ -1478,6 +1479,7 @@ buildtests_cxx: privatelibs_cxx \
$(BINDIR)/$(CONFIG)/bm_fullstack_trickle \
$(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong \
$(BINDIR)/$(CONFIG)/bm_metadata \
$(BINDIR)/$(CONFIG)/bm_pollset \
$(BINDIR)/$(CONFIG)/channel_arguments_test \
$(BINDIR)/$(CONFIG)/channel_filter_test \
$(BINDIR)/$(CONFIG)/cli_call_test \
@ -1593,6 +1595,7 @@ buildtests_cxx: privatelibs_cxx \
$(BINDIR)/$(CONFIG)/bm_fullstack_trickle \
$(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong \
$(BINDIR)/$(CONFIG)/bm_metadata \
$(BINDIR)/$(CONFIG)/bm_pollset \
$(BINDIR)/$(CONFIG)/channel_arguments_test \
$(BINDIR)/$(CONFIG)/channel_filter_test \
$(BINDIR)/$(CONFIG)/cli_call_test \
@ -1943,6 +1946,8 @@ test_cxx: buildtests_cxx
$(Q) $(BINDIR)/$(CONFIG)/bm_fullstack_unary_ping_pong || ( echo test bm_fullstack_unary_ping_pong failed ; exit 1 )
$(E) "[RUN] Testing bm_metadata"
$(Q) $(BINDIR)/$(CONFIG)/bm_metadata || ( echo test bm_metadata failed ; exit 1 )
$(E) "[RUN] Testing bm_pollset"
$(Q) $(BINDIR)/$(CONFIG)/bm_pollset || ( echo test bm_pollset failed ; exit 1 )
$(E) "[RUN] Testing channel_arguments_test"
$(Q) $(BINDIR)/$(CONFIG)/channel_arguments_test || ( echo test channel_arguments_test failed ; exit 1 )
$(E) "[RUN] Testing channel_filter_test"
@ -13069,6 +13074,49 @@ endif
endif
BM_POLLSET_SRC = \
test/cpp/microbenchmarks/bm_pollset.cc \
BM_POLLSET_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(BM_POLLSET_SRC))))
ifeq ($(NO_SECURE),true)
# You can't build secure targets if you don't have OpenSSL.
$(BINDIR)/$(CONFIG)/bm_pollset: openssl_dep_error
else
ifeq ($(NO_PROTOBUF),true)
# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+.
$(BINDIR)/$(CONFIG)/bm_pollset: protobuf_dep_error
else
$(BINDIR)/$(CONFIG)/bm_pollset: $(PROTOBUF_DEP) $(BM_POLLSET_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
$(E) "[LD] Linking $@"
$(Q) mkdir -p `dirname $@`
$(Q) $(LDXX) $(LDFLAGS) $(BM_POLLSET_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/bm_pollset
endif
endif
$(OBJDIR)/$(CONFIG)/test/cpp/microbenchmarks/bm_pollset.o: $(LIBDIR)/$(CONFIG)/libgrpc_benchmark.a $(LIBDIR)/$(CONFIG)/libbenchmark.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
deps_bm_pollset: $(BM_POLLSET_OBJS:.o=.dep)
ifneq ($(NO_SECURE),true)
ifneq ($(NO_DEPS),true)
-include $(BM_POLLSET_OBJS:.o=.dep)
endif
endif
CHANNEL_ARGUMENTS_TEST_SRC = \
test/cpp/common/channel_arguments_test.cc \

@ -3270,6 +3270,26 @@ targets:
- mac
- linux
- posix
- name: bm_pollset
build: test
language: c++
src:
- test/cpp/microbenchmarks/bm_pollset.cc
deps:
- grpc_benchmark
- benchmark
- grpc++_test_util
- grpc_test_util
- grpc++
- grpc
- gpr_test_util
- gpr
args:
- --benchmark_min_time=0
platforms:
- mac
- linux
- posix
- name: channel_arguments_test
gtest: true
build: test

@ -1,6 +1,6 @@
/*
*
* Copyright 2015, Google Inc.
* Copyright 2017, Google Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without

@ -0,0 +1,254 @@
/*
*
* Copyright 2017, Google Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Google Inc. nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
/* Test out pollset latencies */
#include <grpc/grpc.h>
#include <grpc/support/alloc.h>
#include <grpc/support/log.h>
#include <grpc/support/useful.h>
extern "C" {
#include "src/core/lib/iomgr/ev_posix.h"
#include "src/core/lib/iomgr/pollset.h"
#include "src/core/lib/iomgr/port.h"
#include "src/core/lib/iomgr/wakeup_fd_posix.h"
}
#include "test/cpp/microbenchmarks/helpers.h"
#include "third_party/benchmark/include/benchmark/benchmark.h"
#include <string.h>
#ifdef GRPC_LINUX_MULTIPOLL_WITH_EPOLL
#include <sys/epoll.h>
#include <sys/eventfd.h>
#include <unistd.h>
#endif
auto& force_library_initialization = Library::get();
static void shutdown_ps(grpc_exec_ctx* exec_ctx, void* ps, grpc_error* error) {
grpc_pollset_destroy(static_cast<grpc_pollset*>(ps));
}
static void BM_CreateDestroyPollset(benchmark::State& state) {
TrackCounters track_counters;
size_t ps_sz = grpc_pollset_size();
grpc_pollset* ps = static_cast<grpc_pollset*>(gpr_malloc(ps_sz));
gpr_mu* mu;
grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
grpc_closure shutdown_ps_closure;
grpc_closure_init(&shutdown_ps_closure, shutdown_ps, ps,
grpc_schedule_on_exec_ctx);
while (state.KeepRunning()) {
memset(ps, 0, ps_sz);
grpc_pollset_init(ps, &mu);
gpr_mu_lock(mu);
grpc_pollset_shutdown(&exec_ctx, ps, &shutdown_ps_closure);
gpr_mu_unlock(mu);
grpc_exec_ctx_flush(&exec_ctx);
}
grpc_exec_ctx_finish(&exec_ctx);
gpr_free(ps);
track_counters.Finish(state);
}
BENCHMARK(BM_CreateDestroyPollset);
#ifdef GRPC_LINUX_MULTIPOLL_WITH_EPOLL
static void BM_PollEmptyPollset_SpeedOfLight(benchmark::State& state) {
// equivalent to BM_PollEmptyPollset, but just use the OS primitives to guage
// what the speed of light would be if we abstracted perfectly
TrackCounters track_counters;
int epfd = epoll_create1(0);
GPR_ASSERT(epfd != -1);
size_t nev = state.range(0);
size_t nfd = state.range(1);
epoll_event* ev = new epoll_event[nev];
std::vector<int> fds;
for (size_t i = 0; i < nfd; i++) {
fds.push_back(eventfd(0, 0));
epoll_event ev;
ev.events = EPOLLIN;
epoll_ctl(epfd, EPOLL_CTL_ADD, fds.back(), &ev);
}
while (state.KeepRunning()) {
epoll_wait(epfd, ev, nev, 0);
}
for (auto fd : fds) {
close(fd);
}
close(epfd);
delete[] ev;
track_counters.Finish(state);
}
BENCHMARK(BM_PollEmptyPollset_SpeedOfLight)
->Args({1, 0})
->Args({1, 1})
->Args({1, 10})
->Args({1, 100})
->Args({1, 1000})
->Args({1, 10000})
->Args({1, 100000})
->Args({10, 1})
->Args({100, 1})
->Args({1000, 1});
#endif
static void BM_PollEmptyPollset(benchmark::State& state) {
TrackCounters track_counters;
size_t ps_sz = grpc_pollset_size();
grpc_pollset* ps = static_cast<grpc_pollset*>(gpr_zalloc(ps_sz));
gpr_mu* mu;
grpc_pollset_init(ps, &mu);
grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
gpr_timespec now = gpr_time_0(GPR_CLOCK_MONOTONIC);
gpr_timespec deadline = gpr_inf_past(GPR_CLOCK_MONOTONIC);
gpr_mu_lock(mu);
while (state.KeepRunning()) {
grpc_pollset_worker* worker;
GRPC_ERROR_UNREF(grpc_pollset_work(&exec_ctx, ps, &worker, now, deadline));
}
grpc_closure shutdown_ps_closure;
grpc_closure_init(&shutdown_ps_closure, shutdown_ps, ps,
grpc_schedule_on_exec_ctx);
grpc_pollset_shutdown(&exec_ctx, ps, &shutdown_ps_closure);
gpr_mu_unlock(mu);
grpc_exec_ctx_finish(&exec_ctx);
gpr_free(ps);
track_counters.Finish(state);
}
BENCHMARK(BM_PollEmptyPollset);
class Closure : public grpc_closure {
public:
virtual ~Closure() {}
};
template <class F>
Closure* MakeClosure(F f, grpc_closure_scheduler* scheduler) {
struct C : public Closure {
C(F f, grpc_closure_scheduler* scheduler) : f_(f) {
grpc_closure_init(this, C::cbfn, this, scheduler);
}
static void cbfn(grpc_exec_ctx* exec_ctx, void* arg, grpc_error* error) {
C* p = static_cast<C*>(arg);
p->f_();
}
F f_;
};
return new C(f, scheduler);
}
#ifdef GRPC_LINUX_MULTIPOLL_WITH_EPOLL
static void BM_SingleThreadPollOneFd_SpeedOfLight(benchmark::State& state) {
// equivalent to BM_PollEmptyPollset, but just use the OS primitives to guage
// what the speed of light would be if we abstracted perfectly
TrackCounters track_counters;
int epfd = epoll_create1(0);
GPR_ASSERT(epfd != -1);
epoll_event ev[100];
int fd = eventfd(0, EFD_NONBLOCK);
ev[0].events = EPOLLIN;
epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &ev[0]);
while (state.KeepRunning()) {
int err;
do {
err = eventfd_write(fd, 1);
} while (err < 0 && errno == EINTR);
GPR_ASSERT(err == 0);
do {
err = epoll_wait(epfd, ev, GPR_ARRAY_SIZE(ev), 0);
} while (err < 0 && errno == EINTR);
GPR_ASSERT(err == 1);
eventfd_t value;
do {
err = eventfd_read(fd, &value);
} while (err < 0 && errno == EINTR);
GPR_ASSERT(err == 0);
}
close(fd);
close(epfd);
track_counters.Finish(state);
}
BENCHMARK(BM_SingleThreadPollOneFd_SpeedOfLight);
#endif
static void BM_SingleThreadPollOneFd(benchmark::State& state) {
TrackCounters track_counters;
size_t ps_sz = grpc_pollset_size();
grpc_pollset* ps = static_cast<grpc_pollset*>(gpr_zalloc(ps_sz));
gpr_mu* mu;
grpc_pollset_init(ps, &mu);
grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
gpr_timespec now = gpr_time_0(GPR_CLOCK_MONOTONIC);
gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC);
grpc_wakeup_fd wakeup_fd;
GRPC_ERROR_UNREF(grpc_wakeup_fd_init(&wakeup_fd));
grpc_fd* wakeup = grpc_fd_create(wakeup_fd.read_fd, "wakeup_read");
grpc_pollset_add_fd(&exec_ctx, ps, wakeup);
bool done = false;
Closure* continue_closure = MakeClosure(
[&]() {
GRPC_ERROR_UNREF(grpc_wakeup_fd_consume_wakeup(&wakeup_fd));
if (!state.KeepRunning()) {
done = true;
return;
}
GRPC_ERROR_UNREF(grpc_wakeup_fd_wakeup(&wakeup_fd));
grpc_fd_notify_on_read(&exec_ctx, wakeup, continue_closure);
},
grpc_schedule_on_exec_ctx);
GRPC_ERROR_UNREF(grpc_wakeup_fd_wakeup(&wakeup_fd));
grpc_fd_notify_on_read(&exec_ctx, wakeup, continue_closure);
gpr_mu_lock(mu);
while (!done) {
grpc_pollset_worker* worker;
GRPC_ERROR_UNREF(grpc_pollset_work(&exec_ctx, ps, &worker, now, deadline));
}
grpc_fd_orphan(&exec_ctx, wakeup, NULL, NULL, "done");
wakeup_fd.read_fd = 0;
grpc_closure shutdown_ps_closure;
grpc_closure_init(&shutdown_ps_closure, shutdown_ps, ps,
grpc_schedule_on_exec_ctx);
grpc_pollset_shutdown(&exec_ctx, ps, &shutdown_ps_closure);
gpr_mu_unlock(mu);
grpc_exec_ctx_finish(&exec_ctx);
grpc_wakeup_fd_destroy(&wakeup_fd);
gpr_free(ps);
track_counters.Finish(state);
delete continue_closure;
}
BENCHMARK(BM_SingleThreadPollOneFd);
BENCHMARK_MAIN();

@ -98,6 +98,10 @@ _BM_SPECS = {
'tpl': [],
'dyn': ['request_size'],
},
'BM_PollEmptyPollset_SpeedOfLight': {
'tpl': [],
'dyn': ['request_size', 'request_count'],
}
}
def numericalize(s):

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python
# Copyright 2016, Google Inc.
# All rights reserved.
#

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python
# Copyright 2016, Google Inc.
# All rights reserved.
#

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python
# Copyright 2016, Google Inc.
# All rights reserved.
#

@ -2639,6 +2639,27 @@
"third_party": false,
"type": "target"
},
{
"deps": [
"benchmark",
"gpr",
"gpr_test_util",
"grpc",
"grpc++",
"grpc++_test_util",
"grpc_benchmark",
"grpc_test_util"
],
"headers": [],
"is_filegroup": false,
"language": "c++",
"name": "bm_pollset",
"src": [
"test/cpp/microbenchmarks/bm_pollset.cc"
],
"third_party": false,
"type": "target"
},
{
"deps": [
"gpr",

@ -2847,6 +2847,28 @@
"posix"
]
},
{
"args": [
"--benchmark_min_time=0"
],
"ci_platforms": [
"linux",
"mac",
"posix"
],
"cpu_cost": 1.0,
"exclude_configs": [],
"exclude_iomgrs": [],
"flaky": false,
"gtest": false,
"language": "c++",
"name": "bm_pollset",
"platforms": [
"linux",
"mac",
"posix"
]
},
{
"args": [],
"ci_platforms": [

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python
# Copyright 2016, Google Inc.
# All rights reserved.
#
@ -30,6 +30,8 @@
# Uploads performance benchmark result file to bigquery.
from __future__ import print_function
import argparse
import calendar
import json
@ -70,7 +72,7 @@ def _upload_netperf_latency_csv_to_bigquery(dataset_id, table_id, result_file):
_create_results_table(bq, dataset_id, table_id)
if not _insert_result(bq, dataset_id, table_id, scenario_result, flatten=False):
print 'Error uploading result to bigquery.'
print('Error uploading result to bigquery.')
sys.exit(1)
@ -82,7 +84,7 @@ def _upload_scenario_result_to_bigquery(dataset_id, table_id, result_file):
_create_results_table(bq, dataset_id, table_id)
if not _insert_result(bq, dataset_id, table_id, scenario_result):
print 'Error uploading result to bigquery.'
print('Error uploading result to bigquery.')
sys.exit(1)
@ -179,4 +181,4 @@ if args.file_format == 'netperf_latency_csv':
_upload_netperf_latency_csv_to_bigquery(dataset_id, table_id, args.file_to_upload)
else:
_upload_scenario_result_to_bigquery(dataset_id, table_id, args.file_to_upload)
print 'Successfully uploaded %s to BigQuery.\n' % args.file_to_upload
print('Successfully uploaded %s to BigQuery.\n' % args.file_to_upload)

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python
# Copyright 2015, Google Inc.
# All rights reserved.
#

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python
# Copyright 2015, Google Inc.
# All rights reserved.
#
@ -30,7 +30,10 @@
"""Filter out tests based on file differences compared to merge target branch"""
from __future__ import print_function
import re
import six
from subprocess import check_output
@ -125,7 +128,7 @@ _WHITELIST_DICT = {
}
# Add all triggers to their respective test suites
for trigger, test_suites in _WHITELIST_DICT.iteritems():
for trigger, test_suites in six.iteritems(_WHITELIST_DICT):
for test_suite in test_suites:
test_suite.add_trigger(trigger)

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python
# Copyright 2015, Google Inc.
# All rights reserved.
#

@ -40,6 +40,7 @@ except (ImportError):
import os
import string
import xml.etree.cElementTree as ET
import six
def _filter_msg(msg, output_format):
@ -63,7 +64,7 @@ def render_junit_xml_report(resultset, xml_report, suite_package='grpc',
root = ET.Element('testsuites')
testsuite = ET.SubElement(root, 'testsuite', id='1', package=suite_package,
name=suite_name)
for shortname, results in resultset.iteritems():
for shortname, results in six.iteritems(resultset):
for result in results:
xml_test = ET.SubElement(testsuite, 'testcase', name=shortname)
if result.elapsed_time:

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python
# Copyright 2016, Google Inc.
# All rights reserved.
#
@ -30,6 +30,8 @@
"""Tool to get build statistics from Jenkins and upload to BigQuery."""
from __future__ import print_function
import argparse
import jenkinsapi
from jenkinsapi.custom_exceptions import JenkinsAPIException
@ -243,6 +245,6 @@ for build_name in _BUILDS.keys() if 'all' in args.builds else args.builds:
rows = [big_query_utils.make_row(build_number, build_result)]
if not big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, build_name,
rows):
print '====> Error uploading result to bigquery.'
print('====> Error uploading result to bigquery.')
sys.exit(1)

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python
# Copyright 2015, Google Inc.
# All rights reserved.
#
@ -44,6 +44,7 @@ import sys
import tempfile
import time
import uuid
import six
import python_utils.dockerjob as dockerjob
import python_utils.jobset as jobset
@ -885,7 +886,7 @@ if not args.use_docker and servers:
languages = set(_LANGUAGES[l]
for l in itertools.chain.from_iterable(
_LANGUAGES.iterkeys() if x == 'all' else [x]
six.iterkeys(_LANGUAGES) if x == 'all' else [x]
for x in args.language))
languages_http2_badserver_interop = set()
@ -925,7 +926,7 @@ if args.use_docker:
else:
jobset.message('FAILED', 'Failed to build interop docker images.',
do_newline=True)
for image in docker_images.itervalues():
for image in six.itervalues(docker_images):
dockerjob.remove_image(image, skip_nonexistent=True)
sys.exit(1)
@ -1053,7 +1054,7 @@ try:
if not jobs:
print('No jobs to run.')
for image in docker_images.itervalues():
for image in six.itervalues(docker_images):
dockerjob.remove_image(image, skip_nonexistent=True)
sys.exit(1)
@ -1093,9 +1094,9 @@ finally:
if not job.is_running():
print('Server "%s" has exited prematurely.' % server)
dockerjob.finish_jobs([j for j in server_jobs.itervalues()])
dockerjob.finish_jobs([j for j in six.itervalues(server_jobs)])
for image in docker_images.itervalues():
for image in six.itervalues(docker_images):
if not args.manual_run:
print('Removing docker image %s' % image)
dockerjob.remove_image(image)

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python
# Copyright 2017, Google Inc.
# All rights reserved.
#

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python
# Copyright 2016, Google Inc.
# All rights reserved.
#
@ -46,6 +46,7 @@ import tempfile
import time
import traceback
import uuid
import six
import performance.scenario_config as scenario_config
import python_utils.jobset as jobset
@ -502,8 +503,8 @@ args = argp.parse_args()
languages = set(scenario_config.LANGUAGES[l]
for l in itertools.chain.from_iterable(
scenario_config.LANGUAGES.iterkeys() if x == 'all' else [x]
for x in args.language))
six.iterkeys(scenario_config.LANGUAGES) if x == 'all'
else [x] for x in args.language))
# Put together set of remote hosts where to run and build
@ -572,8 +573,8 @@ for scenario in scenarios:
jobs.append(create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host))
scenario_failures, resultset = jobset.run(jobs, newline_on_success=True, maxjobs=1)
total_scenario_failures += scenario_failures
merged_resultset = dict(itertools.chain(merged_resultset.iteritems(),
resultset.iteritems()))
merged_resultset = dict(itertools.chain(six.iteritems(merged_resultset),
six.iteritems(resultset)))
finally:
# Consider qps workers that need to be killed as failures
qps_workers_killed += finish_qps_workers(scenario.workers)

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python
# Copyright 2015, Google Inc.
# All rights reserved.
#
@ -43,6 +43,7 @@ import sys
import tempfile
import time
import uuid
import six
import python_utils.dockerjob as dockerjob
import python_utils.jobset as jobset
@ -239,9 +240,8 @@ servers = set(
for s in itertools.chain.from_iterable(_SERVERS if x == 'all' else [x]
for x in args.server))
languages = set(_LANGUAGES[l]
for l in itertools.chain.from_iterable(_LANGUAGES.iterkeys(
) if x == 'all' else [x] for x in args.language))
languages = set(_LANGUAGES[l] for l in itertools.chain.from_iterable(
six.iterkeys(_LANGUAGES) if x == 'all' else [x] for x in args.language))
docker_images = {}
# languages for which to build docker images
@ -267,7 +267,7 @@ if build_jobs:
jobset.message('FAILED',
'Failed to build interop docker images.',
do_newline=True)
for image in docker_images.itervalues():
for image in six.itervalues(docker_images):
dockerjob.remove_image(image, skip_nonexistent=True)
sys.exit(1)
@ -306,7 +306,7 @@ try:
if not jobs:
print('No jobs to run.')
for image in docker_images.itervalues():
for image in six.itervalues(docker_images):
dockerjob.remove_image(image, skip_nonexistent=True)
sys.exit(1)
@ -324,8 +324,8 @@ finally:
if not job.is_running():
print('Server "%s" has exited prematurely.' % server)
dockerjob.finish_jobs([j for j in server_jobs.itervalues()])
dockerjob.finish_jobs([j for j in six.itervalues(server_jobs)])
for image in docker_images.itervalues():
for image in six.itervalues(docker_images):
print('Removing docker image %s' % image)
dockerjob.remove_image(image)

@ -54,6 +54,7 @@ import traceback
import time
from six.moves import urllib
import uuid
import six
import python_utils.jobset as jobset
import python_utils.report_utils as report_utils
@ -771,7 +772,7 @@ class CSharpLanguage(object):
runtime_cmd = ['mono']
specs = []
for assembly in tests_by_assembly.iterkeys():
for assembly in six.iterkeys(tests_by_assembly):
assembly_file = 'src/csharp/%s/%s/%s%s' % (assembly,
assembly_subdir,
assembly,

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python
# Copyright 2015, Google Inc.
# All rights reserved.
#
@ -30,6 +30,8 @@
"""Run test matrix."""
from __future__ import print_function
import argparse
import multiprocessing
import os

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python
# Copyright 2015, Google Inc.
# All rights reserved.
#
@ -28,6 +28,8 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from __future__ import print_function
import json
import os
import re

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python
# Copyright 2016, Google Inc.
# All rights reserved.
@ -62,7 +62,7 @@ class TestFilteringTest(unittest.TestCase):
def _get_changed_files(foo):
return changed_files
filter_pull_request_tests._get_changed_files = _get_changed_files
print
print()
filtered_jobs = filter_pull_request_tests.filter_tests(all_jobs, "test")
# Make sure sanity tests aren't being filtered out

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python
# Copyright 2016, Google Inc.
# All rights reserved.
@ -29,6 +29,8 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from __future__ import print_function
import sys
import yaml
import os
@ -48,13 +50,13 @@ try:
'git rev-parse --abbrev-ref HEAD',
shell=True)
except:
print 'WARNING: not a git repository'
print('WARNING: not a git repository')
branch_name = None
if branch_name is not None:
m = re.match(r'^release-([0-9]+)_([0-9]+)$', branch_name)
if m:
print 'RELEASE branch'
print('RELEASE branch')
# version number should align with the branched version
check_version = lambda version: (
version.major == int(m.group(1)) and
@ -78,7 +80,7 @@ settings = build_yaml['settings']
top_version = Version(settings['version'])
if not check_version(top_version):
errors += 1
print warning % ('version', top_version)
print(warning % ('version', top_version))
for tag, value in settings.iteritems():
if re.match(r'^[a-z]+_version$', tag):
@ -86,12 +88,14 @@ for tag, value in settings.iteritems():
if tag != 'core_version':
if value.major != top_version.major:
errors += 1
print 'major version mismatch on %s: %d vs %d' % (tag, value.major, top_version.major)
print('major version mismatch on %s: %d vs %d' % (tag, value.major,
top_version.major))
if value.minor != top_version.minor:
errors += 1
print 'minor version mismatch on %s: %d vs %d' % (tag, value.minor, top_version.minor)
print('minor version mismatch on %s: %d vs %d' % (tag, value.minor,
top_version.minor))
if not check_version(value):
errors += 1
print warning % (tag, value)
print(warning % (tag, value))
sys.exit(errors)

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python
# Copyright 2016, Google Inc.
# All rights reserved.
@ -29,6 +29,8 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from __future__ import print_function
import os
import sys
@ -60,7 +62,7 @@ for root, dirs, files in os.walk('src/core'):
for banned, exceptions in BANNED_EXCEPT.items():
if path in exceptions: continue
if banned in text:
print 'Illegal use of "%s" in %s' % (banned, path)
print('Illegal use of "%s" in %s' % (banned, path))
errors += 1
assert errors == 0

@ -39,8 +39,10 @@ The path to this file is called out in test/core/util/port.c, and printed as
an error message to users.
"""
from __future__ import print_function
import python_utils.start_port_server as start_port_server
start_port_server.start_port_server()
print "Port server started successfully"
print("Port server started successfully")

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python
# Copyright 2016, Google Inc.
# All rights reserved.
#

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python
# Copyright 2015-2016, Google Inc.
# All rights reserved.
#
@ -27,6 +27,9 @@
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from __future__ import print_function
import argparse
import datetime
import json
@ -124,23 +127,24 @@ class DockerImage:
return 'gcr.io/%s/%s' % (project_id, image_name)
def build_image(self):
print 'Building docker image: %s (tag: %s)' % (self.image_name,
self.tag_name)
print('Building docker image: %s (tag: %s)' % (self.image_name,
self.tag_name))
os.environ['INTEROP_IMAGE'] = self.image_name
os.environ['INTEROP_IMAGE_REPOSITORY_TAG'] = self.tag_name
os.environ['BASE_NAME'] = self.dockerfile_dir
os.environ['BUILD_TYPE'] = self.build_type
print 'DEBUG: path: ', self.build_script_path
print('DEBUG: path: ', self.build_script_path)
if subprocess.call(args=[self.build_script_path]) != 0:
print 'Error in building the Docker image'
print('Error in building the Docker image')
return False
return True
def push_to_gke_registry(self):
cmd = ['gcloud', 'docker', 'push', self.tag_name]
print 'Pushing %s to the GKE registry..' % self.tag_name
print('Pushing %s to the GKE registry..' % self.tag_name)
if subprocess.call(args=cmd) != 0:
print 'Error in pushing the image %s to the GKE registry' % self.tag_name
print('Error in pushing the image %s to the GKE registry' %
self.tag_name)
return False
return True
@ -199,11 +203,11 @@ class Gke:
cmd = ['kubectl', 'proxy', '--port=%d' % port]
self.p = subprocess.Popen(args=cmd)
time.sleep(2)
print '\nStarted kubernetes proxy on port: %d' % port
print('\nStarted kubernetes proxy on port: %d' % port)
def __del__(self):
if self.p is not None:
print 'Shutting down Kubernetes proxy..'
print('Shutting down Kubernetes proxy..')
self.p.kill()
def __init__(self, project_id, run_id, dataset_id, summary_table_id,
@ -253,7 +257,7 @@ class Gke:
for pod_name in server_pod_spec.pod_names():
server_env['POD_NAME'] = pod_name
print 'Creating server: %s' % pod_name
print('Creating server: %s' % pod_name)
is_success = kubernetes_api.create_pod_and_service(
'localhost',
self.kubernetes_port,
@ -267,11 +271,11 @@ class Gke:
True # Headless = True for server to that GKE creates a DNS record for pod_name
)
if not is_success:
print 'Error in launching server: %s' % pod_name
print('Error in launching server: %s' % pod_name)
break
if is_success:
print 'Successfully created server(s)'
print('Successfully created server(s)')
return is_success
@ -301,7 +305,7 @@ class Gke:
for pod_name in client_pod_spec.pod_names():
client_env['POD_NAME'] = pod_name
print 'Creating client: %s' % pod_name
print('Creating client: %s' % pod_name)
is_success = kubernetes_api.create_pod_and_service(
'localhost',
self.kubernetes_port,
@ -316,18 +320,18 @@ class Gke:
)
if not is_success:
print 'Error in launching client %s' % pod_name
print('Error in launching client %s' % pod_name)
break
if is_success:
print 'Successfully created all client(s)'
print('Successfully created all client(s)')
return is_success
def _delete_pods(self, pod_name_list):
is_success = True
for pod_name in pod_name_list:
print 'Deleting %s' % pod_name
print('Deleting %s' % pod_name)
is_success = kubernetes_api.delete_pod_and_service(
'localhost',
self.kubernetes_port,
@ -335,11 +339,11 @@ class Gke:
pod_name)
if not is_success:
print 'Error in deleting pod %s' % pod_name
print('Error in deleting pod %s' % pod_name)
break
if is_success:
print 'Successfully deleted all pods'
print('Successfully deleted all pods')
return is_success
@ -353,7 +357,7 @@ class Gke:
class Config:
def __init__(self, config_filename, gcp_project_id):
print 'Loading configuration...'
print('Loading configuration...')
config_dict = self._load_config(config_filename)
self.global_settings = self._parse_global_settings(config_dict,
@ -367,7 +371,7 @@ class Config:
self.client_pod_specs_dict = self._parse_client_pod_specs(
config_dict, self.docker_images_dict, self.client_templates_dict,
self.server_pod_specs_dict)
print 'Loaded Configuaration.'
print('Loaded Configuaration.')
def _parse_global_settings(self, config_dict, gcp_project_id):
global_settings_dict = config_dict['globalSettings']
@ -540,8 +544,8 @@ def run_tests(config):
# run id. This is useful in debugging when looking at records in Biq query)
run_id = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S')
dataset_id = '%s_%s' % (config.global_settings.dataset_id_prefix, run_id)
print 'Run id:', run_id
print 'Dataset id:', dataset_id
print('Run id:', run_id)
print('Dataset id:', dataset_id)
bq_helper = BigQueryHelper(run_id, '', '',
config.global_settings.gcp_project_id, dataset_id,
@ -557,7 +561,7 @@ def run_tests(config):
is_success = True
try:
print 'Launching servers..'
print('Launching servers..')
for name, server_pod_spec in config.server_pod_specs_dict.iteritems():
if not gke.launch_servers(server_pod_spec):
is_success = False # is_success is checked in the 'finally' block
@ -579,11 +583,12 @@ def run_tests(config):
start_time = datetime.datetime.now()
end_time = start_time + datetime.timedelta(
seconds=config.global_settings.test_duration_secs)
print 'Running the test until %s' % end_time.isoformat()
print('Running the test until %s' % end_time.isoformat())
while True:
if datetime.datetime.now() > end_time:
print 'Test was run for %d seconds' % config.global_settings.test_duration_secs
print('Test was run for %d seconds' %
config.global_settings.test_duration_secs)
break
# Check if either stress server or clients have failed (btw, the bq_helper
@ -591,11 +596,12 @@ def run_tests(config):
# have a failure status)
if bq_helper.check_if_any_tests_failed():
is_success = False
print 'Some tests failed.'
print('Some tests failed.')
break # Don't 'return' here. We still want to call bq_helper to print qps/summary tables
# Tests running fine. Wait until next poll time to check the status
print 'Sleeping for %d seconds..' % config.global_settings.test_poll_interval_secs
print('Sleeping for %d seconds..' %
config.global_settings.test_poll_interval_secs)
time.sleep(config.global_settings.test_poll_interval_secs)
# Print BiqQuery tables

@ -1,4 +1,4 @@
#!/usr/bin/env python2.7
#!/usr/bin/env python
# Copyright 2016, Google Inc.
# All rights reserved.
#

Loading…
Cancel
Save