Merge branch 'master' into externC

pull/12692/head
Yash Tibrewal 7 years ago
commit 70d3424005
  1. 3
      .gitmodules
  2. 4
      src/core/ext/transport/chttp2/transport/chttp2_transport.cc
  3. 6
      src/csharp/Grpc.Core/Internal/AsyncCall.cs
  4. 1
      src/ruby/ext/grpc/rb_call.c
  5. 4
      test/cpp/qps/gen_build_yaml.py
  6. 1
      third_party/bloaty
  7. 8
      tools/internal_ci/helper_scripts/prepare_build_macos_rc
  8. 2
      tools/internal_ci/linux/grpc_microbenchmark_diff.sh
  9. 99
      tools/profiling/bloat/bloat_diff.py
  10. 280
      tools/run_tests/generated/tests.json
  11. 9
      tools/run_tests/python_utils/jobset.py
  12. 9
      tools/run_tests/run_tests.py
  13. 1
      tools/run_tests/sanity/check_submodules.sh

3
.gitmodules vendored

@ -24,3 +24,6 @@
path = third_party/cares/cares
url = https://github.com/c-ares/c-ares.git
branch = cares-1_12_0
[submodule "third_party/bloaty"]
path = third_party/bloaty
url = https://github.com/google/bloaty.git

@ -1898,7 +1898,9 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx,
&s->frame_storage);
s->unprocessed_incoming_frames_decompressed = false;
}
if (!s->unprocessed_incoming_frames_decompressed) {
if (!s->unprocessed_incoming_frames_decompressed &&
s->stream_decompression_method !=
GRPC_STREAM_COMPRESSION_IDENTITY_DECOMPRESS) {
GPR_ASSERT(s->decompressed_data_buffer.length == 0);
bool end_of_context;
if (!s->stream_decompression_ctx) {

@ -34,6 +34,9 @@ namespace Grpc.Core.Internal
readonly CallInvocationDetails<TRequest, TResponse> details;
readonly INativeCall injectedNativeCall; // for testing
// Dispose of to de-register cancellation token registration
IDisposable cancellationTokenRegistration;
// Completion of a pending unary response if not null.
TaskCompletionSource<TResponse> unaryResponseTcs;
@ -320,6 +323,7 @@ namespace Grpc.Core.Internal
protected override void OnAfterReleaseResources()
{
details.Channel.RemoveCallReference(this);
cancellationTokenRegistration?.Dispose();
}
protected override bool IsClient
@ -405,7 +409,7 @@ namespace Grpc.Core.Internal
var token = details.Options.CancellationToken;
if (token.CanBeCanceled)
{
token.Register(() => this.Cancel());
cancellationTokenRegistration = token.Register(() => this.Cancel());
}
}

@ -101,6 +101,7 @@ static void grpc_rb_call_destroy(void *p) {
return;
}
destroy_call((grpc_rb_call *)p);
xfree(p);
}
static size_t md_ary_datasize(const void *p) {

@ -77,7 +77,7 @@ print yaml.dump({
'defaults': 'boringssl',
'cpu_cost': guess_cpu(scenario_json, False),
'exclude_configs': ['tsan', 'asan'],
'timeout_seconds': 6*60,
'timeout_seconds': 2*60,
'excluded_poll_engines': scenario_json.get('EXCLUDED_POLL_ENGINES', [])
}
for scenario_json in scenario_config.CXXLanguage().scenarios()
@ -95,7 +95,7 @@ print yaml.dump({
'defaults': 'boringssl',
'cpu_cost': guess_cpu(scenario_json, True),
'exclude_configs': sorted(c for c in configs_from_yaml if c not in ('tsan', 'asan')),
'timeout_seconds': 6*60,
'timeout_seconds': 2*60,
'excluded_poll_engines': scenario_json.get('EXCLUDED_POLL_ENGINES', [])
}
for scenario_json in scenario_config.CXXLanguage().scenarios()

@ -0,0 +1 @@
Subproject commit 73594cde8c9a52a102c4341c244c833aa61b9c06

@ -27,6 +27,14 @@ ulimit -n 10000
# show current limits
ulimit -a
# synchronize the clock
date
sudo systemsetup -setusingnetworktime off
sudo systemsetup -setnetworktimeserver "$( ipconfig getoption en0 server_identifier )"
sudo systemsetup -settimezone America/Los_Angeles
sudo systemsetup -setusingnetworktime on
date
# Add GCP credentials for BQ access
pip install google-api-python-client --user python
export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json

@ -25,6 +25,8 @@ cd $(dirname $0)/../../..
source tools/internal_ci/helper_scripts/prepare_build_linux_perf_rc
tools/run_tests/start_port_server.py
tools/jenkins/run_c_cpp_test.sh tools/profiling/bloat/bloat_diff.py \
-d origin/$ghprbTargetBranch || FAILED="true"
tools/jenkins/run_c_cpp_test.sh tools/profiling/microbenchmarks/bm_diff/bm_main.py \
-d origin/$ghprbTargetBranch \
-b $BENCHMARKS_TO_RUN || FAILED="true"

@ -0,0 +1,99 @@
#!/usr/bin/env python2.7
#
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import glob
import multiprocessing
import os
import shutil
import subprocess
import sys
sys.path.append(
os.path.join(
os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils'))
import comment_on_pr
argp = argparse.ArgumentParser(
description='Perform diff on microbenchmarks')
argp.add_argument(
'-d',
'--diff_base',
type=str,
help='Commit or branch to compare the current one to')
argp.add_argument(
'-j',
'--jobs',
type=int,
default=multiprocessing.cpu_count())
args = argp.parse_args()
LIBS = [
'libgrpc.so',
'libgrpc++.so',
]
def build(where):
subprocess.check_call('make -j%d' % args.jobs,
shell=True, cwd='.')
shutil.rmtree('bloat_diff_%s' % where, ignore_errors=True)
os.rename('libs', 'bloat_diff_%s' % where)
build('new')
if args.diff_base:
old = 'old'
where_am_i = subprocess.check_output(
['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
subprocess.check_call(['git', 'checkout', args.diff_base])
subprocess.check_call(['git', 'submodule', 'update'])
try:
try:
build('old')
except subprocess.CalledProcessError, e:
subprocess.check_call(['make', 'clean'])
build('old')
finally:
subprocess.check_call(['git', 'checkout', where_am_i])
subprocess.check_call(['git', 'submodule', 'update'])
subprocess.check_call('make -j%d' % args.jobs,
shell=True, cwd='third_party/bloaty')
text = ''
for lib in LIBS:
text += '****************************************************************\n\n'
text += lib + '\n\n'
old_version = glob.glob('bloat_diff_old/opt/%s' % lib)
new_version = glob.glob('bloat_diff_new/opt/%s' % lib)
assert len(new_version) == 1
cmd = 'third_party/bloaty/bloaty -d compileunits,symbols'
if old_version:
assert len(old_version) == 1
text += subprocess.check_output('%s %s -- %s' %
(cmd, new_version[0], old_version[0]),
shell=True)
else:
text += subprocess.check_output('%s %s' %
(cmd, new_version[0]),
shell=True)
text += '\n\n'
print text
comment_on_pr.comment_on_pr('```\n%s\n```' % text)

File diff suppressed because it is too large Load Diff

@ -364,7 +364,7 @@ class Job(object):
class Jobset(object):
"""Manages one run of jobs."""
def __init__(self, check_cancelled, maxjobs, newline_on_success, travis,
def __init__(self, check_cancelled, maxjobs, maxjobs_cpu_agnostic, newline_on_success, travis,
stop_on_failure, add_env, quiet_success, max_time):
self._running = set()
self._check_cancelled = check_cancelled
@ -372,6 +372,7 @@ class Jobset(object):
self._failures = 0
self._completed = 0
self._maxjobs = maxjobs
self._maxjobs_cpu_agnostic = maxjobs_cpu_agnostic
self._newline_on_success = newline_on_success
self._travis = travis
self._stop_on_failure = stop_on_failure
@ -406,7 +407,9 @@ class Jobset(object):
if self.cancelled(): return False
current_cpu_cost = self.cpu_cost()
if current_cpu_cost == 0: break
if current_cpu_cost + spec.cpu_cost <= self._maxjobs: break
if current_cpu_cost + spec.cpu_cost <= self._maxjobs:
if len(self._running) < self._maxjobs_cpu_agnostic:
break
self.reap()
if self.cancelled(): return False
job = Job(spec,
@ -491,6 +494,7 @@ def tag_remaining(xs):
def run(cmdlines,
check_cancelled=_never_cancelled,
maxjobs=None,
maxjobs_cpu_agnostic=None,
newline_on_success=False,
travis=False,
infinite_runs=False,
@ -509,6 +513,7 @@ def run(cmdlines,
return 0, resultset
js = Jobset(check_cancelled,
maxjobs if maxjobs is not None else _DEFAULT_MAX_JOBS,
maxjobs_cpu_agnostic if maxjobs_cpu_agnostic is not None else _DEFAULT_MAX_JOBS,
newline_on_success, travis, stop_on_failure, add_env,
quiet_success, max_time)
for cmdline, remaining in tag_remaining(cmdlines):

@ -117,6 +117,13 @@ def run_shell_command(cmd, env=None, cwd=None):
e.cmd, e.returncode, e.output)
raise
def max_parallel_tests_for_current_platform():
# Too much test parallelization has only been seen to be a problem
# so far on windows.
if jobset.platform_string() == 'windows':
return 64
return 128
# SimpleConfig: just compile with CONFIG=config, and run the binary to test
class Config(object):
@ -1618,7 +1625,7 @@ def _build_and_run(
jobset.message('START', 'Running tests quietly, only failing tests will be reported', do_newline=True)
num_test_failures, resultset = jobset.run(
all_runs, check_cancelled, newline_on_success=newline_on_success,
travis=args.travis, maxjobs=args.jobs,
travis=args.travis, maxjobs=args.jobs, maxjobs_cpu_agnostic=max_parallel_tests_for_current_platform(),
stop_on_failure=args.stop_on_failure,
quiet_success=args.quiet_success, max_time=args.max_time)
if resultset:

@ -34,6 +34,7 @@ cat << EOF | awk '{ print $1 }' | sort > $want_submodules
80a37e0782d2d702d52234b62dd4b9ec74fd2c95 third_party/protobuf (v3.4.0)
cacf7f1d4e3d44d871b605da3b647f07d718623f third_party/zlib (v1.2.11)
3be1924221e1326df520f8498d704a5c4c8d0cce third_party/cares/cares (cares-1_13_0)
73594cde8c9a52a102c4341c244c833aa61b9c06 third_party/bloaty
EOF
diff -u $submodules $want_submodules

Loading…
Cancel
Save