From 0ae9959a1c93bedeb6f8ac009cb338eeee08a8ba Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Sat, 1 Apr 2017 21:57:42 -0700 Subject: [PATCH 01/60] Revert "Revert "Add selected fullstack benchmarks to pr suite"" --- tools/jenkins/run_performance.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/jenkins/run_performance.sh b/tools/jenkins/run_performance.sh index dee033a951a..544e31dcbdb 100755 --- a/tools/jenkins/run_performance.sh +++ b/tools/jenkins/run_performance.sh @@ -32,7 +32,7 @@ set -ex # List of benchmarks that provide good signal for analyzing performance changes in pull requests -BENCHMARKS_TO_RUN="bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata" +BENCHMARKS_TO_RUN="bm_fullstack_unary_ping_pong bm_fullstack_streaming_ping_pong bm_fullstack_streaming_pump bm_closure bm_cq bm_call_create bm_error bm_chttp2_hpack bm_chttp2_transport bm_pollset bm_metadata" # Enter the gRPC repo root cd $(dirname $0)/../.. From f69317cf03cd69475e65e1e5ca2ef751947f3c12 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Sun, 2 Apr 2017 16:03:43 -0700 Subject: [PATCH 02/60] Increase reps to decrease noise --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 2337f2b282e..5350b5739f3 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -101,7 +101,7 @@ argp.add_argument('-t', '--track', help='Which metrics to track') argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) -argp.add_argument('-r', '--repetitions', type=int, default=4) +argp.add_argument('-r', '--repetitions', type=int, default=7) argp.add_argument('-p', '--p_threshold', type=float, default=0.01) args = argp.parse_args() From 691eefbe0ff634fe49aacf06efccdd7e84864dfc Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 3 Apr 2017 08:06:32 -0700 Subject: [PATCH 03/60] Parallel bm_diff --- tools/profiling/microbenchmarks/bm_diff.py | 33 +++++++++++----------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 5350b5739f3..1602e83fcab 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +args.benchmarks#!/usr/bin/env python2.7 # Copyright 2017, Google Inc. # All rights reserved. # @@ -41,6 +41,7 @@ import pipes import os sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils')) import comment_on_pr +import jobset def changed_ratio(n, o): if float(o) <= .0001: o = 0 @@ -101,8 +102,9 @@ argp.add_argument('-t', '--track', help='Which metrics to track') argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) -argp.add_argument('-r', '--repetitions', type=int, default=7) +argp.add_argument('-r', '--repetitions', type=int, default=30) argp.add_argument('-p', '--p_threshold', type=float, default=0.01) +argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count()) args = argp.parse_args() assert args.diff_base @@ -117,7 +119,7 @@ def avg(lst): def make_cmd(cfg): return ['make'] + args.benchmarks + [ - 'CONFIG=%s' % cfg, '-j', '%d' % multiprocessing.cpu_count()] + 'CONFIG=%s' % cfg, '-j', '%d' % args.jobs] def build(): subprocess.check_call(['git', 'submodule', 'update']) @@ -135,27 +137,24 @@ def collect1(bm, cfg, ver): '--benchmark_out_format=json', '--benchmark_repetitions=%d' % (args.repetitions) ] - print cmd - subprocess.check_call(cmd) + return jobset.JobSpec(cmd, shortname='%s %s %s' % (bm, cfg, ver), + verbose_success=True) build() -for bm in args.benchmarks: - collect1(bm, 'opt', 'new') - collect1(bm, 'counters', 'new') +jobset.run(itertools.chain( + (collect1(bm, 'opt', 'new') for bm in args.benchmarks), + (collect1(bm, 'counters', 'new') for bm in args.benchmarks), +), maxjobs=args.jobs) where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() subprocess.check_call(['git', 'checkout', args.diff_base]) try: build() - comparables = [] - for bm in args.benchmarks: - try: - collect1(bm, 'opt', 'old') - collect1(bm, 'counters', 'old') - comparables.append(bm) - except subprocess.CalledProcessError, e: - pass + jobset.run(itertools.chain( + (collect1(bm, 'opt', 'new') for bm in args.benchmarks), + (collect1(bm, 'counters', 'new') for bm in args.benchmarks), + ), maxjobs=args.jobs) finally: subprocess.check_call(['git', 'checkout', where_am_i]) subprocess.check_call(['git', 'submodule', 'update']) @@ -201,7 +200,7 @@ class Benchmark: benchmarks = collections.defaultdict(Benchmark) -for bm in comparables: +for bm in args.benchmarks: with open('%s.counters.new.json' % bm) as f: js_new_ctr = json.loads(f.read()) with open('%s.opt.new.json' % bm) as f: From cc89f7801bd1c6822e1805525073a69cf070320c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 3 Apr 2017 08:08:25 -0700 Subject: [PATCH 04/60] Fix typo --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 1602e83fcab..82fbe084977 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -1,4 +1,4 @@ -args.benchmarks#!/usr/bin/env python2.7 +#!/usr/bin/env python2.7 # Copyright 2017, Google Inc. # All rights reserved. # From 2fbbcd1fd49203347b668bc2b896d8cbaab158fd Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 3 Apr 2017 08:29:08 -0700 Subject: [PATCH 05/60] Missing import --- tools/profiling/microbenchmarks/bm_diff.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 82fbe084977..84712e7e0dc 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -42,6 +42,7 @@ import os sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'run_tests', 'python_utils')) import comment_on_pr import jobset +import itertools def changed_ratio(n, o): if float(o) <= .0001: o = 0 From 942324235cc03dd3437df8c282796f5711bfa131 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 3 Apr 2017 10:44:03 -0700 Subject: [PATCH 06/60] s/new/old --- tools/profiling/microbenchmarks/bm_diff.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 84712e7e0dc..aa4caca43d7 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -153,8 +153,8 @@ subprocess.check_call(['git', 'checkout', args.diff_base]) try: build() jobset.run(itertools.chain( - (collect1(bm, 'opt', 'new') for bm in args.benchmarks), - (collect1(bm, 'counters', 'new') for bm in args.benchmarks), + (collect1(bm, 'opt', 'old') for bm in args.benchmarks), + (collect1(bm, 'counters', 'old') for bm in args.benchmarks), ), maxjobs=args.jobs) finally: subprocess.check_call(['git', 'checkout', where_am_i]) From 7e313858636917ceba4b37c673b8676afed75544 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 3 Apr 2017 15:14:08 -0700 Subject: [PATCH 07/60] Remove timeout --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index aa4caca43d7..f6a944babc8 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -139,7 +139,7 @@ def collect1(bm, cfg, ver): '--benchmark_repetitions=%d' % (args.repetitions) ] return jobset.JobSpec(cmd, shortname='%s %s %s' % (bm, cfg, ver), - verbose_success=True) + verbose_success=True, timeout_seconds=None) build() jobset.run(itertools.chain( From 282c1399855c4e65a449b5f535423f17904bbf70 Mon Sep 17 00:00:00 2001 From: Igor Pylypiv Date: Tue, 4 Apr 2017 19:44:06 -0700 Subject: [PATCH 08/60] Refactor pchar_parse() to use switch() Add previously missed '@' and ':' characters --- .../ext/filters/client_channel/uri_parser.c | 54 ++++++++++++------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/src/core/ext/filters/client_channel/uri_parser.c b/src/core/ext/filters/client_channel/uri_parser.c index 01b99911aa4..f28db59e270 100644 --- a/src/core/ext/filters/client_channel/uri_parser.c +++ b/src/core/ext/filters/client_channel/uri_parser.c @@ -83,6 +83,11 @@ static char *decode_and_copy_component(grpc_exec_ctx *exec_ctx, const char *src, return out; } +static bool valid_hex(char c) { + return ((c >= 'a') && (c <= 'f')) || ((c >= 'A') && (c <= 'F')) || + ((c >= '0') && (c <= '9')); +} + /** Returns how many chars to advance if \a uri_text[i] begins a valid \a pchar * production. If \a uri_text[i] introduces an invalid \a pchar (such as percent * sign not followed by two hex digits), NOT_SET is returned. */ @@ -93,27 +98,36 @@ static size_t parse_pchar(const char *uri_text, size_t i) { * sub-delims = "!" / "$" / "&" / "'" / "(" / ")" / "*" / "+" / "," / ";" / "=" */ char c = uri_text[i]; - if (((c >= 'A') && (c <= 'Z')) || ((c >= 'a') && (c <= 'z')) || - ((c >= '0') && (c <= '9')) || - (c == '-' || c == '.' || c == '_' || c == '~') || /* unreserved */ - (c == '!' || c == '$' || c == '&' || c == '\'' || c == '$' || c == '&' || - c == '(' || c == ')' || c == '*' || c == '+' || c == ',' || c == ';' || - c == '=') /* sub-delims */) { - return 1; - } - if (c == '%') { /* pct-encoded */ - size_t j; - if (uri_text[i + 1] == 0 || uri_text[i + 2] == 0) { - return NOT_SET; - } - for (j = i + 1; j < 2; j++) { - c = uri_text[j]; - if (!(((c >= '0') && (c <= '9')) || ((c >= 'a') && (c <= 'f')) || - ((c >= 'A') && (c <= 'F')))) { - return NOT_SET; + switch (c) { + default: + if (((c >= 'a') && (c <= 'z')) || ((c >= 'A') && (c <= 'Z')) || + ((c >= '0') && (c <= '9'))) { + return 1; } - } - return 2; + break; + case ':': + case '@': + case '-': + case '.': + case '_': + case '~': + case '!': + case '$': + case '&': + case '\'': + case '(': + case ')': + case '*': + case '+': + case ',': + case ';': + case '=': + return 1; + case '%': /* pct-encoded */ + if (valid_hex(uri_text[i + 1]) && valid_hex(uri_text[i + 2])) { + return 2; + } + return NOT_SET; } return 0; } From a3589c5411186f0a29b661003eaa641d1a63af37 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 4 Apr 2017 21:09:08 -0700 Subject: [PATCH 09/60] Starting speedup calculation --- tools/profiling/microbenchmarks/speedup.py | 59 ++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 tools/profiling/microbenchmarks/speedup.py diff --git a/tools/profiling/microbenchmarks/speedup.py b/tools/profiling/microbenchmarks/speedup.py new file mode 100644 index 00000000000..09222580ef9 --- /dev/null +++ b/tools/profiling/microbenchmarks/speedup.py @@ -0,0 +1,59 @@ +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from scipy import stats + +new=[66034560.0, 126765693.0, 99074674.0, 98588433.0, 96731372.0, 110179725.0, 103802110.0, 101139800.0, 102357205.0, 99016353.0, 98840824.0, 99585632.0, 98791720.0, 96171521.0, 95327098.0, 95629704.0, 98209772.0, 99779411.0, 100182488.0, 98354192.0, 99644781.0, 98546709.0, 99019176.0, 99543014.0, 99077269.0, 98046601.0, 99319039.0, 98542572.0, 98886614.0, 72560968.0] +old=[60423464.0, 71249570.0, 73213089.0, 73200055.0, 72911768.0, 72347798.0, 72494672.0, 72756976.0, 72116565.0, 71541342.0, 73442538.0, 74817383.0, 73007780.0, 72499062.0, 72404945.0, 71843504.0, 73245405.0, 72778304.0, 74004519.0, 73694464.0, 72919931.0, 72955481.0, 71583857.0, 71350467.0, 71836817.0, 70064115.0, 70355345.0, 72516202.0, 71716777.0, 71532266.0] + +_THRESHOLD = 0.01 + +def scale(a, mul): + return [x*mul for x in a] + +def cmp(a, b): + return stats.mannwhitneyu(a, b, True, 'two-sided') + +def speedup(new, old): + s0, p0 = cmp(new, old) + print s0, p0 + if s0 == 0: return 0 + if p0 >= _THRESHOLD: return 0 + if s0 < 0: + pass + else: + pct = 1 + while True: + sp, pp = cmp(new, scale(old, 1 + pct / 100.0)) + if sp <= 0: break + if pp >= _THRESHOLD: break + pct += 1 + return pct - 1 + +print speedup(new, old) From 125b04fcedd572fc171abe39cf181bafc8d08361 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 4 Apr 2017 21:31:44 -0700 Subject: [PATCH 10/60] adding speedup --- tools/profiling/microbenchmarks/bm_diff.py | 16 ++++--------- tools/profiling/microbenchmarks/speedup.py | 28 +++++++++++++--------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index f6a944babc8..5c8b93d471c 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -43,6 +43,7 @@ sys.path.append(os.path.join(os.path.dirname(sys.argv[0]), '..', '..', 'run_test import comment_on_pr import jobset import itertools +import speedup def changed_ratio(n, o): if float(o) <= .0001: o = 0 @@ -180,16 +181,9 @@ class Benchmark: new = self.samples[True][f] old = self.samples[False][f] if not new or not old: continue - p = stats.ttest_ind(new, old)[1] - new_mdn = median(new) - old_mdn = median(old) - delta = new_mdn - old_mdn - ratio = changed_ratio(new_mdn, old_mdn) - print '%s: new=%r old=%r new_mdn=%f old_mdn=%f delta=%f(%f:%f) ratio=%f(%f:%f) p=%f' % ( - f, new, old, new_mdn, old_mdn, delta, abs(delta), _INTERESTING[f]['abs_diff'], ratio, abs(ratio), _INTERESTING[f]['pct_diff']/100.0, p - ) - if p < args.p_threshold and abs(delta) > _INTERESTING[f]['abs_diff'] and abs(ratio) > _INTERESTING[f]['pct_diff']/100.0: - self.final[f] = delta + s = speedup.speedup(new, old) + if s: + self.final[f] = '%d%%' % s return self.final.keys() def skip(self): @@ -226,7 +220,7 @@ really_interesting = set() for name, bm in benchmarks.items(): print name really_interesting.update(bm.process()) -fields = [f for f in args.track if f in args.track] +fields = [f for f in args.track if f in really_interesting] headers = ['Benchmark'] + fields rows = [] diff --git a/tools/profiling/microbenchmarks/speedup.py b/tools/profiling/microbenchmarks/speedup.py index 09222580ef9..9b022da0c46 100644 --- a/tools/profiling/microbenchmarks/speedup.py +++ b/tools/profiling/microbenchmarks/speedup.py @@ -29,31 +29,37 @@ from scipy import stats -new=[66034560.0, 126765693.0, 99074674.0, 98588433.0, 96731372.0, 110179725.0, 103802110.0, 101139800.0, 102357205.0, 99016353.0, 98840824.0, 99585632.0, 98791720.0, 96171521.0, 95327098.0, 95629704.0, 98209772.0, 99779411.0, 100182488.0, 98354192.0, 99644781.0, 98546709.0, 99019176.0, 99543014.0, 99077269.0, 98046601.0, 99319039.0, 98542572.0, 98886614.0, 72560968.0] -old=[60423464.0, 71249570.0, 73213089.0, 73200055.0, 72911768.0, 72347798.0, 72494672.0, 72756976.0, 72116565.0, 71541342.0, 73442538.0, 74817383.0, 73007780.0, 72499062.0, 72404945.0, 71843504.0, 73245405.0, 72778304.0, 74004519.0, 73694464.0, 72919931.0, 72955481.0, 71583857.0, 71350467.0, 71836817.0, 70064115.0, 70355345.0, 72516202.0, 71716777.0, 71532266.0] - _THRESHOLD = 0.01 def scale(a, mul): return [x*mul for x in a] def cmp(a, b): - return stats.mannwhitneyu(a, b, True, 'two-sided') + return stats.ttest_ind(a, b) def speedup(new, old): s0, p0 = cmp(new, old) - print s0, p0 if s0 == 0: return 0 - if p0 >= _THRESHOLD: return 0 + if p0 > _THRESHOLD: return 0 if s0 < 0: - pass + pct = 1 + while True: + sp, pp = cmp(new, scale(old, 1 - pct/100.0)) + if sp > 0: break + if pp > _THRESHOLD: break + pct += 1 + return -(pct - 1) else: pct = 1 while True: - sp, pp = cmp(new, scale(old, 1 + pct / 100.0)) - if sp <= 0: break - if pp >= _THRESHOLD: break + sp, pp = cmp(new, scale(old, 1 + pct/100.0)) + if sp < 0: break + if pp > _THRESHOLD: break pct += 1 return pct - 1 -print speedup(new, old) +if __name__ == "__main__": + new=[66034560.0, 126765693.0, 99074674.0, 98588433.0, 96731372.0, 110179725.0, 103802110.0, 101139800.0, 102357205.0, 99016353.0, 98840824.0, 99585632.0, 98791720.0, 96171521.0, 95327098.0, 95629704.0, 98209772.0, 99779411.0, 100182488.0, 98354192.0, 99644781.0, 98546709.0, 99019176.0, 99543014.0, 99077269.0, 98046601.0, 99319039.0, 98542572.0, 98886614.0, 72560968.0] + old=[60423464.0, 71249570.0, 73213089.0, 73200055.0, 72911768.0, 72347798.0, 72494672.0, 72756976.0, 72116565.0, 71541342.0, 73442538.0, 74817383.0, 73007780.0, 72499062.0, 72404945.0, 71843504.0, 73245405.0, 72778304.0, 74004519.0, 73694464.0, 72919931.0, 72955481.0, 71583857.0, 71350467.0, 71836817.0, 70064115.0, 70355345.0, 72516202.0, 71716777.0, 71532266.0] + print speedup(new, old) + print speedup(old, new) From e749e34f07be72b9f804039d1210b4ca5b045026 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 5 Apr 2017 06:32:47 -0700 Subject: [PATCH 11/60] Provide bounds on speedup --- tools/profiling/microbenchmarks/speedup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/profiling/microbenchmarks/speedup.py b/tools/profiling/microbenchmarks/speedup.py index 9b022da0c46..ac268fb6790 100644 --- a/tools/profiling/microbenchmarks/speedup.py +++ b/tools/profiling/microbenchmarks/speedup.py @@ -43,7 +43,7 @@ def speedup(new, old): if p0 > _THRESHOLD: return 0 if s0 < 0: pct = 1 - while True: + while pct < 101: sp, pp = cmp(new, scale(old, 1 - pct/100.0)) if sp > 0: break if pp > _THRESHOLD: break @@ -51,7 +51,7 @@ def speedup(new, old): return -(pct - 1) else: pct = 1 - while True: + while pct < 101: sp, pp = cmp(new, scale(old, 1 + pct/100.0)) if sp < 0: break if pp > _THRESHOLD: break From d6d2da11492f52a4170be34c826f6adb8ca837d1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 5 Apr 2017 06:34:07 -0700 Subject: [PATCH 12/60] Handle nans --- tools/profiling/microbenchmarks/speedup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/profiling/microbenchmarks/speedup.py b/tools/profiling/microbenchmarks/speedup.py index ac268fb6790..a06f2a91e1d 100644 --- a/tools/profiling/microbenchmarks/speedup.py +++ b/tools/profiling/microbenchmarks/speedup.py @@ -28,6 +28,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from scipy import stats +import math _THRESHOLD = 0.01 @@ -39,6 +40,7 @@ def cmp(a, b): def speedup(new, old): s0, p0 = cmp(new, old) + if math.isnan(p0) return 0 if s0 == 0: return 0 if p0 > _THRESHOLD: return 0 if s0 < 0: From b1d3dcdb7db540b7dfa2efad269cd56738ff7e79 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 5 Apr 2017 06:35:49 -0700 Subject: [PATCH 13/60] fix typo --- tools/profiling/microbenchmarks/speedup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/speedup.py b/tools/profiling/microbenchmarks/speedup.py index a06f2a91e1d..ff114539604 100644 --- a/tools/profiling/microbenchmarks/speedup.py +++ b/tools/profiling/microbenchmarks/speedup.py @@ -40,7 +40,7 @@ def cmp(a, b): def speedup(new, old): s0, p0 = cmp(new, old) - if math.isnan(p0) return 0 + if math.isnan(p0): return 0 if s0 == 0: return 0 if p0 > _THRESHOLD: return 0 if s0 < 0: From 5c708f4aeee99261ed7851304f4bc74c46a91831 Mon Sep 17 00:00:00 2001 From: Dan Zhang Date: Thu, 30 Mar 2017 17:44:26 -0400 Subject: [PATCH 14/60] modify udp_server --- src/core/lib/iomgr/udp_server.c | 38 +++++++++++++++++++++++-------- src/core/lib/iomgr/udp_server.h | 4 +++- test/core/iomgr/udp_server_test.c | 14 ++++++------ 3 files changed, 39 insertions(+), 17 deletions(-) diff --git a/src/core/lib/iomgr/udp_server.c b/src/core/lib/iomgr/udp_server.c index 60579e18ba5..ca283d034f9 100644 --- a/src/core/lib/iomgr/udp_server.c +++ b/src/core/lib/iomgr/udp_server.c @@ -79,10 +79,15 @@ struct grpc_udp_listener { grpc_resolved_address addr; grpc_closure read_closure; grpc_closure write_closure; + // To be called when corresponding QuicGrpcServer closes all active + // connections. + grpc_closure orphan_fd_closure; grpc_closure destroyed_closure; grpc_udp_server_read_cb read_cb; grpc_udp_server_write_cb write_cb; grpc_udp_server_orphan_cb orphan_cb; + // True if orphan_cb is trigered. + bool orphan_notified; struct grpc_udp_listener *next; }; @@ -146,6 +151,14 @@ grpc_udp_server *grpc_udp_server_create(const grpc_channel_args *args) { return s; } +static void shutdown_fd(grpc_exec_ctx *exec_ctx, void *fd, grpc_error *error) { + grpc_fd_shutdown(exec_ctx, (grpc_fd *)fd, GRPC_ERROR_REF(error)); +} + +static void dummy_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { + // No-op. +} + static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_udp_server *s) { if (s->shutdown_complete != NULL) { grpc_closure_sched(exec_ctx, s->shutdown_complete, GRPC_ERROR_NONE); @@ -195,12 +208,16 @@ static void deactivated_all_ports(grpc_exec_ctx *exec_ctx, grpc_udp_server *s) { grpc_closure_init(&sp->destroyed_closure, destroyed_port, s, grpc_schedule_on_exec_ctx); - - /* Call the orphan_cb to signal that the FD is about to be closed and - * should no longer be used. */ - GPR_ASSERT(sp->orphan_cb); - sp->orphan_cb(exec_ctx, sp->emfd, sp->server->user_data); - + if (!sp->orphan_notified) { + /* Call the orphan_cb to signal that the FD is about to be closed and + * should no longer be used. Because at this point, all listening ports + * have been shutdown already, no need to shutdown again.*/ + grpc_closure_init(&sp->orphan_fd_closure, dummy_cb, sp->emfd, + grpc_schedule_on_exec_ctx); + GPR_ASSERT(sp->orphan_cb); + sp->orphan_cb(exec_ctx, sp->emfd, &sp->orphan_fd_closure, + sp->server->user_data); + } grpc_fd_orphan(exec_ctx, sp->emfd, &sp->destroyed_closure, NULL, "udp_listener_shutdown"); } @@ -225,9 +242,11 @@ void grpc_udp_server_destroy(grpc_exec_ctx *exec_ctx, grpc_udp_server *s, if (s->active_ports) { for (sp = s->head; sp; sp = sp->next) { GPR_ASSERT(sp->orphan_cb); - sp->orphan_cb(exec_ctx, sp->emfd, sp->server->user_data); - grpc_fd_shutdown(exec_ctx, sp->emfd, GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Server destroyed")); + grpc_closure_init(&sp->orphan_fd_closure, shutdown_fd, sp->emfd, + grpc_schedule_on_exec_ctx); + sp->orphan_cb(exec_ctx, sp->emfd, &sp->orphan_fd_closure, + sp->server->user_data); + sp->orphan_notified = true; } gpr_mu_unlock(&s->mu); } else { @@ -391,6 +410,7 @@ static int add_socket_to_server(grpc_udp_server *s, int fd, sp->read_cb = read_cb; sp->write_cb = write_cb; sp->orphan_cb = orphan_cb; + sp->orphan_notified = false; GPR_ASSERT(sp->emfd); gpr_mu_unlock(&s->mu); gpr_free(name); diff --git a/src/core/lib/iomgr/udp_server.h b/src/core/lib/iomgr/udp_server.h index 9df3fe4d1f0..2a72638709b 100644 --- a/src/core/lib/iomgr/udp_server.h +++ b/src/core/lib/iomgr/udp_server.h @@ -55,7 +55,9 @@ typedef void (*grpc_udp_server_write_cb)(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, /* Called when the grpc_fd is about to be orphaned (and the FD closed). */ typedef void (*grpc_udp_server_orphan_cb)(grpc_exec_ctx *exec_ctx, - grpc_fd *emfd, void *user_data); + grpc_fd *emfd, + grpc_closure* shutdown_fd_callback, + void *user_data); /* Create a server, initially not bound to any ports */ grpc_udp_server *grpc_udp_server_create(const grpc_channel_args *args); diff --git a/test/core/iomgr/udp_server_test.c b/test/core/iomgr/udp_server_test.c index 12d84063239..9cc0ea94983 100644 --- a/test/core/iomgr/udp_server_test.c +++ b/test/core/iomgr/udp_server_test.c @@ -91,7 +91,7 @@ static void on_write(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, void *user_data) { } static void on_fd_orphaned(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, - void *user_data) { + grpc_closure* closure, void *user_data) { gpr_log(GPR_INFO, "gRPC FD about to be orphaned: %d", grpc_fd_wrapped_fd(emfd)); g_number_of_orphan_calls++; @@ -228,9 +228,9 @@ static void test_no_op_with_port_and_start(void) { grpc_udp_server_destroy(&exec_ctx, s, NULL); grpc_exec_ctx_finish(&exec_ctx); - /* The server had a single FD, which is orphaned once in * - * deactivated_all_ports, and once in grpc_udp_server_destroy. */ - GPR_ASSERT(g_number_of_orphan_calls == 2); + /* The server had a single FD, which is orphaned exactly once in * + * grpc_udp_server_destroy. */ + GPR_ASSERT(g_number_of_orphan_calls == 1); } static void test_receive(int number_of_clients) { @@ -297,9 +297,9 @@ static void test_receive(int number_of_clients) { grpc_udp_server_destroy(&exec_ctx, s, NULL); grpc_exec_ctx_finish(&exec_ctx); - /* The server had a single FD, which is orphaned once in * - * deactivated_all_ports, and once in grpc_udp_server_destroy. */ - GPR_ASSERT(g_number_of_orphan_calls == 2); + /* The server had a single FD, which is orphaned exactly once in * + * grpc_udp_server_destroy. */ + GPR_ASSERT(g_number_of_orphan_calls == 1); /* The write callback should have fired a few times. */ GPR_ASSERT(g_number_of_writes > 0); From e5f18281755cbacf51613124c3f762ad0129b9d1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 5 Apr 2017 09:08:37 -0700 Subject: [PATCH 15/60] Tweak parameters --- tools/profiling/microbenchmarks/bm_diff.py | 3 ++- tools/profiling/microbenchmarks/speedup.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 5c8b93d471c..37d500d114f 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -181,9 +181,10 @@ class Benchmark: new = self.samples[True][f] old = self.samples[False][f] if not new or not old: continue + print '%s: new=%r old=%r' % (f, new, old) s = speedup.speedup(new, old) if s: - self.final[f] = '%d%%' % s + self.final[f] = '%+d%%' % s return self.final.keys() def skip(self): diff --git a/tools/profiling/microbenchmarks/speedup.py b/tools/profiling/microbenchmarks/speedup.py index ff114539604..35d392a57db 100644 --- a/tools/profiling/microbenchmarks/speedup.py +++ b/tools/profiling/microbenchmarks/speedup.py @@ -30,7 +30,7 @@ from scipy import stats import math -_THRESHOLD = 0.01 +_THRESHOLD = 0.001 def scale(a, mul): return [x*mul for x in a] From 37f80e8a8a86612f44fd8e6cd3b15d73bc737404 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 5 Apr 2017 12:18:30 -0700 Subject: [PATCH 16/60] Fix reuse bug? --- tools/profiling/microbenchmarks/bm_json.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/profiling/microbenchmarks/bm_json.py b/tools/profiling/microbenchmarks/bm_json.py index e885444f410..917269823d0 100644 --- a/tools/profiling/microbenchmarks/bm_json.py +++ b/tools/profiling/microbenchmarks/bm_json.py @@ -203,4 +203,5 @@ def expand_json(js, js2 = None): row['real_time'] = bm2['real_time'] row['iterations'] = bm2['iterations'] bm2['already_used'] = True + break yield row From 32873d171be6f874cd81312f6c951c80b975edc4 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 5 Apr 2017 12:19:51 -0700 Subject: [PATCH 17/60] accelerate --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 37d500d114f..09018594794 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -104,7 +104,7 @@ argp.add_argument('-t', '--track', help='Which metrics to track') argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) -argp.add_argument('-r', '--repetitions', type=int, default=30) +argp.add_argument('-r', '--repetitions', type=int, default=10) argp.add_argument('-p', '--p_threshold', type=float, default=0.01) argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count()) args = argp.parse_args() From b1c02cac82b9ec7988d60d53c3c2af8884c57f1a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 5 Apr 2017 14:11:28 -0700 Subject: [PATCH 18/60] exclude small absolute changes --- tools/profiling/microbenchmarks/bm_diff.py | 25 +++------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 09018594794..3223cc1004d 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -63,26 +63,6 @@ def median(ary): def min_change(pct): return lambda n, o: abs(changed_ratio(n,o)) > pct/100.0 -nanos = { - 'abs_diff': 5, - 'pct_diff': 10, -} -counter = { - 'abs_diff': 0.5, - 'pct_diff': 10, -} - -_INTERESTING = { - 'cpu_time': nanos, - 'real_time': nanos, - 'locks_per_iteration': counter, - 'allocs_per_iteration': counter, - 'writes_per_iteration': counter, - 'atm_cas_per_iteration': counter, - 'atm_add_per_iteration': counter, -} - - _AVAILABLE_BENCHMARK_TESTS = ['bm_fullstack_unary_ping_pong', 'bm_fullstack_streaming_ping_pong', 'bm_fullstack_streaming_pump', @@ -181,9 +161,10 @@ class Benchmark: new = self.samples[True][f] old = self.samples[False][f] if not new or not old: continue - print '%s: new=%r old=%r' % (f, new, old) + mdn_diff = abs(median(new) - median(old)) + print '%s: new=%r old=%r mdn_diff=%r' % (f, new, old, mdn_diff) s = speedup.speedup(new, old) - if s: + if s and mdn_diff > 0.5: self.final[f] = '%+d%%' % s return self.final.keys() From bdeb857ab4e2233bbb796378f7212eb52cf71bb3 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 5 Apr 2017 14:12:53 -0700 Subject: [PATCH 19/60] more samples --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 3223cc1004d..6c63f20b7df 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -84,7 +84,7 @@ argp.add_argument('-t', '--track', help='Which metrics to track') argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) -argp.add_argument('-r', '--repetitions', type=int, default=10) +argp.add_argument('-r', '--repetitions', type=int, default=30) argp.add_argument('-p', '--p_threshold', type=float, default=0.01) argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count()) args = argp.parse_args() From 7281d19de374852abf38521a20ef96b88e2916e9 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 5 Apr 2017 15:04:55 -0700 Subject: [PATCH 20/60] fix --- tools/profiling/microbenchmarks/bm_diff.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 6c63f20b7df..e4ef8df91c7 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -45,6 +45,16 @@ import jobset import itertools import speedup +_INTERESTING = ( + 'cpu_time', + 'real_time', + 'locks_per_iteration', + 'allocs_per_iteration', + 'writes_per_iteration', + 'atm_cas_per_iteration', + 'atm_add_per_iteration', +) + def changed_ratio(n, o): if float(o) <= .0001: o = 0 if float(n) <= .0001: n = 0 @@ -78,9 +88,9 @@ _AVAILABLE_BENCHMARK_TESTS = ['bm_fullstack_unary_ping_pong', argp = argparse.ArgumentParser(description='Perform diff on microbenchmarks') argp.add_argument('-t', '--track', - choices=sorted(_INTERESTING.keys()), + choices=sorted(_INTERESTING), nargs='+', - default=sorted(_INTERESTING.keys()), + default=sorted(_INTERESTING), help='Which metrics to track') argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) From 2d38b05671ca0c6006c775ccb8cbb945118a794c Mon Sep 17 00:00:00 2001 From: Dan Zhang Date: Thu, 6 Apr 2017 13:09:34 -0400 Subject: [PATCH 21/60] clang format --- src/core/lib/iomgr/udp_server.h | 2 +- test/core/iomgr/udp_server_test.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/lib/iomgr/udp_server.h b/src/core/lib/iomgr/udp_server.h index 2a72638709b..8006037644d 100644 --- a/src/core/lib/iomgr/udp_server.h +++ b/src/core/lib/iomgr/udp_server.h @@ -56,7 +56,7 @@ typedef void (*grpc_udp_server_write_cb)(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, /* Called when the grpc_fd is about to be orphaned (and the FD closed). */ typedef void (*grpc_udp_server_orphan_cb)(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, - grpc_closure* shutdown_fd_callback, + grpc_closure *shutdown_fd_callback, void *user_data); /* Create a server, initially not bound to any ports */ diff --git a/test/core/iomgr/udp_server_test.c b/test/core/iomgr/udp_server_test.c index 9cc0ea94983..1f1696a7a7a 100644 --- a/test/core/iomgr/udp_server_test.c +++ b/test/core/iomgr/udp_server_test.c @@ -91,7 +91,7 @@ static void on_write(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, void *user_data) { } static void on_fd_orphaned(grpc_exec_ctx *exec_ctx, grpc_fd *emfd, - grpc_closure* closure, void *user_data) { + grpc_closure *closure, void *user_data) { gpr_log(GPR_INFO, "gRPC FD about to be orphaned: %d", grpc_fd_wrapped_fd(emfd)); g_number_of_orphan_calls++; From f6a4d9fc271f0b12df3878ae1bcecd5cb438a117 Mon Sep 17 00:00:00 2001 From: Edmundo Rodrigues Date: Mon, 20 Mar 2017 13:35:31 -0300 Subject: [PATCH 22/60] Make min_backoff_ms timeout configurable --- include/grpc/impl/codegen/grpc_types.h | 2 ++ src/core/ext/filters/client_channel/subchannel.c | 10 ++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index fdedf0a84f3..0b0ef71caf1 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -237,6 +237,8 @@ typedef struct { /** Secondary user agent: goes at the end of the user-agent metadata sent on each request. A string. */ #define GRPC_ARG_SECONDARY_USER_AGENT_STRING "grpc.secondary_user_agent" +/** The minimum time between subsequent connection attempts, in ms */ +#define GRPC_ARG_MIN_RECONNECT_BACKOFF_MS "grpc.min_reconnect_backoff_ms" /** The maximum time between subsequent connection attempts, in ms */ #define GRPC_ARG_MAX_RECONNECT_BACKOFF_MS "grpc.max_reconnect_backoff_ms" /** The time between the first and second connection attempts, in ms */ diff --git a/src/core/ext/filters/client_channel/subchannel.c b/src/core/ext/filters/client_channel/subchannel.c index 9a7a7a0ee5f..0fc73eb116c 100644 --- a/src/core/ext/filters/client_channel/subchannel.c +++ b/src/core/ext/filters/client_channel/subchannel.c @@ -59,9 +59,9 @@ #define INTERNAL_REF_BITS 16 #define STRONG_REF_MASK (~(gpr_atm)((1 << INTERNAL_REF_BITS) - 1)) -#define GRPC_SUBCHANNEL_MIN_CONNECT_TIMEOUT_SECONDS 20 #define GRPC_SUBCHANNEL_INITIAL_CONNECT_BACKOFF_SECONDS 1 #define GRPC_SUBCHANNEL_RECONNECT_BACKOFF_MULTIPLIER 1.6 +#define GRPC_SUBCHANNEL_RECONNECT_MIN_BACKOFF_SECONDS 20 #define GRPC_SUBCHANNEL_RECONNECT_MAX_BACKOFF_SECONDS 120 #define GRPC_SUBCHANNEL_RECONNECT_JITTER 0.2 @@ -353,8 +353,8 @@ grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx, "subchannel"); int initial_backoff_ms = GRPC_SUBCHANNEL_INITIAL_CONNECT_BACKOFF_SECONDS * 1000; + int min_backoff_ms = GRPC_SUBCHANNEL_RECONNECT_MIN_BACKOFF_SECONDS * 1000; int max_backoff_ms = GRPC_SUBCHANNEL_RECONNECT_MAX_BACKOFF_SECONDS * 1000; - int min_backoff_ms = GRPC_SUBCHANNEL_MIN_CONNECT_TIMEOUT_SECONDS * 1000; bool fixed_reconnect_backoff = false; if (c->args) { for (size_t i = 0; i < c->args->num_args; i++) { @@ -365,6 +365,12 @@ grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx, grpc_channel_arg_get_integer( &c->args->args[i], (grpc_integer_options){initial_backoff_ms, 100, INT_MAX}); + } else if (0 == strcmp(c->args->args[i].key, + GRPC_ARG_MIN_RECONNECT_BACKOFF_MS)) { + fixed_reconnect_backoff = false; + min_backoff_ms = grpc_channel_arg_get_integer( + &c->args->args[i], + (grpc_integer_options){min_backoff_ms, 100, INT_MAX}); } else if (0 == strcmp(c->args->args[i].key, GRPC_ARG_MAX_RECONNECT_BACKOFF_MS)) { fixed_reconnect_backoff = false; From 790e38a3a5be11f9b2a725d5382d4525a1e07897 Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Mon, 10 Apr 2017 11:00:37 -0700 Subject: [PATCH 23/60] Run start_port_server.py in performance profiling --- tools/jenkins/run_performance_profile_hourly.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/jenkins/run_performance_profile_hourly.sh b/tools/jenkins/run_performance_profile_hourly.sh index dfcc2bb1163..c352a9f4acf 100755 --- a/tools/jenkins/run_performance_profile_hourly.sh +++ b/tools/jenkins/run_performance_profile_hourly.sh @@ -32,6 +32,8 @@ set -ex cd $(dirname $0)/../.. +./tools/run_tests/start_port_server.py || true + CPUS=`python -c 'import multiprocessing; print multiprocessing.cpu_count()'` make CONFIG=opt memory_profile_test memory_profile_client memory_profile_server -j $CPUS From f597c7ccdd1b808e131862094b3ef175202f088c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 11 Apr 2017 14:39:32 -0700 Subject: [PATCH 24/60] Tighter threshold --- tools/profiling/microbenchmarks/speedup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/speedup.py b/tools/profiling/microbenchmarks/speedup.py index 35d392a57db..8f9023d2c89 100644 --- a/tools/profiling/microbenchmarks/speedup.py +++ b/tools/profiling/microbenchmarks/speedup.py @@ -30,7 +30,7 @@ from scipy import stats import math -_THRESHOLD = 0.001 +_THRESHOLD = 0.0001 def scale(a, mul): return [x*mul for x in a] From 2d7d5be813a53fc32046f338709b8c32d6bdd1fe Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 12 Apr 2017 06:54:40 -0700 Subject: [PATCH 25/60] Increase threshold --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index e4ef8df91c7..c3738605194 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -174,7 +174,7 @@ class Benchmark: mdn_diff = abs(median(new) - median(old)) print '%s: new=%r old=%r mdn_diff=%r' % (f, new, old, mdn_diff) s = speedup.speedup(new, old) - if s and mdn_diff > 0.5: + if s and mdn_diff > 0.5 and abs(s) >= 10: self.final[f] = '%+d%%' % s return self.final.keys() From b4e97270cc13849bb5a4585c88bc6385872d643b Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Wed, 12 Apr 2017 12:16:28 -0700 Subject: [PATCH 26/60] replaced hardcoded server path with cmdline arg The host and port of the remote server can be specified via command line argument to xcodebuild. --- src/objective-c/tests/InteropTestsLocalCleartext.m | 6 +++++- src/objective-c/tests/InteropTestsLocalSSL.m | 7 +++++-- src/objective-c/tests/InteropTestsRemote.m | 6 +++++- .../InteropTestsRemoteWithCronet.m | 7 ++++++- src/objective-c/tests/Tests.xcodeproj/project.pbxproj | 1 + src/objective-c/tests/run_tests.sh | 4 ++++ 6 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/objective-c/tests/InteropTestsLocalCleartext.m b/src/objective-c/tests/InteropTestsLocalCleartext.m index 94cdf859653..cdcdd8a88c7 100644 --- a/src/objective-c/tests/InteropTestsLocalCleartext.m +++ b/src/objective-c/tests/InteropTestsLocalCleartext.m @@ -36,7 +36,11 @@ #import "InteropTests.h" -static NSString * const kLocalCleartextHost = @"localhost:5050"; +// The server address is derived from preprocessor macro, which is +// in turn derived from environment variable of the same name. +#define NSStringize_helper(x) #x +#define NSStringize(x) @NSStringize_helper(x) +static NSString * const kLocalCleartextHost = NSStringize(HOST_PORT_LOCAL); // The Protocol Buffers encoding overhead of local interop server. Acquired // by experiment. Adjust this when server's proto file changes. diff --git a/src/objective-c/tests/InteropTestsLocalSSL.m b/src/objective-c/tests/InteropTestsLocalSSL.m index 3c78b65ede9..45e62d29b9a 100644 --- a/src/objective-c/tests/InteropTestsLocalSSL.m +++ b/src/objective-c/tests/InteropTestsLocalSSL.m @@ -35,8 +35,11 @@ #import #import "InteropTests.h" - -static NSString * const kLocalSSLHost = @"localhost:5051"; +// The server address is derived from preprocessor macro, which is +// in turn derived from environment variable of the same name. +#define NSStringize_helper(x) #x +#define NSStringize(x) @NSStringize_helper(x) +static NSString * const kLocalSSLHost = NSStringize(HOST_PORT_LOCALSSL); // The Protocol Buffers encoding overhead of local interop server. Acquired // by experiment. Adjust this when server's proto file changes. diff --git a/src/objective-c/tests/InteropTestsRemote.m b/src/objective-c/tests/InteropTestsRemote.m index ff1193302b0..5260fe4570b 100644 --- a/src/objective-c/tests/InteropTestsRemote.m +++ b/src/objective-c/tests/InteropTestsRemote.m @@ -36,7 +36,11 @@ #import "InteropTests.h" -static NSString * const kRemoteSSLHost = @"grpc-test.sandbox.googleapis.com"; +// The server address is derived from preprocessor macro, which is +// in turn derived from environment variable of the same name. +#define NSStringize_helper(x) #x +#define NSStringize(x) @NSStringize_helper(x) +static NSString * const kRemoteSSLHost = NSStringize(HOST_PORT_REMOTE); // The Protocol Buffers encoding overhead of remote interop server. Acquired // by experiment. Adjust this when server's proto file changes. diff --git a/src/objective-c/tests/InteropTestsRemoteWithCronet/InteropTestsRemoteWithCronet.m b/src/objective-c/tests/InteropTestsRemoteWithCronet/InteropTestsRemoteWithCronet.m index 9edfbc2639d..a7f190d2b4a 100644 --- a/src/objective-c/tests/InteropTestsRemoteWithCronet/InteropTestsRemoteWithCronet.m +++ b/src/objective-c/tests/InteropTestsRemoteWithCronet/InteropTestsRemoteWithCronet.m @@ -39,7 +39,12 @@ #import "InteropTests.h" -static NSString * const kRemoteSSLHost = @"grpc-test.sandbox.googleapis.com"; +// The server address is derived from preprocessor macro, which is +// in turn derived from environment variable of the same name. +#define NSStringize_helper(x) #x +#define NSStringize(x) @NSStringize_helper(x) +static NSString * const kRemoteSSLHost = NSStringize(HOST_PORT_REMOTE); + // The Protocol Buffers encoding overhead of remote interop server. Acquired // by experiment. Adjust this when server's proto file changes. diff --git a/src/objective-c/tests/Tests.xcodeproj/project.pbxproj b/src/objective-c/tests/Tests.xcodeproj/project.pbxproj index 97de723a22a..eee4897ca80 100644 --- a/src/objective-c/tests/Tests.xcodeproj/project.pbxproj +++ b/src/objective-c/tests/Tests.xcodeproj/project.pbxproj @@ -1282,6 +1282,7 @@ GCC_PREPROCESSOR_DEFINITIONS = ( "DEBUG=1", "$(inherited)", + "HOST_PORT=$(HOST_PORT)", ); GCC_SYMBOLS_PRIVATE_EXTERN = NO; GCC_TREAT_WARNINGS_AS_ERRORS = YES; diff --git a/src/objective-c/tests/run_tests.sh b/src/objective-c/tests/run_tests.sh index 0e82bcaa44a..2432209f4f1 100755 --- a/src/objective-c/tests/run_tests.sh +++ b/src/objective-c/tests/run_tests.sh @@ -59,6 +59,9 @@ xcodebuild \ -workspace Tests.xcworkspace \ -scheme AllTests \ -destination name="iPhone 6" \ + HOST_PORT_LOCALSSL=localhost:5051 \ + HOST_PORT_LOCAL=localhost:5050 \ + HOST_PORT_REMOTE=grpc-test.sandbox.googleapis.com \ test | xcpretty echo "TIME: $(date)" @@ -84,4 +87,5 @@ xcodebuild \ -workspace Tests.xcworkspace \ -scheme InteropTestsRemoteWithCronet \ -destination name="iPhone 6" \ + HOST_PORT_REMOTE=grpc-test.sandbox.googleapis.com \ test | xcpretty From f70b9ee3d43c0786fb1eef16b7f50ab7f74173d7 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Wed, 12 Apr 2017 13:21:34 -0700 Subject: [PATCH 27/60] added correct preprocessor defines to xcodeproj --- src/objective-c/tests/Tests.xcodeproj/project.pbxproj | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/objective-c/tests/Tests.xcodeproj/project.pbxproj b/src/objective-c/tests/Tests.xcodeproj/project.pbxproj index eee4897ca80..f838f4c2214 100644 --- a/src/objective-c/tests/Tests.xcodeproj/project.pbxproj +++ b/src/objective-c/tests/Tests.xcodeproj/project.pbxproj @@ -1282,7 +1282,9 @@ GCC_PREPROCESSOR_DEFINITIONS = ( "DEBUG=1", "$(inherited)", - "HOST_PORT=$(HOST_PORT)", + "HOST_PORT_LOCALSSL=$(HOST_PORT_LOCALSSL)", + "HOST_PORT_LOCAL=$(HOST_PORT_LOCAL)", + "HOST_PORT_REMOTE=$(HOST_PORT_REMOTE)", ); GCC_SYMBOLS_PRIVATE_EXTERN = NO; GCC_TREAT_WARNINGS_AS_ERRORS = YES; From 67dd855337d890656b46c8a6247ee924850fa935 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 12 Apr 2017 16:42:30 -0700 Subject: [PATCH 28/60] Combine multiple temporally disconnected runs --- tools/profiling/microbenchmarks/bm_diff.py | 79 +++++++++++----------- 1 file changed, 41 insertions(+), 38 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index c3738605194..5bc0eacd22c 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -94,7 +94,8 @@ argp.add_argument('-t', '--track', help='Which metrics to track') argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) -argp.add_argument('-r', '--repetitions', type=int, default=30) +argp.add_argument('-r', '--repetitions', type=int, default=5) +argp.add_argument('-l', '--loops', type=int, default=5) argp.add_argument('-p', '--p_threshold', type=float, default=0.01) argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count()) args = argp.parse_args() @@ -123,33 +124,34 @@ def build(): subprocess.check_call(make_cmd('opt')) subprocess.check_call(make_cmd('counters')) -def collect1(bm, cfg, ver): +def collect1(bm, cfg, ver, idx): cmd = ['bins/%s/%s' % (cfg, bm), - '--benchmark_out=%s.%s.%s.json' % (bm, cfg, ver), + '--benchmark_out=%s.%s.%s.%d.json' % (bm, cfg, ver, idx), '--benchmark_out_format=json', '--benchmark_repetitions=%d' % (args.repetitions) ] return jobset.JobSpec(cmd, shortname='%s %s %s' % (bm, cfg, ver), verbose_success=True, timeout_seconds=None) -build() -jobset.run(itertools.chain( - (collect1(bm, 'opt', 'new') for bm in args.benchmarks), - (collect1(bm, 'counters', 'new') for bm in args.benchmarks), -), maxjobs=args.jobs) - -where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() -subprocess.check_call(['git', 'checkout', args.diff_base]) - -try: +for loop in range(0, args.loops): build() jobset.run(itertools.chain( - (collect1(bm, 'opt', 'old') for bm in args.benchmarks), - (collect1(bm, 'counters', 'old') for bm in args.benchmarks), + (collect1(bm, 'opt', 'new', loop) for bm in args.benchmarks), + (collect1(bm, 'counters', 'new', loop) for bm in args.benchmarks), ), maxjobs=args.jobs) -finally: - subprocess.check_call(['git', 'checkout', where_am_i]) - subprocess.check_call(['git', 'submodule', 'update']) + + where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() + subprocess.check_call(['git', 'checkout', args.diff_base]) + + try: + build() + jobset.run(itertools.chain( + (collect1(bm, 'opt', 'old', loop) for bm in args.benchmarks), + (collect1(bm, 'counters', 'old', loop) for bm in args.benchmarks), + ), maxjobs=args.jobs) + finally: + subprocess.check_call(['git', 'checkout', where_am_i]) + subprocess.check_call(['git', 'submodule', 'update']) class Benchmark: @@ -174,7 +176,7 @@ class Benchmark: mdn_diff = abs(median(new) - median(old)) print '%s: new=%r old=%r mdn_diff=%r' % (f, new, old, mdn_diff) s = speedup.speedup(new, old) - if s and mdn_diff > 0.5 and abs(s) >= 10: + if s and mdn_diff > 0.5: self.final[f] = '%+d%%' % s return self.final.keys() @@ -188,25 +190,26 @@ class Benchmark: benchmarks = collections.defaultdict(Benchmark) for bm in args.benchmarks: - with open('%s.counters.new.json' % bm) as f: - js_new_ctr = json.loads(f.read()) - with open('%s.opt.new.json' % bm) as f: - js_new_opt = json.loads(f.read()) - with open('%s.counters.old.json' % bm) as f: - js_old_ctr = json.loads(f.read()) - with open('%s.opt.old.json' % bm) as f: - js_old_opt = json.loads(f.read()) - - for row in bm_json.expand_json(js_new_ctr, js_new_opt): - print row - name = row['cpp_name'] - if name.endswith('_mean') or name.endswith('_stddev'): continue - benchmarks[name].add_sample(row, True) - for row in bm_json.expand_json(js_old_ctr, js_old_opt): - print row - name = row['cpp_name'] - if name.endswith('_mean') or name.endswith('_stddev'): continue - benchmarks[name].add_sample(row, False) + for loop in args.loops: + with open('%s.counters.new.%d.json' % (bm, loop)) as f: + js_new_ctr = json.loads(f.read()) + with open('%s.opt.new.%d.json' % (bm, loop)) as f: + js_new_opt = json.loads(f.read()) + with open('%s.counters.old.%d.json' % (bm, loop)) as f: + js_old_ctr = json.loads(f.read()) + with open('%s.opt.old.%d.json' % (bm, loop)) as f: + js_old_opt = json.loads(f.read()) + + for row in bm_json.expand_json(js_new_ctr, js_new_opt): + print row + name = row['cpp_name'] + if name.endswith('_mean') or name.endswith('_stddev'): continue + benchmarks[name].add_sample(row, True) + for row in bm_json.expand_json(js_old_ctr, js_old_opt): + print row + name = row['cpp_name'] + if name.endswith('_mean') or name.endswith('_stddev'): continue + benchmarks[name].add_sample(row, False) really_interesting = set() for name, bm in benchmarks.items(): From d02c9d2b54f8ab0f263351328451df9718c7cbdf Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Wed, 12 Apr 2017 16:52:39 -0700 Subject: [PATCH 29/60] add ability to run objc interop tests --- tools/run_tests/run_interop_tests.py | 43 ++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index d2bf529fef3..fcc752fdea4 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -371,6 +371,39 @@ class PHP7Language: def __str__(self): return 'php7' +class ObjcLanguage: + + def __init__(self): + self.client_cwd = 'src/objective-c/tests' + self.safename = str(self) + + def client_cmd(self, args): + # from args, extract the server port and craft xcodebuild command out of it + for arg in args: + port = re.search('--server_port=(\d+)', arg) + if port: + portnum = port.group(1) + cmdline = 'pod install && xcodebuild -workspace Tests.xcworkspace -scheme InteropTestsLocalSSL -destination name="iPhone 6" HOST_PORT_LOCALSSL=localhost:%s test'%portnum + return [cmdline] + + def cloud_to_prod_env(self): + return {} + + def global_env(self): + return {} + + def unimplemented_test_cases(self): + # ObjC test runs all cases with the same command. It ignores the testcase + # cmdline argument. Here we return all but one test cases as unimplemented, + # and depend upon ObjC test's behavior that it runs all cases even when + # we tell it to run just one. + return _TEST_CASES[1:] + + def unimplemented_test_cases_server(self): + return _SKIP_COMPRESSION + + def __str__(self): + return 'objc' class RubyLanguage: @@ -402,7 +435,6 @@ class RubyLanguage: def __str__(self): return 'ruby' - class PythonLanguage: def __init__(self): @@ -460,6 +492,7 @@ _LANGUAGES = { 'node' : NodeLanguage(), 'php' : PHPLanguage(), 'php7' : PHP7Language(), + 'objc' : ObjcLanguage(), 'ruby' : RubyLanguage(), 'python' : PythonLanguage(), } @@ -667,7 +700,8 @@ def cloud_to_cloud_jobspec(language, test_case, server_name, server_host, cwd = language.client_cwd environ = language.global_env() - if docker_image: + if docker_image and language.safename != 'objc': + # we can't run client in docker for objc. container_name = dockerjob.random_name('interop_client_%s' % language.safename) cmdline = docker_run_cmdline(cmdline, image=docker_image, @@ -820,7 +854,7 @@ argp.add_argument('-l', '--language', choices=['all'] + sorted(_LANGUAGES), nargs='+', default=['all'], - help='Clients to run.') + help='Clients to run. Objc client can be only run on OSX.') argp.add_argument('-j', '--jobs', default=multiprocessing.cpu_count(), type=int) argp.add_argument('--cloud_to_prod', default=False, @@ -942,6 +976,9 @@ if args.use_docker: build_jobs = [] for l in languages_to_build: + if str(l) == 'objc': + # we don't need to build a docker image for objc + continue job = build_interop_image_jobspec(l) docker_images[str(l)] = job.tag build_jobs.append(job) From 29723ee1ba8a52bc8b373d9bbe3dcfae266b95f1 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 12 Apr 2017 20:24:42 -0700 Subject: [PATCH 30/60] Revert "Revert "Implement lazy deframe"" --- .../chttp2/transport/chttp2_transport.c | 338 ++++++++++------ .../transport/chttp2/transport/frame_data.c | 365 ++++++++++-------- .../transport/chttp2/transport/frame_data.h | 24 +- .../ext/transport/chttp2/transport/internal.h | 51 ++- .../ext/transport/chttp2/transport/parsing.c | 5 +- .../cronet/transport/cronet_transport.c | 17 +- src/core/lib/channel/compress_filter.c | 14 +- src/core/lib/channel/http_client_filter.c | 14 +- src/core/lib/surface/call.c | 44 ++- src/core/lib/transport/byte_stream.c | 32 +- src/core/lib/transport/byte_stream.h | 21 +- .../microbenchmarks/bm_chttp2_transport.cc | 9 +- 12 files changed, 584 insertions(+), 350 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index a7d047d6e7a..29ed4bf90e5 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -44,6 +44,7 @@ #include #include +#include "src/core/ext/transport/chttp2/transport/frame_data.h" #include "src/core/ext/transport/chttp2/transport/internal.h" #include "src/core/ext/transport/chttp2/transport/varint.h" #include "src/core/lib/channel/channel_args.h" @@ -129,6 +130,11 @@ static void incoming_byte_stream_update_flow_control(grpc_exec_ctx *exec_ctx, static void incoming_byte_stream_destroy_locked(grpc_exec_ctx *exec_ctx, void *byte_stream, grpc_error *error_ignored); +static void incoming_byte_stream_publish_error( + grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs, + grpc_error *error); +static void incoming_byte_stream_unref(grpc_exec_ctx *exec_ctx, + grpc_chttp2_incoming_byte_stream *bs); static void benign_reclaimer_locked(grpc_exec_ctx *exec_ctx, void *t, grpc_error *error); @@ -174,6 +180,9 @@ static void finish_keepalive_ping_locked(grpc_exec_ctx *exec_ctx, void *arg, static void keepalive_watchdog_fired_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error); +static void reset_byte_stream(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error); + /******************************************************************************* * CONSTRUCTION/DESTRUCTION/REFCOUNTING */ @@ -655,7 +664,6 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, /* We reserve one 'active stream' that's dropped when the stream is read-closed. The others are for incoming_byte_streams that are actively reading */ - gpr_ref_init(&s->active_streams, 1); GRPC_CHTTP2_STREAM_REF(s, "chttp2"); grpc_chttp2_incoming_metadata_buffer_init(&s->metadata_buffer[0], arena); @@ -665,6 +673,11 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, s->deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC); grpc_closure_init(&s->complete_fetch_locked, complete_fetch_locked, s, grpc_schedule_on_exec_ctx); + grpc_slice_buffer_init(&s->unprocessed_incoming_frames_buffer); + grpc_slice_buffer_init(&s->frame_storage); + s->pending_byte_stream = false; + grpc_closure_init(&s->reset_byte_stream, reset_byte_stream, s, + grpc_combiner_scheduler(t->combiner, false)); GRPC_CHTTP2_REF_TRANSPORT(t, "stream"); @@ -682,7 +695,6 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, static void destroy_stream_locked(grpc_exec_ctx *exec_ctx, void *sp, grpc_error *error) { - grpc_byte_stream *bs; grpc_chttp2_stream *s = sp; grpc_chttp2_transport *t = s->t; @@ -693,9 +705,9 @@ static void destroy_stream_locked(grpc_exec_ctx *exec_ctx, void *sp, GPR_ASSERT(grpc_chttp2_stream_map_find(&t->stream_map, s->id) == NULL); } - while ((bs = grpc_chttp2_incoming_frame_queue_pop(&s->incoming_frames))) { - incoming_byte_stream_destroy_locked(exec_ctx, bs, GRPC_ERROR_NONE); - } + grpc_slice_buffer_destroy_internal(exec_ctx, + &s->unprocessed_incoming_frames_buffer); + grpc_slice_buffer_destroy_internal(exec_ctx, &s->frame_storage); grpc_chttp2_list_remove_stalled_by_transport(t, s); grpc_chttp2_list_remove_stalled_by_stream(t, s); @@ -722,6 +734,7 @@ static void destroy_stream_locked(grpc_exec_ctx *exec_ctx, void *sp, grpc_slice_buffer_destroy_internal(exec_ctx, &s->flow_controlled_buffer); GRPC_ERROR_UNREF(s->read_closed_error); GRPC_ERROR_UNREF(s->write_closed_error); + GRPC_ERROR_UNREF(s->byte_stream_error); if (s->incoming_window_delta > 0) { GRPC_CHTTP2_FLOW_DEBIT_STREAM_INCOMING_WINDOW_DELTA( @@ -1175,8 +1188,9 @@ static void continue_fetching_send_locked(grpc_exec_ctx *exec_ctx, s->fetching_send_message = NULL; return; /* early out */ } else if (grpc_byte_stream_next(exec_ctx, s->fetching_send_message, - &s->fetching_slice, UINT32_MAX, - &s->complete_fetch_locked)) { + UINT32_MAX, &s->complete_fetch_locked)) { + grpc_byte_stream_pull(exec_ctx, s->fetching_send_message, + &s->fetching_slice); add_fetched_slice_locked(exec_ctx, t, s); } } @@ -1187,9 +1201,15 @@ static void complete_fetch_locked(grpc_exec_ctx *exec_ctx, void *gs, grpc_chttp2_stream *s = gs; grpc_chttp2_transport *t = s->t; if (error == GRPC_ERROR_NONE) { - add_fetched_slice_locked(exec_ctx, t, s); - continue_fetching_send_locked(exec_ctx, t, s); - } else { + error = grpc_byte_stream_pull(exec_ctx, s->fetching_send_message, + &s->fetching_slice); + if (error == GRPC_ERROR_NONE) { + add_fetched_slice_locked(exec_ctx, t, s); + continue_fetching_send_locked(exec_ctx, t, s); + } + } + + if (error != GRPC_ERROR_NONE) { /* TODO(ctiller): what to do here */ abort(); } @@ -1424,8 +1444,7 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, GPR_ASSERT(s->recv_message_ready == NULL); s->recv_message_ready = op_payload->recv_message.recv_message_ready; s->recv_message = op_payload->recv_message.recv_message; - if (s->id != 0 && - (s->incoming_frames.head == NULL || s->incoming_frames.head->is_tail)) { + if (s->id != 0 && s->frame_storage.length == 0) { incoming_byte_stream_update_flow_control(exec_ctx, t, s, 5, 0); } grpc_chttp2_maybe_complete_recv_message(exec_ctx, t, s); @@ -1614,13 +1633,13 @@ static void perform_transport_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, void grpc_chttp2_maybe_complete_recv_initial_metadata(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_chttp2_stream *s) { - grpc_byte_stream *bs; if (s->recv_initial_metadata_ready != NULL && s->published_metadata[0] != GRPC_METADATA_NOT_PUBLISHED) { if (s->seen_error) { - while ((bs = grpc_chttp2_incoming_frame_queue_pop(&s->incoming_frames)) != - NULL) { - incoming_byte_stream_destroy_locked(exec_ctx, bs, GRPC_ERROR_NONE); + grpc_slice_buffer_reset_and_unref_internal(exec_ctx, &s->frame_storage); + if (!s->pending_byte_stream) { + grpc_slice_buffer_reset_and_unref_internal( + exec_ctx, &s->unprocessed_incoming_frames_buffer); } } grpc_chttp2_incoming_metadata_buffer_publish( @@ -1633,39 +1652,65 @@ void grpc_chttp2_maybe_complete_recv_initial_metadata(grpc_exec_ctx *exec_ctx, void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_chttp2_stream *s) { - grpc_byte_stream *bs; + grpc_error *error = GRPC_ERROR_NONE; if (s->recv_message_ready != NULL) { - while (s->final_metadata_requested && s->seen_error && - (bs = grpc_chttp2_incoming_frame_queue_pop(&s->incoming_frames)) != - NULL) { - incoming_byte_stream_destroy_locked(exec_ctx, bs, GRPC_ERROR_NONE); + *s->recv_message = NULL; + if (s->final_metadata_requested && s->seen_error) { + grpc_slice_buffer_reset_and_unref_internal(exec_ctx, &s->frame_storage); + if (!s->pending_byte_stream) { + grpc_slice_buffer_reset_and_unref_internal( + exec_ctx, &s->unprocessed_incoming_frames_buffer); + } } - if (s->incoming_frames.head != NULL) { - *s->recv_message = - grpc_chttp2_incoming_frame_queue_pop(&s->incoming_frames); - GPR_ASSERT(*s->recv_message != NULL); + if (!s->pending_byte_stream) { + while (s->unprocessed_incoming_frames_buffer.length > 0 || + s->frame_storage.length > 0) { + if (s->unprocessed_incoming_frames_buffer.length == 0) { + grpc_slice_buffer_swap(&s->unprocessed_incoming_frames_buffer, + &s->frame_storage); + } + error = deframe_unprocessed_incoming_frames( + exec_ctx, &s->data_parser, s, + &s->unprocessed_incoming_frames_buffer, NULL, s->recv_message); + if (error != GRPC_ERROR_NONE) { + s->seen_error = true; + grpc_slice_buffer_reset_and_unref_internal(exec_ctx, + &s->frame_storage); + grpc_slice_buffer_reset_and_unref_internal( + exec_ctx, &s->unprocessed_incoming_frames_buffer); + break; + } else if (*s->recv_message != NULL) { + break; + } + } + } + if (error == GRPC_ERROR_NONE && *s->recv_message != NULL) { null_then_run_closure(exec_ctx, &s->recv_message_ready, GRPC_ERROR_NONE); } else if (s->published_metadata[1] != GRPC_METADATA_NOT_PUBLISHED) { *s->recv_message = NULL; null_then_run_closure(exec_ctx, &s->recv_message_ready, GRPC_ERROR_NONE); } + GRPC_ERROR_UNREF(error); } } void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_chttp2_stream *s) { - grpc_byte_stream *bs; grpc_chttp2_maybe_complete_recv_message(exec_ctx, t, s); if (s->recv_trailing_metadata_finished != NULL && s->read_closed && s->write_closed) { if (s->seen_error) { - while ((bs = grpc_chttp2_incoming_frame_queue_pop(&s->incoming_frames)) != - NULL) { - incoming_byte_stream_destroy_locked(exec_ctx, bs, GRPC_ERROR_NONE); + grpc_slice_buffer_reset_and_unref_internal(exec_ctx, &s->frame_storage); + if (!s->pending_byte_stream) { + grpc_slice_buffer_reset_and_unref_internal( + exec_ctx, &s->unprocessed_incoming_frames_buffer); } } - if (s->all_incoming_byte_streams_finished && + bool pending_data = s->pending_byte_stream || + s->unprocessed_incoming_frames_buffer.length > 0; + if (s->read_closed && s->frame_storage.length == 0 && + (!pending_data || s->seen_error) && s->recv_trailing_metadata_finished != NULL) { grpc_chttp2_incoming_metadata_buffer_publish( exec_ctx, &s->metadata_buffer[1], s->recv_trailing_metadata); @@ -1676,14 +1721,6 @@ void grpc_chttp2_maybe_complete_recv_trailing_metadata(grpc_exec_ctx *exec_ctx, } } -static void decrement_active_streams_locked(grpc_exec_ctx *exec_ctx, - grpc_chttp2_transport *t, - grpc_chttp2_stream *s) { - if ((s->all_incoming_byte_streams_finished = gpr_unref(&s->active_streams))) { - grpc_chttp2_maybe_complete_recv_trailing_metadata(exec_ctx, t, s); - } -} - static void remove_stream(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, uint32_t id, grpc_error *error) { grpc_chttp2_stream *s = grpc_chttp2_stream_map_delete(&t->stream_map, id); @@ -1692,10 +1729,19 @@ static void remove_stream(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, t->incoming_stream = NULL; grpc_chttp2_parsing_become_skip_parser(exec_ctx, t); } - if (s->data_parser.parsing_frame != NULL) { - grpc_chttp2_incoming_byte_stream_finished( - exec_ctx, s->data_parser.parsing_frame, GRPC_ERROR_REF(error)); - s->data_parser.parsing_frame = NULL; + if (s->pending_byte_stream) { + if (s->on_next != NULL) { + grpc_chttp2_incoming_byte_stream *bs = s->data_parser.parsing_frame; + if (error == GRPC_ERROR_NONE) { + error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Truncated message"); + } + incoming_byte_stream_publish_error(exec_ctx, bs, error); + incoming_byte_stream_unref(exec_ctx, bs); + s->data_parser.parsing_frame = NULL; + } else { + GRPC_ERROR_UNREF(s->byte_stream_error); + s->byte_stream_error = GRPC_ERROR_REF(error); + } } if (grpc_chttp2_stream_map_size(&t->stream_map) == 0) { @@ -1881,7 +1927,6 @@ void grpc_chttp2_mark_stream_closed(grpc_exec_ctx *exec_ctx, s->published_metadata[i] = GPRC_METADATA_PUBLISHED_AT_CLOSE; } } - decrement_active_streams_locked(exec_ctx, t, s); grpc_chttp2_maybe_complete_recv_initial_metadata(exec_ctx, t, s); grpc_chttp2_maybe_complete_recv_message(exec_ctx, t, s); } @@ -2419,12 +2464,28 @@ static void set_pollset_set(grpc_exec_ctx *exec_ctx, grpc_transport *gt, * BYTE STREAM */ +static void reset_byte_stream(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + grpc_chttp2_stream *s = (grpc_chttp2_stream *)arg; + + s->pending_byte_stream = false; + if (error == GRPC_ERROR_NONE) { + grpc_chttp2_maybe_complete_recv_message(exec_ctx, s->t, s); + grpc_chttp2_maybe_complete_recv_trailing_metadata(exec_ctx, s->t, s); + } else { + GPR_ASSERT(error != GRPC_ERROR_NONE); + grpc_closure_sched(exec_ctx, s->on_next, GRPC_ERROR_REF(error)); + s->on_next = NULL; + GRPC_ERROR_UNREF(s->byte_stream_error); + s->byte_stream_error = GRPC_ERROR_NONE; + grpc_chttp2_cancel_stream(exec_ctx, s->t, s, GRPC_ERROR_REF(error)); + s->byte_stream_error = error; + } +} + static void incoming_byte_stream_unref(grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs) { if (gpr_unref(&bs->refs)) { - GRPC_ERROR_UNREF(bs->error); - grpc_slice_buffer_destroy_internal(exec_ctx, &bs->slices); - gpr_mu_destroy(&bs->slice_mu); gpr_free(bs); } } @@ -2484,47 +2545,90 @@ static void incoming_byte_stream_next_locked(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t = bs->transport; grpc_chttp2_stream *s = bs->stream; - if (bs->is_tail) { - gpr_mu_lock(&bs->slice_mu); - size_t cur_length = bs->slices.length; - gpr_mu_unlock(&bs->slice_mu); - incoming_byte_stream_update_flow_control( - exec_ctx, t, s, bs->next_action.max_size_hint, cur_length); - } - gpr_mu_lock(&bs->slice_mu); - if (bs->slices.count > 0) { - *bs->next_action.slice = grpc_slice_buffer_take_first(&bs->slices); - grpc_closure_run(exec_ctx, bs->next_action.on_complete, GRPC_ERROR_NONE); - } else if (bs->error != GRPC_ERROR_NONE) { - grpc_closure_run(exec_ctx, bs->next_action.on_complete, - GRPC_ERROR_REF(bs->error)); + size_t cur_length = s->frame_storage.length; + incoming_byte_stream_update_flow_control( + exec_ctx, t, s, bs->next_action.max_size_hint, cur_length); + + GPR_ASSERT(s->unprocessed_incoming_frames_buffer.length == 0); + if (s->frame_storage.length > 0) { + grpc_slice_buffer_swap(&s->frame_storage, + &s->unprocessed_incoming_frames_buffer); + grpc_closure_sched(exec_ctx, bs->next_action.on_complete, GRPC_ERROR_NONE); + } else if (s->byte_stream_error != GRPC_ERROR_NONE) { + grpc_closure_sched(exec_ctx, bs->next_action.on_complete, + GRPC_ERROR_REF(s->byte_stream_error)); + if (s->data_parser.parsing_frame != NULL) { + incoming_byte_stream_unref(exec_ctx, s->data_parser.parsing_frame); + s->data_parser.parsing_frame = NULL; + } + } else if (s->read_closed) { + if (bs->remaining_bytes != 0) { + s->byte_stream_error = + GRPC_ERROR_CREATE_FROM_STATIC_STRING("Truncated message"); + grpc_closure_sched(exec_ctx, bs->next_action.on_complete, + GRPC_ERROR_REF(s->byte_stream_error)); + if (s->data_parser.parsing_frame != NULL) { + incoming_byte_stream_unref(exec_ctx, s->data_parser.parsing_frame); + s->data_parser.parsing_frame = NULL; + } + } else { + /* Should never reach here. */ + GPR_ASSERT(false); + } } else { - bs->on_next = bs->next_action.on_complete; - bs->next = bs->next_action.slice; + s->on_next = bs->next_action.on_complete; } - gpr_mu_unlock(&bs->slice_mu); incoming_byte_stream_unref(exec_ctx, bs); } -static int incoming_byte_stream_next(grpc_exec_ctx *exec_ctx, - grpc_byte_stream *byte_stream, - grpc_slice *slice, size_t max_size_hint, - grpc_closure *on_complete) { +static bool incoming_byte_stream_next(grpc_exec_ctx *exec_ctx, + grpc_byte_stream *byte_stream, + size_t max_size_hint, + grpc_closure *on_complete) { GPR_TIMER_BEGIN("incoming_byte_stream_next", 0); grpc_chttp2_incoming_byte_stream *bs = (grpc_chttp2_incoming_byte_stream *)byte_stream; - gpr_ref(&bs->refs); - bs->next_action.slice = slice; - bs->next_action.max_size_hint = max_size_hint; - bs->next_action.on_complete = on_complete; - grpc_closure_sched( - exec_ctx, - grpc_closure_init( - &bs->next_action.closure, incoming_byte_stream_next_locked, bs, - grpc_combiner_scheduler(bs->transport->combiner, false)), - GRPC_ERROR_NONE); - GPR_TIMER_END("incoming_byte_stream_next", 0); - return 0; + grpc_chttp2_stream *s = bs->stream; + if (s->unprocessed_incoming_frames_buffer.length > 0) { + return true; + } else { + gpr_ref(&bs->refs); + bs->next_action.max_size_hint = max_size_hint; + bs->next_action.on_complete = on_complete; + grpc_closure_sched( + exec_ctx, + grpc_closure_init( + &bs->next_action.closure, incoming_byte_stream_next_locked, bs, + grpc_combiner_scheduler(bs->transport->combiner, false)), + GRPC_ERROR_NONE); + GPR_TIMER_END("incoming_byte_stream_next", 0); + return false; + } +} + +static grpc_error *incoming_byte_stream_pull(grpc_exec_ctx *exec_ctx, + grpc_byte_stream *byte_stream, + grpc_slice *slice) { + GPR_TIMER_BEGIN("incoming_byte_stream_pull", 0); + grpc_chttp2_incoming_byte_stream *bs = + (grpc_chttp2_incoming_byte_stream *)byte_stream; + grpc_chttp2_stream *s = bs->stream; + + if (s->unprocessed_incoming_frames_buffer.length > 0) { + grpc_error *error = deframe_unprocessed_incoming_frames( + exec_ctx, &s->data_parser, s, &s->unprocessed_incoming_frames_buffer, + slice, NULL); + if (error != GRPC_ERROR_NONE) { + return error; + } + } else { + grpc_error *error = + GRPC_ERROR_CREATE_FROM_STATIC_STRING("Truncated message"); + grpc_closure_sched(exec_ctx, &s->reset_byte_stream, GRPC_ERROR_REF(error)); + return error; + } + GPR_TIMER_END("incoming_byte_stream_pull", 0); + return GRPC_ERROR_NONE; } static void incoming_byte_stream_destroy(grpc_exec_ctx *exec_ctx, @@ -2534,9 +2638,14 @@ static void incoming_byte_stream_destroy_locked(grpc_exec_ctx *exec_ctx, void *byte_stream, grpc_error *error_ignored) { grpc_chttp2_incoming_byte_stream *bs = byte_stream; + grpc_chttp2_stream *s = bs->stream; + grpc_chttp2_transport *t = s->t; + GPR_ASSERT(bs->base.destroy == incoming_byte_stream_destroy); - decrement_active_streams_locked(exec_ctx, bs->transport, bs->stream); incoming_byte_stream_unref(exec_ctx, bs); + s->pending_byte_stream = false; + grpc_chttp2_maybe_complete_recv_message(exec_ctx, t, s); + grpc_chttp2_maybe_complete_recv_trailing_metadata(exec_ctx, t, s); } static void incoming_byte_stream_destroy(grpc_exec_ctx *exec_ctx, @@ -2556,50 +2665,53 @@ static void incoming_byte_stream_destroy(grpc_exec_ctx *exec_ctx, static void incoming_byte_stream_publish_error( grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs, grpc_error *error) { + grpc_chttp2_stream *s = bs->stream; + GPR_ASSERT(error != GRPC_ERROR_NONE); - grpc_closure_sched(exec_ctx, bs->on_next, GRPC_ERROR_REF(error)); - bs->on_next = NULL; - GRPC_ERROR_UNREF(bs->error); + grpc_closure_sched(exec_ctx, s->on_next, GRPC_ERROR_REF(error)); + s->on_next = NULL; + GRPC_ERROR_UNREF(s->byte_stream_error); + s->byte_stream_error = GRPC_ERROR_REF(error); grpc_chttp2_cancel_stream(exec_ctx, bs->transport, bs->stream, GRPC_ERROR_REF(error)); - bs->error = error; } -void grpc_chttp2_incoming_byte_stream_push(grpc_exec_ctx *exec_ctx, - grpc_chttp2_incoming_byte_stream *bs, - grpc_slice slice) { - gpr_mu_lock(&bs->slice_mu); +grpc_error *grpc_chttp2_incoming_byte_stream_push( + grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs, + grpc_slice slice, grpc_slice *slice_out) { + grpc_chttp2_stream *s = bs->stream; + if (bs->remaining_bytes < GRPC_SLICE_LENGTH(slice)) { - incoming_byte_stream_publish_error( - exec_ctx, bs, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Too many bytes in stream")); + grpc_error *error = + GRPC_ERROR_CREATE_FROM_STATIC_STRING("Too many bytes in stream"); + + grpc_closure_sched(exec_ctx, &s->reset_byte_stream, GRPC_ERROR_REF(error)); + grpc_slice_unref_internal(exec_ctx, slice); + return error; } else { bs->remaining_bytes -= (uint32_t)GRPC_SLICE_LENGTH(slice); - if (bs->on_next != NULL) { - *bs->next = slice; - grpc_closure_sched(exec_ctx, bs->on_next, GRPC_ERROR_NONE); - bs->on_next = NULL; - } else { - grpc_slice_buffer_add(&bs->slices, slice); + if (slice_out != NULL) { + *slice_out = slice; } + return GRPC_ERROR_NONE; } - gpr_mu_unlock(&bs->slice_mu); } -void grpc_chttp2_incoming_byte_stream_finished( +grpc_error *grpc_chttp2_incoming_byte_stream_finished( grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs, - grpc_error *error) { + grpc_error *error, bool reset_on_error) { + grpc_chttp2_stream *s = bs->stream; + if (error == GRPC_ERROR_NONE) { - gpr_mu_lock(&bs->slice_mu); if (bs->remaining_bytes != 0) { error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Truncated message"); } - gpr_mu_unlock(&bs->slice_mu); } - if (error != GRPC_ERROR_NONE) { - incoming_byte_stream_publish_error(exec_ctx, bs, error); + if (error != GRPC_ERROR_NONE && reset_on_error) { + grpc_closure_sched(exec_ctx, &s->reset_byte_stream, GRPC_ERROR_REF(error)); } incoming_byte_stream_unref(exec_ctx, bs); + return error; } grpc_chttp2_incoming_byte_stream *grpc_chttp2_incoming_byte_stream_create( @@ -2611,26 +2723,12 @@ grpc_chttp2_incoming_byte_stream *grpc_chttp2_incoming_byte_stream_create( incoming_byte_stream->remaining_bytes = frame_size; incoming_byte_stream->base.flags = flags; incoming_byte_stream->base.next = incoming_byte_stream_next; + incoming_byte_stream->base.pull = incoming_byte_stream_pull; incoming_byte_stream->base.destroy = incoming_byte_stream_destroy; - gpr_mu_init(&incoming_byte_stream->slice_mu); gpr_ref_init(&incoming_byte_stream->refs, 2); - incoming_byte_stream->next_message = NULL; incoming_byte_stream->transport = t; incoming_byte_stream->stream = s; - gpr_ref(&incoming_byte_stream->stream->active_streams); - grpc_slice_buffer_init(&incoming_byte_stream->slices); - incoming_byte_stream->on_next = NULL; - incoming_byte_stream->is_tail = 1; - incoming_byte_stream->error = GRPC_ERROR_NONE; - grpc_chttp2_incoming_frame_queue *q = &s->incoming_frames; - if (q->head == NULL) { - q->head = incoming_byte_stream; - } else { - q->tail->is_tail = 0; - q->tail->next_message = incoming_byte_stream; - } - q->tail = incoming_byte_stream; - grpc_chttp2_maybe_complete_recv_message(exec_ctx, t, s); + s->byte_stream_error = GRPC_ERROR_NONE; return incoming_byte_stream; } diff --git a/src/core/ext/transport/chttp2/transport/frame_data.c b/src/core/ext/transport/chttp2/transport/frame_data.c index 6e9258ee7ee..5d382d80a8b 100644 --- a/src/core/ext/transport/chttp2/transport/frame_data.c +++ b/src/core/ext/transport/chttp2/transport/frame_data.c @@ -40,6 +40,7 @@ #include #include #include "src/core/ext/transport/chttp2/transport/internal.h" +#include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/support/string.h" #include "src/core/lib/transport/transport.h" @@ -53,16 +54,17 @@ grpc_error *grpc_chttp2_data_parser_init(grpc_chttp2_data_parser *parser) { void grpc_chttp2_data_parser_destroy(grpc_exec_ctx *exec_ctx, grpc_chttp2_data_parser *parser) { if (parser->parsing_frame != NULL) { - grpc_chttp2_incoming_byte_stream_finished( + GRPC_ERROR_UNREF(grpc_chttp2_incoming_byte_stream_finished( exec_ctx, parser->parsing_frame, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Parser destroyed")); + GRPC_ERROR_CREATE_FROM_STATIC_STRING("Parser destroyed"), false)); } GRPC_ERROR_UNREF(parser->error); } grpc_error *grpc_chttp2_data_parser_begin_frame(grpc_chttp2_data_parser *parser, uint8_t flags, - uint32_t stream_id) { + uint32_t stream_id, + grpc_chttp2_stream *s) { if (flags & ~GRPC_CHTTP2_DATA_FLAG_END_STREAM) { char *msg; gpr_asprintf(&msg, "unsupported data flags: 0x%02x", flags); @@ -74,47 +76,14 @@ grpc_error *grpc_chttp2_data_parser_begin_frame(grpc_chttp2_data_parser *parser, } if (flags & GRPC_CHTTP2_DATA_FLAG_END_STREAM) { - parser->is_last_frame = 1; + s->received_last_frame = true; } else { - parser->is_last_frame = 0; + s->received_last_frame = false; } return GRPC_ERROR_NONE; } -void grpc_chttp2_incoming_frame_queue_merge( - grpc_chttp2_incoming_frame_queue *head_dst, - grpc_chttp2_incoming_frame_queue *tail_src) { - if (tail_src->head == NULL) { - return; - } - - if (head_dst->head == NULL) { - *head_dst = *tail_src; - memset(tail_src, 0, sizeof(*tail_src)); - return; - } - - head_dst->tail->next_message = tail_src->head; - head_dst->tail = tail_src->tail; - memset(tail_src, 0, sizeof(*tail_src)); -} - -grpc_byte_stream *grpc_chttp2_incoming_frame_queue_pop( - grpc_chttp2_incoming_frame_queue *q) { - grpc_byte_stream *out; - if (q->head == NULL) { - return NULL; - } - out = &q->head->base; - if (q->head == q->tail) { - memset(q, 0, sizeof(*q)); - } else { - q->head = q->head->next_message; - } - return out; -} - void grpc_chttp2_encode_data(uint32_t id, grpc_slice_buffer *inbuf, uint32_t write_bytes, int is_eof, grpc_transport_one_way_stats *stats, @@ -143,145 +112,217 @@ void grpc_chttp2_encode_data(uint32_t id, grpc_slice_buffer *inbuf, stats->data_bytes += write_bytes; } -static grpc_error *parse_inner(grpc_exec_ctx *exec_ctx, - grpc_chttp2_data_parser *p, - grpc_chttp2_transport *t, grpc_chttp2_stream *s, - grpc_slice slice) { - uint8_t *const beg = GRPC_SLICE_START_PTR(slice); - uint8_t *const end = GRPC_SLICE_END_PTR(slice); - uint8_t *cur = beg; - uint32_t message_flags; - grpc_chttp2_incoming_byte_stream *incoming_byte_stream; - char *msg; +grpc_error *deframe_unprocessed_incoming_frames(grpc_exec_ctx *exec_ctx, + grpc_chttp2_data_parser *p, + grpc_chttp2_stream *s, + grpc_slice_buffer *slices, + grpc_slice *slice_out, + grpc_byte_stream **stream_out) { + grpc_error *error = GRPC_ERROR_NONE; + grpc_chttp2_transport *t = s->t; - if (cur == end) { - return GRPC_ERROR_NONE; - } + while (slices->count > 0) { + uint8_t *beg = NULL; + uint8_t *end = NULL; + uint8_t *cur = NULL; - switch (p->state) { - case GRPC_CHTTP2_DATA_ERROR: - p->state = GRPC_CHTTP2_DATA_ERROR; - return GRPC_ERROR_REF(p->error); - fh_0: - case GRPC_CHTTP2_DATA_FH_0: - s->stats.incoming.framing_bytes++; - p->frame_type = *cur; - switch (p->frame_type) { - case 0: - p->is_frame_compressed = 0; /* GPR_FALSE */ - break; - case 1: - p->is_frame_compressed = 1; /* GPR_TRUE */ - break; - default: - gpr_asprintf(&msg, "Bad GRPC frame type 0x%02x", p->frame_type); - p->error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg); - p->error = grpc_error_set_int(p->error, GRPC_ERROR_INT_STREAM_ID, - (intptr_t)s->id); - gpr_free(msg); - msg = grpc_dump_slice(slice, GPR_DUMP_HEX | GPR_DUMP_ASCII); - p->error = grpc_error_set_str(p->error, GRPC_ERROR_STR_RAW_BYTES, - grpc_slice_from_copied_string(msg)); - gpr_free(msg); - p->error = - grpc_error_set_int(p->error, GRPC_ERROR_INT_OFFSET, cur - beg); - p->state = GRPC_CHTTP2_DATA_ERROR; - return GRPC_ERROR_REF(p->error); - } - if (++cur == end) { - p->state = GRPC_CHTTP2_DATA_FH_1; - return GRPC_ERROR_NONE; - } - /* fallthrough */ - case GRPC_CHTTP2_DATA_FH_1: - s->stats.incoming.framing_bytes++; - p->frame_size = ((uint32_t)*cur) << 24; - if (++cur == end) { - p->state = GRPC_CHTTP2_DATA_FH_2; - return GRPC_ERROR_NONE; - } - /* fallthrough */ - case GRPC_CHTTP2_DATA_FH_2: - s->stats.incoming.framing_bytes++; - p->frame_size |= ((uint32_t)*cur) << 16; - if (++cur == end) { - p->state = GRPC_CHTTP2_DATA_FH_3; - return GRPC_ERROR_NONE; - } - /* fallthrough */ - case GRPC_CHTTP2_DATA_FH_3: - s->stats.incoming.framing_bytes++; - p->frame_size |= ((uint32_t)*cur) << 8; - if (++cur == end) { - p->state = GRPC_CHTTP2_DATA_FH_4; - return GRPC_ERROR_NONE; - } - /* fallthrough */ - case GRPC_CHTTP2_DATA_FH_4: - s->stats.incoming.framing_bytes++; - p->frame_size |= ((uint32_t)*cur); - p->state = GRPC_CHTTP2_DATA_FRAME; - ++cur; - message_flags = 0; - if (p->is_frame_compressed) { - message_flags |= GRPC_WRITE_INTERNAL_COMPRESS; - } - p->parsing_frame = incoming_byte_stream = - grpc_chttp2_incoming_byte_stream_create(exec_ctx, t, s, p->frame_size, - message_flags); - /* fallthrough */ - case GRPC_CHTTP2_DATA_FRAME: - if (cur == end) { - return GRPC_ERROR_NONE; - } - uint32_t remaining = (uint32_t)(end - cur); - if (remaining == p->frame_size) { - s->stats.incoming.data_bytes += p->frame_size; - grpc_chttp2_incoming_byte_stream_push( - exec_ctx, p->parsing_frame, - grpc_slice_sub(slice, (size_t)(cur - beg), (size_t)(end - beg))); - grpc_chttp2_incoming_byte_stream_finished(exec_ctx, p->parsing_frame, - GRPC_ERROR_NONE); - p->parsing_frame = NULL; - p->state = GRPC_CHTTP2_DATA_FH_0; - return GRPC_ERROR_NONE; - } else if (remaining > p->frame_size) { - s->stats.incoming.data_bytes += p->frame_size; - grpc_chttp2_incoming_byte_stream_push( - exec_ctx, p->parsing_frame, - grpc_slice_sub(slice, (size_t)(cur - beg), - (size_t)(cur + p->frame_size - beg))); - grpc_chttp2_incoming_byte_stream_finished(exec_ctx, p->parsing_frame, - GRPC_ERROR_NONE); - p->parsing_frame = NULL; - cur += p->frame_size; - goto fh_0; /* loop */ - } else { - GPR_ASSERT(remaining <= p->frame_size); - grpc_chttp2_incoming_byte_stream_push( - exec_ctx, p->parsing_frame, - grpc_slice_sub(slice, (size_t)(cur - beg), (size_t)(end - beg))); - p->frame_size -= remaining; - s->stats.incoming.data_bytes += remaining; + grpc_slice slice = grpc_slice_buffer_take_first(slices); + + beg = GRPC_SLICE_START_PTR(slice); + end = GRPC_SLICE_END_PTR(slice); + cur = beg; + uint32_t message_flags; + char *msg; + + if (cur == end) { + grpc_slice_unref_internal(exec_ctx, slice); + continue; + } + + switch (p->state) { + case GRPC_CHTTP2_DATA_ERROR: + p->state = GRPC_CHTTP2_DATA_ERROR; + grpc_slice_unref_internal(exec_ctx, slice); + return GRPC_ERROR_REF(p->error); + case GRPC_CHTTP2_DATA_FH_0: + p->frame_type = *cur; + switch (p->frame_type) { + case 0: + p->is_frame_compressed = false; /* GPR_FALSE */ + break; + case 1: + p->is_frame_compressed = true; /* GPR_TRUE */ + break; + default: + gpr_asprintf(&msg, "Bad GRPC frame type 0x%02x", p->frame_type); + p->error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg); + p->error = grpc_error_set_int(p->error, GRPC_ERROR_INT_STREAM_ID, + (intptr_t)s->id); + gpr_free(msg); + msg = grpc_dump_slice(slice, GPR_DUMP_HEX | GPR_DUMP_ASCII); + p->error = grpc_error_set_str(p->error, GRPC_ERROR_STR_RAW_BYTES, + grpc_slice_from_copied_string(msg)); + gpr_free(msg); + p->error = + grpc_error_set_int(p->error, GRPC_ERROR_INT_OFFSET, cur - beg); + p->state = GRPC_CHTTP2_DATA_ERROR; + grpc_slice_unref_internal(exec_ctx, slice); + return GRPC_ERROR_REF(p->error); + } + if (++cur == end) { + p->state = GRPC_CHTTP2_DATA_FH_1; + grpc_slice_unref_internal(exec_ctx, slice); + continue; + } + /* fallthrough */ + case GRPC_CHTTP2_DATA_FH_1: + p->frame_size = ((uint32_t)*cur) << 24; + if (++cur == end) { + p->state = GRPC_CHTTP2_DATA_FH_2; + grpc_slice_unref_internal(exec_ctx, slice); + continue; + } + /* fallthrough */ + case GRPC_CHTTP2_DATA_FH_2: + p->frame_size |= ((uint32_t)*cur) << 16; + if (++cur == end) { + p->state = GRPC_CHTTP2_DATA_FH_3; + grpc_slice_unref_internal(exec_ctx, slice); + continue; + } + /* fallthrough */ + case GRPC_CHTTP2_DATA_FH_3: + p->frame_size |= ((uint32_t)*cur) << 8; + if (++cur == end) { + p->state = GRPC_CHTTP2_DATA_FH_4; + grpc_slice_unref_internal(exec_ctx, slice); + continue; + } + /* fallthrough */ + case GRPC_CHTTP2_DATA_FH_4: + GPR_ASSERT(stream_out != NULL); + GPR_ASSERT(p->parsing_frame == NULL); + p->frame_size |= ((uint32_t)*cur); + p->state = GRPC_CHTTP2_DATA_FRAME; + ++cur; + message_flags = 0; + if (p->is_frame_compressed) { + message_flags |= GRPC_WRITE_INTERNAL_COMPRESS; + } + p->parsing_frame = grpc_chttp2_incoming_byte_stream_create( + exec_ctx, t, s, p->frame_size, message_flags); + *stream_out = &p->parsing_frame->base; + if (p->parsing_frame->remaining_bytes == 0) { + GRPC_ERROR_UNREF(grpc_chttp2_incoming_byte_stream_finished( + exec_ctx, p->parsing_frame, GRPC_ERROR_NONE, true)); + p->parsing_frame = NULL; + p->state = GRPC_CHTTP2_DATA_FH_0; + } + s->pending_byte_stream = true; + + if (cur != end) { + grpc_slice_buffer_undo_take_first( + &s->unprocessed_incoming_frames_buffer, + grpc_slice_sub(slice, (size_t)(cur - beg), (size_t)(end - beg))); + } + grpc_slice_unref_internal(exec_ctx, slice); return GRPC_ERROR_NONE; + case GRPC_CHTTP2_DATA_FRAME: { + GPR_ASSERT(p->parsing_frame != NULL); + GPR_ASSERT(slice_out != NULL); + if (cur == end) { + grpc_slice_unref_internal(exec_ctx, slice); + continue; + } + uint32_t remaining = (uint32_t)(end - cur); + if (remaining == p->frame_size) { + if (GRPC_ERROR_NONE != (error = grpc_chttp2_incoming_byte_stream_push( + exec_ctx, p->parsing_frame, + grpc_slice_sub(slice, (size_t)(cur - beg), + (size_t)(end - beg)), + slice_out))) { + grpc_slice_unref_internal(exec_ctx, slice); + return error; + } + if (GRPC_ERROR_NONE != + (error = grpc_chttp2_incoming_byte_stream_finished( + exec_ctx, p->parsing_frame, GRPC_ERROR_NONE, true))) { + grpc_slice_unref_internal(exec_ctx, slice); + return error; + } + p->parsing_frame = NULL; + p->state = GRPC_CHTTP2_DATA_FH_0; + grpc_slice_unref_internal(exec_ctx, slice); + return GRPC_ERROR_NONE; + } else if (remaining < p->frame_size) { + if (GRPC_ERROR_NONE != (error = grpc_chttp2_incoming_byte_stream_push( + exec_ctx, p->parsing_frame, + grpc_slice_sub(slice, (size_t)(cur - beg), + (size_t)(end - beg)), + slice_out))) { + return error; + } + p->frame_size -= remaining; + grpc_slice_unref_internal(exec_ctx, slice); + return GRPC_ERROR_NONE; + } else { + GPR_ASSERT(remaining > p->frame_size); + if (GRPC_ERROR_NONE != + (grpc_chttp2_incoming_byte_stream_push( + exec_ctx, p->parsing_frame, + grpc_slice_sub(slice, (size_t)(cur - beg), + (size_t)(cur + p->frame_size - beg)), + slice_out))) { + grpc_slice_unref_internal(exec_ctx, slice); + return error; + } + if (GRPC_ERROR_NONE != + (error = grpc_chttp2_incoming_byte_stream_finished( + exec_ctx, p->parsing_frame, GRPC_ERROR_NONE, true))) { + grpc_slice_unref_internal(exec_ctx, slice); + return error; + } + p->parsing_frame = NULL; + p->state = GRPC_CHTTP2_DATA_FH_0; + cur += p->frame_size; + grpc_slice_buffer_undo_take_first( + &s->unprocessed_incoming_frames_buffer, + grpc_slice_sub(slice, (size_t)(cur - beg), (size_t)(end - beg))); + grpc_slice_unref_internal(exec_ctx, slice); + return GRPC_ERROR_NONE; + } } + } } - GPR_UNREACHABLE_CODE( - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Should never reach here")); + return GRPC_ERROR_NONE; } grpc_error *grpc_chttp2_data_parser_parse(grpc_exec_ctx *exec_ctx, void *parser, grpc_chttp2_transport *t, grpc_chttp2_stream *s, grpc_slice slice, int is_last) { - grpc_chttp2_data_parser *p = parser; - grpc_error *error = parse_inner(exec_ctx, p, t, s, slice); + /* grpc_error *error = parse_inner_buffer(exec_ctx, p, t, s, slice); */ + s->stats.incoming.framing_bytes += GRPC_SLICE_LENGTH(slice); + if (!s->pending_byte_stream) { + grpc_slice_ref_internal(slice); + grpc_slice_buffer_add(&s->frame_storage, slice); + grpc_chttp2_maybe_complete_recv_message(exec_ctx, t, s); + } else if (s->on_next) { + GPR_ASSERT(s->frame_storage.length == 0); + grpc_slice_ref_internal(slice); + grpc_slice_buffer_add(&s->unprocessed_incoming_frames_buffer, slice); + grpc_closure_sched(exec_ctx, s->on_next, GRPC_ERROR_NONE); + s->on_next = NULL; + } else { + grpc_slice_ref_internal(slice); + grpc_slice_buffer_add(&s->frame_storage, slice); + } - if (is_last && p->is_last_frame) { + if (is_last && s->received_last_frame) { grpc_chttp2_mark_stream_closed(exec_ctx, t, s, true, false, GRPC_ERROR_NONE); } - return error; + return GRPC_ERROR_NONE; } diff --git a/src/core/ext/transport/chttp2/transport/frame_data.h b/src/core/ext/transport/chttp2/transport/frame_data.h index 264ad146085..2fb8983c38e 100644 --- a/src/core/ext/transport/chttp2/transport/frame_data.h +++ b/src/core/ext/transport/chttp2/transport/frame_data.h @@ -56,28 +56,16 @@ typedef enum { typedef struct grpc_chttp2_incoming_byte_stream grpc_chttp2_incoming_byte_stream; -typedef struct grpc_chttp2_incoming_frame_queue { - grpc_chttp2_incoming_byte_stream *head; - grpc_chttp2_incoming_byte_stream *tail; -} grpc_chttp2_incoming_frame_queue; - typedef struct { grpc_chttp2_stream_state state; - uint8_t is_last_frame; uint8_t frame_type; uint32_t frame_size; grpc_error *error; - int is_frame_compressed; + bool is_frame_compressed; grpc_chttp2_incoming_byte_stream *parsing_frame; } grpc_chttp2_data_parser; -void grpc_chttp2_incoming_frame_queue_merge( - grpc_chttp2_incoming_frame_queue *head_dst, - grpc_chttp2_incoming_frame_queue *tail_src); -grpc_byte_stream *grpc_chttp2_incoming_frame_queue_pop( - grpc_chttp2_incoming_frame_queue *q); - /* initialize per-stream state for data frame parsing */ grpc_error *grpc_chttp2_data_parser_init(grpc_chttp2_data_parser *parser); @@ -87,7 +75,8 @@ void grpc_chttp2_data_parser_destroy(grpc_exec_ctx *exec_ctx, /* start processing a new data frame */ grpc_error *grpc_chttp2_data_parser_begin_frame(grpc_chttp2_data_parser *parser, uint8_t flags, - uint32_t stream_id); + uint32_t stream_id, + grpc_chttp2_stream *s); /* handle a slice of a data frame - is_last indicates the last slice of a frame */ @@ -101,4 +90,11 @@ void grpc_chttp2_encode_data(uint32_t id, grpc_slice_buffer *inbuf, grpc_transport_one_way_stats *stats, grpc_slice_buffer *outbuf); +grpc_error *deframe_unprocessed_incoming_frames(grpc_exec_ctx *exec_ctx, + grpc_chttp2_data_parser *p, + grpc_chttp2_stream *s, + grpc_slice_buffer *slices, + grpc_slice *slice_out, + grpc_byte_stream **stream_out); + #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_DATA_H */ diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 6eb848b8d77..a10e3886ea1 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -195,22 +195,20 @@ typedef struct grpc_chttp2_write_cb { struct grpc_chttp2_incoming_byte_stream { grpc_byte_stream base; gpr_refcount refs; - struct grpc_chttp2_incoming_byte_stream *next_message; - grpc_error *error; - grpc_chttp2_transport *transport; - grpc_chttp2_stream *stream; - bool is_tail; + grpc_chttp2_transport *transport; /* immutable */ + grpc_chttp2_stream *stream; /* immutable */ - gpr_mu slice_mu; // protects slices, on_next - grpc_slice_buffer slices; - grpc_closure *on_next; - grpc_slice *next; + /* Accessed only by transport thread when stream->pending_byte_stream == false + * Accessed only by application thread when stream->pending_byte_stream == + * true */ uint32_t remaining_bytes; + /* Accessed only by transport thread when stream->pending_byte_stream == false + * Accessed only by application thread when stream->pending_byte_stream == + * true */ struct { grpc_closure closure; - grpc_slice *slice; size_t max_size_hint; grpc_closure *on_complete; } next_action; @@ -445,8 +443,8 @@ struct grpc_chttp2_stream { uint32_t id; /** window available for us to send to peer, over or under the initial window - * size of the transport... ie: - * outgoing_window = outgoing_window_delta + transport.initial_window_size */ + * size of the transport... ie: + * outgoing_window = outgoing_window_delta + transport.initial_window_size */ int64_t outgoing_window_delta; /** things the upper layers would like to send */ grpc_metadata_batch *send_initial_metadata; @@ -473,9 +471,6 @@ struct grpc_chttp2_stream { grpc_transport_stream_stats *collecting_stats; grpc_transport_stream_stats stats; - /** number of streams that are currently being read */ - gpr_refcount active_streams; - /** Is this stream closed for writing. */ bool write_closed; /** Is this stream reading half-closed. */ @@ -499,7 +494,17 @@ struct grpc_chttp2_stream { grpc_chttp2_incoming_metadata_buffer metadata_buffer[2]; - grpc_chttp2_incoming_frame_queue incoming_frames; + grpc_slice_buffer frame_storage; /* protected by t combiner */ + + /* Accessed only by transport thread when stream->pending_byte_stream == false + * Accessed only by application thread when stream->pending_byte_stream == + * true */ + grpc_slice_buffer unprocessed_incoming_frames_buffer; + grpc_closure *on_next; /* protected by t combiner */ + bool pending_byte_stream; /* protected by t combiner */ + grpc_closure reset_byte_stream; + grpc_error *byte_stream_error; /* protected by t combiner */ + bool received_last_frame; /* protected by t combiner */ gpr_timespec deadline; @@ -512,6 +517,9 @@ struct grpc_chttp2_stream { * incoming_window = incoming_window_delta + transport.initial_window_size */ int64_t incoming_window_delta; /** parsing state for data frames */ + /* Accessed only by transport thread when stream->pending_byte_stream == false + * Accessed only by application thread when stream->pending_byte_stream == + * true */ grpc_chttp2_data_parser data_parser; /** number of bytes received - reset at end of parse thread execution */ int64_t received_bytes; @@ -790,10 +798,13 @@ void grpc_chttp2_ref_transport(grpc_chttp2_transport *t); grpc_chttp2_incoming_byte_stream *grpc_chttp2_incoming_byte_stream_create( grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, grpc_chttp2_stream *s, uint32_t frame_size, uint32_t flags); -void grpc_chttp2_incoming_byte_stream_push(grpc_exec_ctx *exec_ctx, - grpc_chttp2_incoming_byte_stream *bs, - grpc_slice slice); -void grpc_chttp2_incoming_byte_stream_finished( +grpc_error *grpc_chttp2_incoming_byte_stream_push( + grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs, + grpc_slice slice, grpc_slice *slice_out); +grpc_error *grpc_chttp2_incoming_byte_stream_finished( + grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs, + grpc_error *error, bool reset_on_error); +void grpc_chttp2_incoming_byte_stream_notify( grpc_exec_ctx *exec_ctx, grpc_chttp2_incoming_byte_stream *bs, grpc_error *error); diff --git a/src/core/ext/transport/chttp2/transport/parsing.c b/src/core/ext/transport/chttp2/transport/parsing.c index 7e457ced270..638b137316f 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.c +++ b/src/core/ext/transport/chttp2/transport/parsing.c @@ -458,12 +458,13 @@ static grpc_error *init_data_frame_parser(grpc_exec_ctx *exec_ctx, return init_skip_frame_parser(exec_ctx, t, 0); } if (err == GRPC_ERROR_NONE) { - err = grpc_chttp2_data_parser_begin_frame(&s->data_parser, - t->incoming_frame_flags, s->id); + err = grpc_chttp2_data_parser_begin_frame( + &s->data_parser, t->incoming_frame_flags, s->id, s); } error_handler: if (err == GRPC_ERROR_NONE) { t->incoming_stream = s; + /* t->parser = grpc_chttp2_data_parser_parse;*/ t->parser = grpc_chttp2_data_parser_parse; t->parser_data = &s->data_parser; return GRPC_ERROR_NONE; diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 88335ecd0be..7896c70f9eb 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -973,9 +973,20 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, grpc_slice_buffer write_slice_buffer; grpc_slice slice; grpc_slice_buffer_init(&write_slice_buffer); - grpc_byte_stream_next( - NULL, stream_op->payload->send_message.send_message, &slice, - stream_op->payload->send_message.send_message->length, NULL); + if (1 != grpc_byte_stream_next( + exec_ctx, stream_op->payload->send_message.send_message, + stream_op->payload->send_message.send_message->length, + NULL)) { + /* Should never reach here */ + GPR_ASSERT(false); + } + if (GRPC_ERROR_NONE != + grpc_byte_stream_pull(exec_ctx, + stream_op->payload->send_message.send_message, + &slice)) { + /* Should never reach here */ + GPR_ASSERT(false); + } grpc_slice_buffer_add(&write_slice_buffer, slice); if (write_slice_buffer.count != 1) { /* Empty request not handled yet */ diff --git a/src/core/lib/channel/compress_filter.c b/src/core/lib/channel/compress_filter.c index 4625cba0d25..764524b24d1 100644 --- a/src/core/lib/channel/compress_filter.c +++ b/src/core/lib/channel/compress_filter.c @@ -221,6 +221,13 @@ static void finish_send_message(grpc_exec_ctx *exec_ctx, static void got_slice(grpc_exec_ctx *exec_ctx, void *elemp, grpc_error *error) { grpc_call_element *elem = elemp; call_data *calld = elem->call_data; + if (GRPC_ERROR_NONE != + grpc_byte_stream_pull(exec_ctx, + calld->send_op->payload->send_message.send_message, + &calld->incoming_slice)) { + /* Should never reach here */ + abort(); + } grpc_slice_buffer_add(&calld->slices, calld->incoming_slice); if (calld->send_length == calld->slices.length) { finish_send_message(exec_ctx, elem); @@ -233,8 +240,11 @@ static void continue_send_message(grpc_exec_ctx *exec_ctx, grpc_call_element *elem) { call_data *calld = elem->call_data; while (grpc_byte_stream_next( - exec_ctx, calld->send_op->payload->send_message.send_message, - &calld->incoming_slice, ~(size_t)0, &calld->got_slice)) { + exec_ctx, calld->send_op->payload->send_message.send_message, ~(size_t)0, + &calld->got_slice)) { + grpc_byte_stream_pull(exec_ctx, + calld->send_op->payload->send_message.send_message, + &calld->incoming_slice); grpc_slice_buffer_add(&calld->slices, calld->incoming_slice); if (calld->send_length == calld->slices.length) { finish_send_message(exec_ctx, elem); diff --git a/src/core/lib/channel/http_client_filter.c b/src/core/lib/channel/http_client_filter.c index 4e47c5c658f..151fb9885de 100644 --- a/src/core/lib/channel/http_client_filter.c +++ b/src/core/lib/channel/http_client_filter.c @@ -220,8 +220,11 @@ static void continue_send_message(grpc_exec_ctx *exec_ctx, call_data *calld = elem->call_data; uint8_t *wrptr = calld->payload_bytes; while (grpc_byte_stream_next( - exec_ctx, calld->send_op->payload->send_message.send_message, - &calld->incoming_slice, ~(size_t)0, &calld->got_slice)) { + exec_ctx, calld->send_op->payload->send_message.send_message, ~(size_t)0, + &calld->got_slice)) { + grpc_byte_stream_pull(exec_ctx, + calld->send_op->payload->send_message.send_message, + &calld->incoming_slice); memcpy(wrptr, GRPC_SLICE_START_PTR(calld->incoming_slice), GRPC_SLICE_LENGTH(calld->incoming_slice)); wrptr += GRPC_SLICE_LENGTH(calld->incoming_slice); @@ -237,6 +240,13 @@ static void got_slice(grpc_exec_ctx *exec_ctx, void *elemp, grpc_error *error) { grpc_call_element *elem = elemp; call_data *calld = elem->call_data; calld->send_message_blocked = false; + if (GRPC_ERROR_NONE != + grpc_byte_stream_pull(exec_ctx, + calld->send_op->payload->send_message.send_message, + &calld->incoming_slice)) { + /* Should never reach here */ + abort(); + } grpc_slice_buffer_add(&calld->slices, calld->incoming_slice); if (calld->send_length == calld->slices.length) { /* Pass down the original send_message op that was blocked.*/ diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c index 97d50a91be5..3e96d097985 100644 --- a/src/core/lib/surface/call.c +++ b/src/core/lib/surface/call.c @@ -1187,6 +1187,7 @@ static void finish_batch_step(grpc_exec_ctx *exec_ctx, batch_control *bctl) { static void continue_receiving_slices(grpc_exec_ctx *exec_ctx, batch_control *bctl) { + grpc_error *error; grpc_call *call = bctl->call; for (;;) { size_t remaining = call->receiving_stream->length - @@ -1198,11 +1199,22 @@ static void continue_receiving_slices(grpc_exec_ctx *exec_ctx, finish_batch_step(exec_ctx, bctl); return; } - if (grpc_byte_stream_next(exec_ctx, call->receiving_stream, - &call->receiving_slice, remaining, + if (grpc_byte_stream_next(exec_ctx, call->receiving_stream, remaining, &call->receiving_slice_ready)) { - grpc_slice_buffer_add(&(*call->receiving_buffer)->data.raw.slice_buffer, - call->receiving_slice); + error = grpc_byte_stream_pull(exec_ctx, call->receiving_stream, + &call->receiving_slice); + if (error == GRPC_ERROR_NONE) { + grpc_slice_buffer_add(&(*call->receiving_buffer)->data.raw.slice_buffer, + call->receiving_slice); + } else { + grpc_byte_stream_destroy(exec_ctx, call->receiving_stream); + call->receiving_stream = NULL; + grpc_byte_buffer_destroy(*call->receiving_buffer); + *call->receiving_buffer = NULL; + call->receiving_message = 0; + finish_batch_step(exec_ctx, bctl); + return; + } } else { return; } @@ -1213,12 +1225,24 @@ static void receiving_slice_ready(grpc_exec_ctx *exec_ctx, void *bctlp, grpc_error *error) { batch_control *bctl = bctlp; grpc_call *call = bctl->call; + grpc_byte_stream *bs = call->receiving_stream; + bool release_error = false; if (error == GRPC_ERROR_NONE) { - grpc_slice_buffer_add(&(*call->receiving_buffer)->data.raw.slice_buffer, - call->receiving_slice); - continue_receiving_slices(exec_ctx, bctl); - } else { + grpc_slice slice; + error = grpc_byte_stream_pull(exec_ctx, bs, &slice); + if (error == GRPC_ERROR_NONE) { + grpc_slice_buffer_add(&(*call->receiving_buffer)->data.raw.slice_buffer, + slice); + continue_receiving_slices(exec_ctx, bctl); + } else { + /* Error returned by grpc_byte_stream_pull needs to be released manually + */ + release_error = true; + } + } + + if (error != GRPC_ERROR_NONE) { if (grpc_trace_operation_failures) { GRPC_LOG_IF_ERROR("receiving_slice_ready", GRPC_ERROR_REF(error)); } @@ -1226,7 +1250,11 @@ static void receiving_slice_ready(grpc_exec_ctx *exec_ctx, void *bctlp, call->receiving_stream = NULL; grpc_byte_buffer_destroy(*call->receiving_buffer); *call->receiving_buffer = NULL; + call->receiving_message = 0; finish_batch_step(exec_ctx, bctl); + if (release_error) { + GRPC_ERROR_UNREF(error); + } } } diff --git a/src/core/lib/transport/byte_stream.c b/src/core/lib/transport/byte_stream.c index 4d4206189e7..5800c70ef44 100644 --- a/src/core/lib/transport/byte_stream.c +++ b/src/core/lib/transport/byte_stream.c @@ -40,10 +40,15 @@ #include "src/core/lib/slice/slice_internal.h" int grpc_byte_stream_next(grpc_exec_ctx *exec_ctx, - grpc_byte_stream *byte_stream, grpc_slice *slice, - size_t max_size_hint, grpc_closure *on_complete) { - return byte_stream->next(exec_ctx, byte_stream, slice, max_size_hint, - on_complete); + grpc_byte_stream *byte_stream, size_t max_size_hint, + grpc_closure *on_complete) { + return byte_stream->next(exec_ctx, byte_stream, max_size_hint, on_complete); +} + +grpc_error *grpc_byte_stream_pull(grpc_exec_ctx *exec_ctx, + grpc_byte_stream *byte_stream, + grpc_slice *slice) { + return byte_stream->pull(exec_ctx, byte_stream, slice); } void grpc_byte_stream_destroy(grpc_exec_ctx *exec_ctx, @@ -53,16 +58,24 @@ void grpc_byte_stream_destroy(grpc_exec_ctx *exec_ctx, /* slice_buffer_stream */ -static int slice_buffer_stream_next(grpc_exec_ctx *exec_ctx, - grpc_byte_stream *byte_stream, - grpc_slice *slice, size_t max_size_hint, - grpc_closure *on_complete) { +static bool slice_buffer_stream_next(grpc_exec_ctx *exec_ctx, + grpc_byte_stream *byte_stream, + size_t max_size_hint, + grpc_closure *on_complete) { + grpc_slice_buffer_stream *stream = (grpc_slice_buffer_stream *)byte_stream; + GPR_ASSERT(stream->cursor < stream->backing_buffer->count); + return true; +} + +static grpc_error *slice_buffer_stream_pull(grpc_exec_ctx *exec_ctx, + grpc_byte_stream *byte_stream, + grpc_slice *slice) { grpc_slice_buffer_stream *stream = (grpc_slice_buffer_stream *)byte_stream; GPR_ASSERT(stream->cursor < stream->backing_buffer->count); *slice = grpc_slice_ref_internal(stream->backing_buffer->slices[stream->cursor]); stream->cursor++; - return 1; + return GRPC_ERROR_NONE; } static void slice_buffer_stream_destroy(grpc_exec_ctx *exec_ctx, @@ -75,6 +88,7 @@ void grpc_slice_buffer_stream_init(grpc_slice_buffer_stream *stream, stream->base.length = (uint32_t)slice_buffer->length; stream->base.flags = flags; stream->base.next = slice_buffer_stream_next; + stream->base.pull = slice_buffer_stream_pull; stream->base.destroy = slice_buffer_stream_destroy; stream->backing_buffer = slice_buffer; stream->cursor = 0; diff --git a/src/core/lib/transport/byte_stream.h b/src/core/lib/transport/byte_stream.h index 1fdd5b4d775..381c65fb044 100644 --- a/src/core/lib/transport/byte_stream.h +++ b/src/core/lib/transport/byte_stream.h @@ -49,9 +49,10 @@ typedef struct grpc_byte_stream grpc_byte_stream; struct grpc_byte_stream { uint32_t length; uint32_t flags; - int (*next)(grpc_exec_ctx *exec_ctx, grpc_byte_stream *byte_stream, - grpc_slice *slice, size_t max_size_hint, - grpc_closure *on_complete); + bool (*next)(grpc_exec_ctx *exec_ctx, grpc_byte_stream *byte_stream, + size_t max_size_hint, grpc_closure *on_complete); + grpc_error *(*pull)(grpc_exec_ctx *exec_ctx, grpc_byte_stream *byte_stream, + grpc_slice *slice); void (*destroy)(grpc_exec_ctx *exec_ctx, grpc_byte_stream *byte_stream); }; @@ -61,12 +62,20 @@ struct grpc_byte_stream { * * max_size_hint can be set as a hint as to the maximum number * of bytes that would be acceptable to read. + */ +int grpc_byte_stream_next(grpc_exec_ctx *exec_ctx, + grpc_byte_stream *byte_stream, size_t max_size_hint, + grpc_closure *on_complete); + +/* returns the next slice in the byte stream when it is ready (indicated by + * either grpc_byte_stream_next returning 1 or on_complete passed to + * grpc_byte_stream_next is called). * * once a slice is returned into *slice, it is owned by the caller. */ -int grpc_byte_stream_next(grpc_exec_ctx *exec_ctx, - grpc_byte_stream *byte_stream, grpc_slice *slice, - size_t max_size_hint, grpc_closure *on_complete); +grpc_error *grpc_byte_stream_pull(grpc_exec_ctx *exec_ctx, + grpc_byte_stream *byte_stream, + grpc_slice *slice); void grpc_byte_stream_destroy(grpc_exec_ctx *exec_ctx, grpc_byte_stream *byte_stream); diff --git a/test/cpp/microbenchmarks/bm_chttp2_transport.cc b/test/cpp/microbenchmarks/bm_chttp2_transport.cc index c89f349ca7f..8c5413b5fda 100644 --- a/test/cpp/microbenchmarks/bm_chttp2_transport.cc +++ b/test/cpp/microbenchmarks/bm_chttp2_transport.cc @@ -569,12 +569,17 @@ static void BM_TransportStreamRecv(benchmark::State &state) { grpc_closure_sched(exec_ctx, c.get(), GRPC_ERROR_NONE); return; } - } while (grpc_byte_stream_next(exec_ctx, recv_stream, &recv_slice, + } while (grpc_byte_stream_next(exec_ctx, recv_stream, recv_stream->length - received, - drain_continue.get())); + drain_continue.get()) && + GRPC_ERROR_NONE == + grpc_byte_stream_pull(exec_ctx, recv_stream, &recv_slice) && + (received += GRPC_SLICE_LENGTH(recv_slice), + grpc_slice_unref_internal(exec_ctx, recv_slice), true)); }); drain_continue = MakeClosure([&](grpc_exec_ctx *exec_ctx, grpc_error *error) { + grpc_byte_stream_pull(exec_ctx, recv_stream, &recv_slice); received += GRPC_SLICE_LENGTH(recv_slice); grpc_slice_unref_internal(exec_ctx, recv_slice); grpc_closure_run(exec_ctx, drain.get(), GRPC_ERROR_NONE); From f03607658a00f63e16363433466b6675efb4ca46 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 12 Apr 2017 20:37:40 -0700 Subject: [PATCH 31/60] Fix flow control problem --- .../ext/transport/chttp2/transport/chttp2_transport.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 29ed4bf90e5..16cf40c2c96 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1441,12 +1441,19 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, } if (op->recv_message) { + size_t already_received; GPR_ASSERT(s->recv_message_ready == NULL); + GPR_ASSERT(!s->pending_byte_stream); s->recv_message_ready = op_payload->recv_message.recv_message_ready; s->recv_message = op_payload->recv_message.recv_message; - if (s->id != 0 && s->frame_storage.length == 0) { - incoming_byte_stream_update_flow_control(exec_ctx, t, s, 5, 0); + if (s->pending_byte_stream) { + already_received = s->frame_storage.length; + } else { + already_received = s->frame_storage.length + + s->unprocessed_incoming_frames_buffer.length; } + incoming_byte_stream_update_flow_control(exec_ctx, t, s, 5, + already_received); grpc_chttp2_maybe_complete_recv_message(exec_ctx, t, s); } From 59bf91c72e4f8d249a004e4ac7879bec27be8bfd Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 13 Apr 2017 08:43:51 -0700 Subject: [PATCH 32/60] tweaks, fix bug --- tools/profiling/microbenchmarks/bm_diff.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 5bc0eacd22c..4fa4869b642 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -94,8 +94,8 @@ argp.add_argument('-t', '--track', help='Which metrics to track') argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) -argp.add_argument('-r', '--repetitions', type=int, default=5) -argp.add_argument('-l', '--loops', type=int, default=5) +argp.add_argument('-r', '--repetitions', type=int, default=3) +argp.add_argument('-l', '--loops', type=int, default=7) argp.add_argument('-p', '--p_threshold', type=float, default=0.01) argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count()) args = argp.parse_args() @@ -190,7 +190,7 @@ class Benchmark: benchmarks = collections.defaultdict(Benchmark) for bm in args.benchmarks: - for loop in args.loops: + for loop in range(0, args.loops): with open('%s.counters.new.%d.json' % (bm, loop)) as f: js_new_ctr = json.loads(f.read()) with open('%s.opt.new.%d.json' % (bm, loop)) as f: From dd475487c20d8ae44fc5bb36afc50592807b44de Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 13 Apr 2017 08:54:22 -0700 Subject: [PATCH 33/60] Minor patch --- .../chttp2/transport/chttp2_transport.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 16cf40c2c96..e2f7bb433a4 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1446,14 +1446,16 @@ static void perform_stream_op_locked(grpc_exec_ctx *exec_ctx, void *stream_op, GPR_ASSERT(!s->pending_byte_stream); s->recv_message_ready = op_payload->recv_message.recv_message_ready; s->recv_message = op_payload->recv_message.recv_message; - if (s->pending_byte_stream) { - already_received = s->frame_storage.length; - } else { - already_received = s->frame_storage.length + - s->unprocessed_incoming_frames_buffer.length; + if (s->id != 0) { + if (s->pending_byte_stream) { + already_received = s->frame_storage.length; + } else { + already_received = s->frame_storage.length + + s->unprocessed_incoming_frames_buffer.length; + } + incoming_byte_stream_update_flow_control(exec_ctx, t, s, 5, + already_received); } - incoming_byte_stream_update_flow_control(exec_ctx, t, s, 5, - already_received); grpc_chttp2_maybe_complete_recv_message(exec_ctx, t, s); } From 6163da513578c2526c3b1de16a3deb8c913e2c34 Mon Sep 17 00:00:00 2001 From: yang-g Date: Thu, 13 Apr 2017 09:39:42 -0700 Subject: [PATCH 34/60] Disallow empty method slice in api_fuzzer --- test/core/end2end/fuzzers/api_fuzzer.c | 3 +++ .../clusterfuzz-testcase-5867145026076672 | Bin 0 -> 46 bytes tools/run_tests/generated/tests.json | 23 ++++++++++++++++++ 3 files changed, 26 insertions(+) create mode 100644 test/core/end2end/fuzzers/api_fuzzer_corpus/clusterfuzz-testcase-5867145026076672 diff --git a/test/core/end2end/fuzzers/api_fuzzer.c b/test/core/end2end/fuzzers/api_fuzzer.c index a0acf5bf602..cd983a70a65 100644 --- a/test/core/end2end/fuzzers/api_fuzzer.c +++ b/test/core/end2end/fuzzers/api_fuzzer.c @@ -932,6 +932,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { } uint32_t propagation_mask = read_uint32(&inp); grpc_slice method = read_string_like_slice(&inp); + if (GRPC_SLICE_LENGTH(method) == 0) { + ok = false; + } grpc_slice host = read_string_like_slice(&inp); gpr_timespec deadline = gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), diff --git a/test/core/end2end/fuzzers/api_fuzzer_corpus/clusterfuzz-testcase-5867145026076672 b/test/core/end2end/fuzzers/api_fuzzer_corpus/clusterfuzz-testcase-5867145026076672 new file mode 100644 index 0000000000000000000000000000000000000000..3fd5427b46662d86534978f322d512466c8765af GIT binary patch literal 46 wcmZQ7PAw`+En;9`Vc<$+Wc Date: Thu, 13 Apr 2017 10:29:38 -0700 Subject: [PATCH 35/60] added env var to the Cronet target --- src/objective-c/tests/Tests.xcodeproj/project.pbxproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/objective-c/tests/Tests.xcodeproj/project.pbxproj b/src/objective-c/tests/Tests.xcodeproj/project.pbxproj index f838f4c2214..b01d5ffceaa 100644 --- a/src/objective-c/tests/Tests.xcodeproj/project.pbxproj +++ b/src/objective-c/tests/Tests.xcodeproj/project.pbxproj @@ -1568,6 +1568,7 @@ GCC_PREPROCESSOR_DEFINITIONS = ( "DEBUG=1", "$(inherited)", + "HOST_PORT_REMOTE=$(HOST_PORT_REMOTE)", ); GCC_SYMBOLS_PRIVATE_EXTERN = NO; GCC_TREAT_WARNINGS_AS_ERRORS = YES; From 956f70098eced839f590b9dded76939f6c0b9fd0 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Thu, 13 Apr 2017 15:40:06 -0700 Subject: [PATCH 36/60] Keep LB policy alive during high freq of resolver updates --- .../filters/client_channel/client_channel.c | 77 ++++++++++++++++--- 1 file changed, 66 insertions(+), 11 deletions(-) diff --git a/src/core/ext/filters/client_channel/client_channel.c b/src/core/ext/filters/client_channel/client_channel.c index 83e3b8f118d..e446f0543a8 100644 --- a/src/core/ext/filters/client_channel/client_channel.c +++ b/src/core/ext/filters/client_channel/client_channel.c @@ -236,14 +236,19 @@ static void set_channel_connectivity_state_locked(grpc_exec_ctx *exec_ctx, grpc_connectivity_state state, grpc_error *error, const char *reason) { - if ((state == GRPC_CHANNEL_TRANSIENT_FAILURE || - state == GRPC_CHANNEL_SHUTDOWN) && - chand->lb_policy != NULL) { - /* cancel picks with wait_for_ready=false */ - grpc_lb_policy_cancel_picks_locked( - exec_ctx, chand->lb_policy, - /* mask= */ GRPC_INITIAL_METADATA_WAIT_FOR_READY, - /* check= */ 0, GRPC_ERROR_REF(error)); + if (chand->lb_policy != NULL) { + if (state == GRPC_CHANNEL_TRANSIENT_FAILURE) { + /* cancel picks with wait_for_ready=false */ + grpc_lb_policy_cancel_picks_locked( + exec_ctx, chand->lb_policy, + /* mask= */ GRPC_INITIAL_METADATA_WAIT_FOR_READY, + /* check= */ 0, GRPC_ERROR_REF(error)); + } else if (state == GRPC_CHANNEL_SHUTDOWN) { + /* cancel all picks */ + grpc_lb_policy_cancel_picks_locked(exec_ctx, chand->lb_policy, + /* mask= */ 0, /* check= */ 0, + GRPC_ERROR_REF(error)); + } } grpc_connectivity_state_set(exec_ctx, &chand->state_tracker, state, error, reason); @@ -346,6 +351,37 @@ static void parse_retry_throttle_params(const grpc_json *field, void *arg) { } } +// Wrap a closure associated with \a lb_policy. The associated callback (\a +// wrapped_on_pick_closure_cb) is responsible for unref'ing \a lb_policy after +// scheduling \a wrapped_closure. +typedef struct wrapped_on_pick_closure_arg { + /* the closure instance using this struct as argument */ + grpc_closure wrapper_closure; + + /* the original closure. Usually a on_complete/notify cb for pick() and ping() + * calls against the internal RR instance, respectively. */ + grpc_closure *wrapped_closure; + + /* The policy instance related to the closure */ + grpc_lb_policy *lb_policy; + + /* heap memory to be freed upon closure execution. Usually this arg. */ + void *free_when_done; +} wrapped_on_pick_closure_arg; + +// Invoke \a arg->wrapped_closure, unref \a arg->lb_policy and free +// arg->free_when_done (usually \a arg itself). +static void wrapped_on_pick_closure_cb(grpc_exec_ctx *exec_ctx, void *arg, + grpc_error *error) { + wrapped_on_pick_closure_arg *wc_arg = arg; + GPR_ASSERT(wc_arg->wrapped_closure != NULL); + GPR_ASSERT(wc_arg->lb_policy != NULL); + GPR_ASSERT(wc_arg->free_when_done != NULL); + grpc_closure_sched(exec_ctx, wc_arg->wrapped_closure, GRPC_ERROR_REF(error)); + GRPC_LB_POLICY_UNREF(exec_ctx, wc_arg->lb_policy, "pick_subchannel_wrapping"); + gpr_free(wc_arg->free_when_done); +} + static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { channel_data *chand = arg; @@ -1031,11 +1067,30 @@ static bool pick_subchannel_locked( const grpc_lb_policy_pick_args inputs = { initial_metadata, initial_metadata_flags, &calld->lb_token_mdelem, gpr_inf_future(GPR_CLOCK_MONOTONIC)}; - const bool result = grpc_lb_policy_pick_locked( - exec_ctx, lb_policy, &inputs, connected_subchannel, NULL, on_ready); + + // Wrap the user-provided callback in order to hold a strong reference to + // the LB policy for the duration of the pick. + wrapped_on_pick_closure_arg *w_on_pick_arg = + gpr_zalloc(sizeof(*w_on_pick_arg)); + grpc_closure_init(&w_on_pick_arg->wrapper_closure, + wrapped_on_pick_closure_cb, w_on_pick_arg, + grpc_schedule_on_exec_ctx); + w_on_pick_arg->wrapped_closure = on_ready; + GRPC_LB_POLICY_REF(lb_policy, "pick_subchannel_wrapping"); + w_on_pick_arg->lb_policy = lb_policy; + w_on_pick_arg->free_when_done = w_on_pick_arg; + const bool pick_done = grpc_lb_policy_pick_locked( + exec_ctx, lb_policy, &inputs, connected_subchannel, NULL, + &w_on_pick_arg->wrapper_closure); + if (pick_done) { + /* synchronous grpc_lb_policy_pick call. Unref the LB policy. */ + GRPC_LB_POLICY_UNREF(exec_ctx, w_on_pick_arg->lb_policy, + "pick_subchannel_wrapping"); + gpr_free(w_on_pick_arg->free_when_done); + } GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "pick_subchannel"); GPR_TIMER_END("pick_subchannel", 0); - return result; + return pick_done; } if (chand->resolver != NULL && !chand->started_resolving) { chand->started_resolving = true; From 964647729e22d3e2fddf11409191e61fe8d1a971 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 13 Apr 2017 22:06:25 -0700 Subject: [PATCH 37/60] Update bm_diff.py --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 4fa4869b642..4f21a76c610 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -176,7 +176,7 @@ class Benchmark: mdn_diff = abs(median(new) - median(old)) print '%s: new=%r old=%r mdn_diff=%r' % (f, new, old, mdn_diff) s = speedup.speedup(new, old) - if s and mdn_diff > 0.5: + if abs(s) > 3 and mdn_diff > 0.5: self.final[f] = '%+d%%' % s return self.final.keys() From 233ad390fa30ff90d927dd29f277a914a0cf0621 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 08:25:53 -0700 Subject: [PATCH 38/60] Accelerate by running more jobs in parallel, and only building twice --- tools/profiling/microbenchmarks/bm_diff.py | 41 ++++++++++++---------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 4f21a76c610..d1d49037cb3 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -44,6 +44,7 @@ import comment_on_pr import jobset import itertools import speedup +import random _INTERESTING = ( 'cpu_time', @@ -114,7 +115,7 @@ def make_cmd(cfg): return ['make'] + args.benchmarks + [ 'CONFIG=%s' % cfg, '-j', '%d' % args.jobs] -def build(): +def build(dest): subprocess.check_call(['git', 'submodule', 'update']) try: subprocess.check_call(make_cmd('opt')) @@ -123,6 +124,7 @@ def build(): subprocess.check_call(['make', 'clean']) subprocess.check_call(make_cmd('opt')) subprocess.check_call(make_cmd('counters')) + os.rename('bin', dest) def collect1(bm, cfg, ver, idx): cmd = ['bins/%s/%s' % (cfg, bm), @@ -130,29 +132,30 @@ def collect1(bm, cfg, ver, idx): '--benchmark_out_format=json', '--benchmark_repetitions=%d' % (args.repetitions) ] - return jobset.JobSpec(cmd, shortname='%s %s %s' % (bm, cfg, ver), + return jobset.JobSpec(cmd, shortname='%s %s %s %d/%d' % (bm, cfg, ver, idx+1, args.loops), verbose_success=True, timeout_seconds=None) -for loop in range(0, args.loops): - build() - jobset.run(itertools.chain( - (collect1(bm, 'opt', 'new', loop) for bm in args.benchmarks), - (collect1(bm, 'counters', 'new', loop) for bm in args.benchmarks), - ), maxjobs=args.jobs) +build('new') - where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() - subprocess.check_call(['git', 'checkout', args.diff_base]) +where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() +subprocess.check_call(['git', 'checkout', args.diff_base]) - try: - build() - jobset.run(itertools.chain( - (collect1(bm, 'opt', 'old', loop) for bm in args.benchmarks), - (collect1(bm, 'counters', 'old', loop) for bm in args.benchmarks), - ), maxjobs=args.jobs) - finally: - subprocess.check_call(['git', 'checkout', where_am_i]) - subprocess.check_call(['git', 'submodule', 'update']) +try: + build('old') +finally: + subprocess.check_call(['git', 'checkout', where_am_i]) + subprocess.check_call(['git', 'submodule', 'update']) + +for loop in args.loops: + jobs.extend(x for x in itertools.chain( + (collect1(bm, 'opt', 'new', loop) for bm in args.benchmarks), + (collect1(bm, 'counters', 'new', loop) for bm in args.benchmarks), + (collect1(bm, 'opt', 'old', loop) for bm in args.benchmarks), + (collect1(bm, 'counters', 'old', loop) for bm in args.benchmarks), + )) +random.shuffle(jobs) +jobset.run(jobs, maxjobs=args.jobs) class Benchmark: From a00e51ccfee3ca6877ef568c8a5d65867ce974ca Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 08:27:39 -0700 Subject: [PATCH 39/60] Fix dir names --- tools/profiling/microbenchmarks/bm_diff.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index d1d49037cb3..634e2814466 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -124,10 +124,10 @@ def build(dest): subprocess.check_call(['make', 'clean']) subprocess.check_call(make_cmd('opt')) subprocess.check_call(make_cmd('counters')) - os.rename('bin', dest) + os.rename('bins', dest) def collect1(bm, cfg, ver, idx): - cmd = ['bins/%s/%s' % (cfg, bm), + cmd = ['%s/%s/%s' % (ver, cfg, bm), '--benchmark_out=%s.%s.%s.%d.json' % (bm, cfg, ver, idx), '--benchmark_out_format=json', '--benchmark_repetitions=%d' % (args.repetitions) @@ -139,7 +139,6 @@ build('new') where_am_i = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() subprocess.check_call(['git', 'checkout', args.diff_base]) - try: build('old') finally: From 0553b9c75d87394ebddb53722e11acaec2305810 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 08:31:17 -0700 Subject: [PATCH 40/60] Properly clean --- tools/profiling/microbenchmarks/bm_diff.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 634e2814466..d22a97fd1ea 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -45,6 +45,7 @@ import jobset import itertools import speedup import random +import shutil _INTERESTING = ( 'cpu_time', @@ -116,6 +117,7 @@ def make_cmd(cfg): 'CONFIG=%s' % cfg, '-j', '%d' % args.jobs] def build(dest): + shutil.rmtree('bm_diff_%s' % dest, ignore_errors=True) subprocess.check_call(['git', 'submodule', 'update']) try: subprocess.check_call(make_cmd('opt')) @@ -124,10 +126,10 @@ def build(dest): subprocess.check_call(['make', 'clean']) subprocess.check_call(make_cmd('opt')) subprocess.check_call(make_cmd('counters')) - os.rename('bins', dest) + os.rename('bins', 'bm_diff_%s' % dest) def collect1(bm, cfg, ver, idx): - cmd = ['%s/%s/%s' % (ver, cfg, bm), + cmd = ['bm_diff_%s/%s/%s' % (ver, cfg, bm), '--benchmark_out=%s.%s.%s.%d.json' % (bm, cfg, ver, idx), '--benchmark_out_format=json', '--benchmark_repetitions=%d' % (args.repetitions) From 0ebff8b2137f799b1beaebf89835b060c53461a3 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 08:32:38 -0700 Subject: [PATCH 41/60] Iterate properly --- tools/profiling/microbenchmarks/bm_diff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index d22a97fd1ea..07c145e4e48 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -147,7 +147,7 @@ finally: subprocess.check_call(['git', 'checkout', where_am_i]) subprocess.check_call(['git', 'submodule', 'update']) -for loop in args.loops: +for loop in range(0, args.loops): jobs.extend(x for x in itertools.chain( (collect1(bm, 'opt', 'new', loop) for bm in args.benchmarks), (collect1(bm, 'counters', 'new', loop) for bm in args.benchmarks), From bc442e1ff0e2fd8107d41219c3c52472324efa77 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 08:33:55 -0700 Subject: [PATCH 42/60] Declare jobs --- tools/profiling/microbenchmarks/bm_diff.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 07c145e4e48..a6e77b375f1 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -147,6 +147,7 @@ finally: subprocess.check_call(['git', 'checkout', where_am_i]) subprocess.check_call(['git', 'submodule', 'update']) +jobs = [] for loop in range(0, args.loops): jobs.extend(x for x in itertools.chain( (collect1(bm, 'opt', 'new', loop) for bm in args.benchmarks), From d71ba8392aadb43eb015270499c711a4221ddd95 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 08:36:11 -0700 Subject: [PATCH 43/60] Tweak defaults --- tools/profiling/microbenchmarks/bm_diff.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index a6e77b375f1..bd0da57598f 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -96,8 +96,8 @@ argp.add_argument('-t', '--track', help='Which metrics to track') argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) -argp.add_argument('-r', '--repetitions', type=int, default=3) -argp.add_argument('-l', '--loops', type=int, default=7) +argp.add_argument('-r', '--repetitions', type=int, default=1) +argp.add_argument('-l', '--loops', type=int, default=30) argp.add_argument('-p', '--p_threshold', type=float, default=0.01) argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count()) args = argp.parse_args() From 13c0e6e549175f29cb2d22bd3b2f7b00e5c2e94c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 08:40:04 -0700 Subject: [PATCH 44/60] Update .gitignore --- .gitignore | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.gitignore b/.gitignore index ad3ec64ab5a..5e38f5fa01b 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,13 @@ gdb.txt # ctags file tags + +# perf data +perf.data +perf.data.old + +# bm_diff +bm_diff_new/ +bm_diff_old/ +bm_*.json + From 97e40da5a1219ccd838e0818d060b217c0830ae5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 09:04:07 -0700 Subject: [PATCH 45/60] Less samples, more random --- tools/profiling/microbenchmarks/bm_diff.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index bd0da57598f..3d4fa2a96a5 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -97,8 +97,7 @@ argp.add_argument('-t', '--track', argp.add_argument('-b', '--benchmarks', nargs='+', choices=_AVAILABLE_BENCHMARK_TESTS, default=['bm_cq']) argp.add_argument('-d', '--diff_base', type=str) argp.add_argument('-r', '--repetitions', type=int, default=1) -argp.add_argument('-l', '--loops', type=int, default=30) -argp.add_argument('-p', '--p_threshold', type=float, default=0.01) +argp.add_argument('-l', '--loops', type=int, default=12) argp.add_argument('-j', '--jobs', type=int, default=multiprocessing.cpu_count()) args = argp.parse_args() @@ -155,7 +154,7 @@ for loop in range(0, args.loops): (collect1(bm, 'opt', 'old', loop) for bm in args.benchmarks), (collect1(bm, 'counters', 'old', loop) for bm in args.benchmarks), )) -random.shuffle(jobs) +random.shuffle(jobs, random.SystemRandom().random) jobset.run(jobs, maxjobs=args.jobs) From eafeea48a981d799c2c893102b6f0cbe25163a7f Mon Sep 17 00:00:00 2001 From: yang-g Date: Fri, 14 Apr 2017 12:34:10 -0700 Subject: [PATCH 46/60] Update ServerGlobalCallback::AddPort api --- include/grpc++/server.h | 3 ++- src/cpp/server/server_cc.cc | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/include/grpc++/server.h b/include/grpc++/server.h index 489937712ec..325bd138c7e 100644 --- a/include/grpc++/server.h +++ b/include/grpc++/server.h @@ -89,7 +89,8 @@ class Server final : public ServerInterface, private GrpcLibraryCodegen { /// Called before server is started. virtual void PreServerStart(Server* server) {} /// Called after a server port is added. - virtual void AddPort(Server* server, int port) {} + virtual void AddPort(Server* server, const grpc::string& addr, + ServerCredentials* creds, int port) {} }; /// Set the global callback object. Can only be called once. Does not take /// ownership of callbacks, and expects the pointed to object to be alive diff --git a/src/cpp/server/server_cc.cc b/src/cpp/server/server_cc.cc index ce173a1ee2d..325f8d3c6f1 100644 --- a/src/cpp/server/server_cc.cc +++ b/src/cpp/server/server_cc.cc @@ -490,7 +490,7 @@ int Server::AddListeningPort(const grpc::string& addr, ServerCredentials* creds) { GPR_ASSERT(!started_); int port = creds->AddPortToServer(addr, server_); - global_callbacks_->AddPort(this, port); + global_callbacks_->AddPort(this, addr, creds, port); return port; } From 349907b6459b639a937130f4de23e785cbf289d1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 13:00:47 -0700 Subject: [PATCH 47/60] Handle EINTR on reading file --- tools/profiling/microbenchmarks/bm_diff.py | 25 +++++++++++++++------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tools/profiling/microbenchmarks/bm_diff.py b/tools/profiling/microbenchmarks/bm_diff.py index 3d4fa2a96a5..ba0c225f6cc 100755 --- a/tools/profiling/microbenchmarks/bm_diff.py +++ b/tools/profiling/microbenchmarks/bm_diff.py @@ -46,6 +46,7 @@ import itertools import speedup import random import shutil +import errno _INTERESTING = ( 'cpu_time', @@ -191,18 +192,26 @@ class Benchmark: return [self.final[f] if f in self.final else '' for f in flds] +def read_file(filename): + while True: + try: + with open(filename) as f: + return f.read() + except IOError, e: + if e.errno != errno.EINTR: + raise + +def read_json(filename): + return json.loads(read_file(filename)) + benchmarks = collections.defaultdict(Benchmark) for bm in args.benchmarks: for loop in range(0, args.loops): - with open('%s.counters.new.%d.json' % (bm, loop)) as f: - js_new_ctr = json.loads(f.read()) - with open('%s.opt.new.%d.json' % (bm, loop)) as f: - js_new_opt = json.loads(f.read()) - with open('%s.counters.old.%d.json' % (bm, loop)) as f: - js_old_ctr = json.loads(f.read()) - with open('%s.opt.old.%d.json' % (bm, loop)) as f: - js_old_opt = json.loads(f.read()) + js_new_ctr = read_json('%s.counters.new.%d.json' % (bm, loop)) + js_new_opt = read_json('%s.opt.new.%d.json' % (bm, loop)) + js_old_ctr = read_json('%s.counters.old.%d.json' % (bm, loop)) + js_old_opt = read_json('%s.opt.old.%d.json' % (bm, loop)) for row in bm_json.expand_json(js_new_ctr, js_new_opt): print row From 2f4679ea348637cd5882ace228f2dc6bc0649d4c Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Fri, 14 Apr 2017 13:06:27 -0700 Subject: [PATCH 48/60] Fix handling of slice in http_client_filter --- include/grpc/slice.h | 6 +++--- src/core/ext/filters/http/client/http_client_filter.c | 4 +--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/include/grpc/slice.h b/include/grpc/slice.h index ea66e094e98..8c32a30d762 100644 --- a/include/grpc/slice.h +++ b/include/grpc/slice.h @@ -102,9 +102,9 @@ GPRAPI grpc_slice grpc_slice_from_static_string(const char *source); /* Create a slice pointing to constant memory */ GPRAPI grpc_slice grpc_slice_from_static_buffer(const void *source, size_t len); -/* Return a result slice derived from s, which shares a ref count with s, where - result.data==s.data+begin, and result.length==end-begin. - The ref count of s is increased by one. +/* Return a result slice derived from s, which shares a ref count with \a s, + where result.data==s.data+begin, and result.length==end-begin. The ref count + of \a s is increased by one. Do not assign result back to \a s. Requires s initialized, begin <= end, begin <= s.length, and end <= source->length. */ GPRAPI grpc_slice grpc_slice_sub(grpc_slice s, size_t begin, size_t end); diff --git a/src/core/ext/filters/http/client/http_client_filter.c b/src/core/ext/filters/http/client/http_client_filter.c index b896c7e7bba..7bab7b1462d 100644 --- a/src/core/ext/filters/http/client/http_client_filter.c +++ b/src/core/ext/filters/http/client/http_client_filter.c @@ -313,7 +313,6 @@ static grpc_error *hc_mutate_op(grpc_exec_ctx *exec_ctx, estimated_len += grpc_base64_estimate_encoded_size( op->payload->send_message.send_message->length, k_url_safe, k_multi_line); - estimated_len += 1; /* for the trailing 0 */ grpc_slice path_with_query_slice = grpc_slice_malloc(estimated_len); /* memcopy individual pieces into this slice */ @@ -335,7 +334,7 @@ static grpc_error *hc_mutate_op(grpc_exec_ctx *exec_ctx, char *t = (char *)GRPC_SLICE_START_PTR(path_with_query_slice); /* safe to use strlen since base64_encode will always add '\0' */ path_with_query_slice = - grpc_slice_sub(path_with_query_slice, 0, strlen(t)); + grpc_slice_sub_no_ref(path_with_query_slice, 0, strlen(t)); /* substitute previous path with the new path+query */ grpc_mdelem mdelem_path_and_query = grpc_mdelem_from_slices( @@ -349,7 +348,6 @@ static grpc_error *hc_mutate_op(grpc_exec_ctx *exec_ctx, calld->on_complete = op->on_complete; op->on_complete = &calld->hc_on_complete; op->send_message = false; - grpc_slice_unref_internal(exec_ctx, path_with_query_slice); } else { /* Not all data is available. Fall back to POST. */ gpr_log(GPR_DEBUG, From 3725128dc659829b69bb4eced8fec3f0621da890 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Fri, 14 Apr 2017 13:46:03 -0700 Subject: [PATCH 49/60] Comments --- .../filters/client_channel/client_channel.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/core/ext/filters/client_channel/client_channel.c b/src/core/ext/filters/client_channel/client_channel.c index e446f0543a8..813d7aa06dd 100644 --- a/src/core/ext/filters/client_channel/client_channel.c +++ b/src/core/ext/filters/client_channel/client_channel.c @@ -236,6 +236,10 @@ static void set_channel_connectivity_state_locked(grpc_exec_ctx *exec_ctx, grpc_connectivity_state state, grpc_error *error, const char *reason) { + /* TODO: Improve failure handling: + * - Make it possible for policies to return GRPC_CHANNEL_TRANSIENT_FAILURE. + * - Hand over pending picks from old policies during the switch that happens + * when resolver provides an update. */ if (chand->lb_policy != NULL) { if (state == GRPC_CHANNEL_TRANSIENT_FAILURE) { /* cancel picks with wait_for_ready=false */ @@ -364,22 +368,18 @@ typedef struct wrapped_on_pick_closure_arg { /* The policy instance related to the closure */ grpc_lb_policy *lb_policy; - - /* heap memory to be freed upon closure execution. Usually this arg. */ - void *free_when_done; } wrapped_on_pick_closure_arg; -// Invoke \a arg->wrapped_closure, unref \a arg->lb_policy and free -// arg->free_when_done (usually \a arg itself). +// Invoke \a arg->wrapped_closure, unref \a arg->lb_policy and free \a arg. static void wrapped_on_pick_closure_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { wrapped_on_pick_closure_arg *wc_arg = arg; + GPR_ASSERT(wc_arg != NULL); GPR_ASSERT(wc_arg->wrapped_closure != NULL); GPR_ASSERT(wc_arg->lb_policy != NULL); - GPR_ASSERT(wc_arg->free_when_done != NULL); - grpc_closure_sched(exec_ctx, wc_arg->wrapped_closure, GRPC_ERROR_REF(error)); + grpc_closure_run(exec_ctx, wc_arg->wrapped_closure, GRPC_ERROR_REF(error)); GRPC_LB_POLICY_UNREF(exec_ctx, wc_arg->lb_policy, "pick_subchannel_wrapping"); - gpr_free(wc_arg->free_when_done); + gpr_free(wc_arg); } static void on_resolver_result_changed_locked(grpc_exec_ctx *exec_ctx, @@ -1078,7 +1078,6 @@ static bool pick_subchannel_locked( w_on_pick_arg->wrapped_closure = on_ready; GRPC_LB_POLICY_REF(lb_policy, "pick_subchannel_wrapping"); w_on_pick_arg->lb_policy = lb_policy; - w_on_pick_arg->free_when_done = w_on_pick_arg; const bool pick_done = grpc_lb_policy_pick_locked( exec_ctx, lb_policy, &inputs, connected_subchannel, NULL, &w_on_pick_arg->wrapper_closure); @@ -1086,7 +1085,7 @@ static bool pick_subchannel_locked( /* synchronous grpc_lb_policy_pick call. Unref the LB policy. */ GRPC_LB_POLICY_UNREF(exec_ctx, w_on_pick_arg->lb_policy, "pick_subchannel_wrapping"); - gpr_free(w_on_pick_arg->free_when_done); + gpr_free(w_on_pick_arg); } GRPC_LB_POLICY_UNREF(exec_ctx, lb_policy, "pick_subchannel"); GPR_TIMER_END("pick_subchannel", 0); From 9c8e9be16bace39e31df10c16327aa7620ff3ca9 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Fri, 14 Apr 2017 14:01:33 -0700 Subject: [PATCH 50/60] removed 'objc' from list of 'all' languages --- tools/run_tests/run_interop_tests.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index fcc752fdea4..89725c10a63 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -947,9 +947,13 @@ if not args.use_docker and servers: print('Running interop servers is only supported with --use_docker option enabled.') sys.exit(1) + +# we want to include everything but objc in 'all' +# because objc won't run on non-mac platforms +all_but_objc = set(six.iterkeys(_LANGUAGES)) - set(['objc']) languages = set(_LANGUAGES[l] for l in itertools.chain.from_iterable( - six.iterkeys(_LANGUAGES) if x == 'all' else [x] + all_but_objc if x == 'all' else [x] for x in args.language)) languages_http2_clients_for_http2_server_interop = set() From d471818d8c986b2e4e86c2150dbbf2d54fb0d526 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Fri, 14 Apr 2017 15:05:41 -0700 Subject: [PATCH 51/60] script to run objc interop tests --- tools/jenkins/run_interop_objc.sh | 39 +++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100755 tools/jenkins/run_interop_objc.sh diff --git a/tools/jenkins/run_interop_objc.sh b/tools/jenkins/run_interop_objc.sh new file mode 100755 index 00000000000..29f9ac989c7 --- /dev/null +++ b/tools/jenkins/run_interop_objc.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Copyright 2017, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# This script is invoked by Jenkins and runs interop test suite. +set -ex + +export LANG=en_US.UTF-8 + +# Enter the gRPC repo root +cd $(dirname $0)/../.. + +tools/run_tests/run_interop_tests.py -l objc -s all --use_docker -t -j 1 $@ || true From c59db4b136a3991f0edce0ac9cc008cf7737f0fb Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Fri, 14 Apr 2017 15:08:31 -0700 Subject: [PATCH 52/60] added more tests to 'unimplemented' --- tools/run_tests/run_interop_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 89725c10a63..44e93eb9cc3 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -397,7 +397,7 @@ class ObjcLanguage: # cmdline argument. Here we return all but one test cases as unimplemented, # and depend upon ObjC test's behavior that it runs all cases even when # we tell it to run just one. - return _TEST_CASES[1:] + return _TEST_CASES[1:] + _SKIP_COMPRESSION + _SKIP_DATA_FRAME_PADDING def unimplemented_test_cases_server(self): return _SKIP_COMPRESSION From 53af23cfbf3b1fd4579ec084dbcb7b89a7ae2e96 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Sat, 15 Apr 2017 10:29:46 -0700 Subject: [PATCH 53/60] Improvements to Fake Resolver --- CMakeLists.txt | 32 +++ Makefile | 36 +++ build.yaml | 10 + .../client_channel/lb_policy_factory.c | 33 ++- .../client_channel/lb_policy_factory.h | 22 +- .../filters/client_channel/parse_address.c | 44 +++- .../filters/client_channel/parse_address.h | 19 +- .../resolver/sockaddr/sockaddr_resolver.c | 6 +- .../ext/filters/client_channel/subchannel.c | 8 +- .../ext/filters/client_channel/uri_parser.c | 4 +- .../ext/filters/client_channel/uri_parser.h | 2 +- src/core/lib/channel/channel_args.c | 11 +- src/core/lib/channel/channel_args.h | 3 +- src/core/lib/iomgr/sockaddr_utils.h | 2 +- .../credentials/fake/fake_credentials.c | 23 ++ .../credentials/fake/fake_credentials.h | 21 +- .../security/transport/security_connector.c | 8 +- test/core/client_channel/parse_address_test.c | 26 +-- test/core/client_channel/resolvers/BUILD | 40 +++- .../resolvers/fake_resolver_test.c | 187 ++++++++++++++++ test/core/end2end/BUILD | 79 ++++--- test/core/end2end/fake_resolver.c | 211 ++++++++++-------- test/core/end2end/fake_resolver.h | 34 +++ test/cpp/grpclb/grpclb_test.cc | 45 ++-- .../generated/sources_and_headers.json | 17 ++ tools/run_tests/generated/tests.json | 22 ++ vsprojects/buildtests_c.sln | 27 +++ .../fake_resolver_test.vcxproj | 199 +++++++++++++++++ .../fake_resolver_test.vcxproj.filters | 24 ++ 29 files changed, 975 insertions(+), 220 deletions(-) create mode 100644 test/core/client_channel/resolvers/fake_resolver_test.c create mode 100644 vsprojects/vcxproj/test/fake_resolver_test/fake_resolver_test.vcxproj create mode 100644 vsprojects/vcxproj/test/fake_resolver_test/fake_resolver_test.vcxproj.filters diff --git a/CMakeLists.txt b/CMakeLists.txt index 0dad894cc4c..a6a2af65d5d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -391,6 +391,7 @@ add_dependencies(buildtests_c error_test) if(_gRPC_PLATFORM_LINUX) add_dependencies(buildtests_c ev_epoll_linux_test) endif() +add_dependencies(buildtests_c fake_resolver_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_c fd_conservation_posix_test) endif() @@ -5444,6 +5445,37 @@ target_link_libraries(ev_epoll_linux_test ) endif() +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) + +add_executable(fake_resolver_test + test/core/client_channel/resolvers/fake_resolver_test.c +) + + +target_include_directories(fake_resolver_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CARES_BUILD_INCLUDE_DIR} + PRIVATE ${CARES_INCLUDE_DIR} + PRIVATE ${CARES_PLATFORM_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include +) + +target_link_libraries(fake_resolver_test + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_test_util + grpc + gpr_test_util + gpr +) + endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) diff --git a/Makefile b/Makefile index b3d8737ecc1..8898939af9e 100644 --- a/Makefile +++ b/Makefile @@ -981,6 +981,7 @@ dualstack_socket_test: $(BINDIR)/$(CONFIG)/dualstack_socket_test endpoint_pair_test: $(BINDIR)/$(CONFIG)/endpoint_pair_test error_test: $(BINDIR)/$(CONFIG)/error_test ev_epoll_linux_test: $(BINDIR)/$(CONFIG)/ev_epoll_linux_test +fake_resolver_test: $(BINDIR)/$(CONFIG)/fake_resolver_test fd_conservation_posix_test: $(BINDIR)/$(CONFIG)/fd_conservation_posix_test fd_posix_test: $(BINDIR)/$(CONFIG)/fd_posix_test fling_client: $(BINDIR)/$(CONFIG)/fling_client @@ -1366,6 +1367,7 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/endpoint_pair_test \ $(BINDIR)/$(CONFIG)/error_test \ $(BINDIR)/$(CONFIG)/ev_epoll_linux_test \ + $(BINDIR)/$(CONFIG)/fake_resolver_test \ $(BINDIR)/$(CONFIG)/fd_conservation_posix_test \ $(BINDIR)/$(CONFIG)/fd_posix_test \ $(BINDIR)/$(CONFIG)/fling_client \ @@ -1777,6 +1779,8 @@ test_c: buildtests_c $(Q) $(BINDIR)/$(CONFIG)/error_test || ( echo test error_test failed ; exit 1 ) $(E) "[RUN] Testing ev_epoll_linux_test" $(Q) $(BINDIR)/$(CONFIG)/ev_epoll_linux_test || ( echo test ev_epoll_linux_test failed ; exit 1 ) + $(E) "[RUN] Testing fake_resolver_test" + $(Q) $(BINDIR)/$(CONFIG)/fake_resolver_test || ( echo test fake_resolver_test failed ; exit 1 ) $(E) "[RUN] Testing fd_conservation_posix_test" $(Q) $(BINDIR)/$(CONFIG)/fd_conservation_posix_test || ( echo test fd_conservation_posix_test failed ; exit 1 ) $(E) "[RUN] Testing fd_posix_test" @@ -9401,6 +9405,38 @@ endif endif +FAKE_RESOLVER_TEST_SRC = \ + test/core/client_channel/resolvers/fake_resolver_test.c \ + +FAKE_RESOLVER_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(FAKE_RESOLVER_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/fake_resolver_test: openssl_dep_error + +else + + + +$(BINDIR)/$(CONFIG)/fake_resolver_test: $(FAKE_RESOLVER_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LD) $(LDFLAGS) $(FAKE_RESOLVER_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/fake_resolver_test + +endif + +$(OBJDIR)/$(CONFIG)/test/core/client_channel/resolvers/fake_resolver_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_fake_resolver_test: $(FAKE_RESOLVER_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(FAKE_RESOLVER_TEST_OBJS:.o=.dep) +endif +endif + + FD_CONSERVATION_POSIX_TEST_SRC = \ test/core/iomgr/fd_conservation_posix_test.c \ diff --git a/build.yaml b/build.yaml index cfa479c4ac4..f4461e40ef1 100644 --- a/build.yaml +++ b/build.yaml @@ -1820,6 +1820,16 @@ targets: - uv platforms: - linux +- name: fake_resolver_test + build: test + language: c + src: + - test/core/client_channel/resolvers/fake_resolver_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: fd_conservation_posix_test build: test language: c diff --git a/src/core/ext/filters/client_channel/lb_policy_factory.c b/src/core/ext/filters/client_channel/lb_policy_factory.c index e2af216b899..89b8bf89515 100644 --- a/src/core/ext/filters/client_channel/lb_policy_factory.c +++ b/src/core/ext/filters/client_channel/lb_policy_factory.c @@ -36,16 +36,18 @@ #include #include +#include "src/core/lib/channel/channel_args.h" + #include "src/core/ext/filters/client_channel/lb_policy_factory.h" +#include "src/core/ext/filters/client_channel/parse_address.h" grpc_lb_addresses* grpc_lb_addresses_create( size_t num_addresses, const grpc_lb_user_data_vtable* user_data_vtable) { - grpc_lb_addresses* addresses = gpr_malloc(sizeof(grpc_lb_addresses)); + grpc_lb_addresses* addresses = gpr_zalloc(sizeof(grpc_lb_addresses)); addresses->num_addresses = num_addresses; addresses->user_data_vtable = user_data_vtable; const size_t addresses_size = sizeof(grpc_lb_address) * num_addresses; - addresses->addresses = gpr_malloc(addresses_size); - memset(addresses->addresses, 0, addresses_size); + addresses->addresses = gpr_zalloc(addresses_size); return addresses; } @@ -69,7 +71,7 @@ grpc_lb_addresses* grpc_lb_addresses_copy(const grpc_lb_addresses* addresses) { void grpc_lb_addresses_set_address(grpc_lb_addresses* addresses, size_t index, void* address, size_t address_len, - bool is_balancer, char* balancer_name, + bool is_balancer, const char* balancer_name, void* user_data) { GPR_ASSERT(index < addresses->num_addresses); if (user_data != NULL) GPR_ASSERT(addresses->user_data_vtable != NULL); @@ -77,10 +79,22 @@ void grpc_lb_addresses_set_address(grpc_lb_addresses* addresses, size_t index, memcpy(target->address.addr, address, address_len); target->address.len = address_len; target->is_balancer = is_balancer; - target->balancer_name = balancer_name; + target->balancer_name = gpr_strdup(balancer_name); target->user_data = user_data; } +bool grpc_lb_addresses_set_address_from_uri(grpc_lb_addresses* addresses, + size_t index, const grpc_uri* uri, + bool is_balancer, + const char* balancer_name, + void* user_data) { + grpc_resolved_address address; + if (!grpc_parse_uri(uri, &address)) return false; + grpc_lb_addresses_set_address(addresses, index, address.addr, address.len, + is_balancer, balancer_name, user_data); + return true; +} + int grpc_lb_addresses_cmp(const grpc_lb_addresses* addresses1, const grpc_lb_addresses* addresses2) { if (addresses1->num_addresses > addresses2->num_addresses) return 1; @@ -147,6 +161,15 @@ grpc_arg grpc_lb_addresses_create_channel_arg( return arg; } +grpc_lb_addresses* grpc_lb_addresses_find_channel_arg( + const grpc_channel_args* channel_args) { + const grpc_arg* lb_addresses_arg = + grpc_channel_args_find(channel_args, GRPC_ARG_LB_ADDRESSES); + if (lb_addresses_arg == NULL || lb_addresses_arg->type != GRPC_ARG_POINTER) + return NULL; + return lb_addresses_arg->value.pointer.p; +} + void grpc_lb_policy_factory_ref(grpc_lb_policy_factory* factory) { factory->vtable->ref(factory); } diff --git a/src/core/ext/filters/client_channel/lb_policy_factory.h b/src/core/ext/filters/client_channel/lb_policy_factory.h index 81ab12ec8f9..9d6c0fc1398 100644 --- a/src/core/ext/filters/client_channel/lb_policy_factory.h +++ b/src/core/ext/filters/client_channel/lb_policy_factory.h @@ -34,12 +34,13 @@ #ifndef GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_FACTORY_H #define GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_FACTORY_H -#include "src/core/ext/filters/client_channel/client_channel_factory.h" -#include "src/core/ext/filters/client_channel/lb_policy.h" - #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/iomgr/resolve_address.h" +#include "src/core/ext/filters/client_channel/client_channel_factory.h" +#include "src/core/ext/filters/client_channel/lb_policy.h" +#include "src/core/ext/filters/client_channel/uri_parser.h" + // Channel arg key for grpc_lb_addresses. #define GRPC_ARG_LB_ADDRESSES "grpc.lb_addresses" @@ -88,9 +89,18 @@ grpc_lb_addresses *grpc_lb_addresses_copy(const grpc_lb_addresses *addresses); * Takes ownership of \a balancer_name. */ void grpc_lb_addresses_set_address(grpc_lb_addresses *addresses, size_t index, void *address, size_t address_len, - bool is_balancer, char *balancer_name, + bool is_balancer, const char *balancer_name, void *user_data); +/** Sets the value of the address at index \a index of \a addresses from \a uri. + * Returns true upon success, false otherwise. Takes ownership of \a + * balancer_name. */ +bool grpc_lb_addresses_set_address_from_uri(grpc_lb_addresses *addresses, + size_t index, const grpc_uri *uri, + bool is_balancer, + const char *balancer_name, + void *user_data); + /** Compares \a addresses1 and \a addresses2. */ int grpc_lb_addresses_cmp(const grpc_lb_addresses *addresses1, const grpc_lb_addresses *addresses2); @@ -103,6 +113,10 @@ void grpc_lb_addresses_destroy(grpc_exec_ctx *exec_ctx, grpc_arg grpc_lb_addresses_create_channel_arg( const grpc_lb_addresses *addresses); +/** Returns the \a grpc_lb_addresses instance in \a channel_args or NULL */ +grpc_lb_addresses *grpc_lb_addresses_find_channel_arg( + const grpc_channel_args *channel_args); + /** Arguments passed to LB policies. */ typedef struct grpc_lb_policy_args { grpc_client_channel_factory *client_channel_factory; diff --git a/src/core/ext/filters/client_channel/parse_address.c b/src/core/ext/filters/client_channel/parse_address.c index 0c970620756..edc6ce697d8 100644 --- a/src/core/ext/filters/client_channel/parse_address.c +++ b/src/core/ext/filters/client_channel/parse_address.c @@ -48,7 +48,12 @@ #ifdef GRPC_HAVE_UNIX_SOCKET -int parse_unix(grpc_uri *uri, grpc_resolved_address *resolved_addr) { +bool grpc_parse_unix(const grpc_uri *uri, + grpc_resolved_address *resolved_addr) { + if (strcmp("unix", uri->scheme) != 0) { + gpr_log(GPR_ERROR, "Expected 'unix' scheme, got '%s'", uri->scheme); + return false; + } struct sockaddr_un *un = (struct sockaddr_un *)resolved_addr->addr; const size_t maxlen = sizeof(un->sun_path); const size_t path_len = strnlen(uri->path, maxlen); @@ -61,21 +66,29 @@ int parse_unix(grpc_uri *uri, grpc_resolved_address *resolved_addr) { #else /* GRPC_HAVE_UNIX_SOCKET */ -int parse_unix(grpc_uri *uri, grpc_resolved_address *resolved_addr) { abort(); } +bool grpc_parse_unix(const grpc_uri *uri, + grpc_resolved_address *resolved_addr) { + abort(); +} #endif /* GRPC_HAVE_UNIX_SOCKET */ -int parse_ipv4(grpc_uri *uri, grpc_resolved_address *resolved_addr) { +bool grpc_parse_ipv4(const grpc_uri *uri, + grpc_resolved_address *resolved_addr) { + if (strcmp("ipv4", uri->scheme) != 0) { + gpr_log(GPR_ERROR, "Expected 'ipv4' scheme, got '%s'", uri->scheme); + return false; + } const char *host_port = uri->path; char *host; char *port; int port_num; - int result = 0; + bool result = false; struct sockaddr_in *in = (struct sockaddr_in *)resolved_addr->addr; if (*host_port == '/') ++host_port; if (!gpr_split_host_port(host_port, &host, &port)) { - return 0; + return false; } memset(resolved_addr, 0, sizeof(grpc_resolved_address)); @@ -98,14 +111,19 @@ int parse_ipv4(grpc_uri *uri, grpc_resolved_address *resolved_addr) { goto done; } - result = 1; + result = true; done: gpr_free(host); gpr_free(port); return result; } -int parse_ipv6(grpc_uri *uri, grpc_resolved_address *resolved_addr) { +bool grpc_parse_ipv6(const grpc_uri *uri, + grpc_resolved_address *resolved_addr) { + if (strcmp("ipv6", uri->scheme) != 0) { + gpr_log(GPR_ERROR, "Expected 'ipv6' scheme, got '%s'", uri->scheme); + return false; + } const char *host_port = uri->path; char *host; char *port; @@ -168,3 +186,15 @@ done: gpr_free(port); return result; } + +bool grpc_parse_uri(const grpc_uri *uri, grpc_resolved_address *resolved_addr) { + if (strcmp("unix", uri->scheme) == 0) { + return grpc_parse_unix(uri, resolved_addr); + } else if (strcmp("ipv4", uri->scheme) == 0) { + return grpc_parse_ipv4(uri, resolved_addr); + } else if (strcmp("ipv6", uri->scheme) == 0) { + return grpc_parse_ipv6(uri, resolved_addr); + } + gpr_log(GPR_ERROR, "Can't parse scheme '%s'", uri->scheme); + return false; +} diff --git a/src/core/ext/filters/client_channel/parse_address.h b/src/core/ext/filters/client_channel/parse_address.h index c8d77baa008..fa7ea33a00b 100644 --- a/src/core/ext/filters/client_channel/parse_address.h +++ b/src/core/ext/filters/client_channel/parse_address.h @@ -39,16 +39,19 @@ #include "src/core/ext/filters/client_channel/uri_parser.h" #include "src/core/lib/iomgr/resolve_address.h" -/** Populate \a addr and \a len from \a uri, whose path is expected to contain a +/** Populate \a resolved_addr from \a uri, whose path is expected to contain a * unix socket path. Returns true upon success. */ -int parse_unix(grpc_uri *uri, grpc_resolved_address *resolved_addr); +bool grpc_parse_unix(const grpc_uri *uri, grpc_resolved_address *resolved_addr); -/** Populate /a addr and \a len from \a uri, whose path is expected to contain a - * host:port pair. Returns true upon success. */ -int parse_ipv4(grpc_uri *uri, grpc_resolved_address *resolved_addr); +/** Populate \a resolved_addr from \a uri, whose path is expected to contain an + * IPv4 host:port pair. Returns true upon success. */ +bool grpc_parse_ipv4(const grpc_uri *uri, grpc_resolved_address *resolved_addr); -/** Populate /a addr and \a len from \a uri, whose path is expected to contain a - * host:port pair. Returns true upon success. */ -int parse_ipv6(grpc_uri *uri, grpc_resolved_address *resolved_addr); +/** Populate \a resolved_addr from \a uri, whose path is expected to contain an + * IPv6 host:port pair. Returns true upon success. */ +bool grpc_parse_ipv6(const grpc_uri *uri, grpc_resolved_address *resolved_addr); + +/** Populate \a resolved_addr from \a uri. Returns true upon success. */ +bool grpc_parse_uri(const grpc_uri *uri, grpc_resolved_address *resolved_addr); #endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_PARSE_ADDRESS_H */ diff --git a/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.c b/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.c index 54f020d6914..4d7d878c237 100644 --- a/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.c +++ b/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.c @@ -157,8 +157,8 @@ static void do_nothing(void *ignored) {} static grpc_resolver *sockaddr_create(grpc_exec_ctx *exec_ctx, grpc_resolver_args *args, - int parse(grpc_uri *uri, - grpc_resolved_address *dst)) { + bool parse(const grpc_uri *uri, + grpc_resolved_address *dst)) { if (0 != strcmp(args->uri->authority, "")) { gpr_log(GPR_ERROR, "authority based uri's not supported by the %s scheme", args->uri->scheme); @@ -209,7 +209,7 @@ static void sockaddr_factory_unref(grpc_resolver_factory *factory) {} static grpc_resolver *name##_factory_create_resolver( \ grpc_exec_ctx *exec_ctx, grpc_resolver_factory *factory, \ grpc_resolver_args *args) { \ - return sockaddr_create(exec_ctx, args, parse_##name); \ + return sockaddr_create(exec_ctx, args, grpc_parse_##name); \ } \ static const grpc_resolver_factory_vtable name##_factory_vtable = { \ sockaddr_factory_ref, sockaddr_factory_unref, \ diff --git a/src/core/ext/filters/client_channel/subchannel.c b/src/core/ext/filters/client_channel/subchannel.c index 9a7a7a0ee5f..362b4a4a283 100644 --- a/src/core/ext/filters/client_channel/subchannel.c +++ b/src/core/ext/filters/client_channel/subchannel.c @@ -797,13 +797,7 @@ static void grpc_uri_to_sockaddr(grpc_exec_ctx *exec_ctx, const char *uri_str, grpc_resolved_address *addr) { grpc_uri *uri = grpc_uri_parse(exec_ctx, uri_str, 0 /* suppress_errors */); GPR_ASSERT(uri != NULL); - if (strcmp(uri->scheme, "ipv4") == 0) { - GPR_ASSERT(parse_ipv4(uri, addr)); - } else if (strcmp(uri->scheme, "ipv6") == 0) { - GPR_ASSERT(parse_ipv6(uri, addr)); - } else { - GPR_ASSERT(parse_unix(uri, addr)); - } + if (!grpc_parse_uri(uri, addr)) memset(addr, 0, sizeof(*addr)); grpc_uri_destroy(uri); } diff --git a/src/core/ext/filters/client_channel/uri_parser.c b/src/core/ext/filters/client_channel/uri_parser.c index f28db59e270..b233d835cbc 100644 --- a/src/core/ext/filters/client_channel/uri_parser.c +++ b/src/core/ext/filters/client_channel/uri_parser.c @@ -50,7 +50,7 @@ #define NOT_SET (~(size_t)0) static grpc_uri *bad_uri(const char *uri_text, size_t pos, const char *section, - int suppress_errors) { + bool suppress_errors) { char *line_prefix; size_t pfx_len; @@ -197,7 +197,7 @@ static void parse_query_parts(grpc_uri *uri) { } grpc_uri *grpc_uri_parse(grpc_exec_ctx *exec_ctx, const char *uri_text, - int suppress_errors) { + bool suppress_errors) { grpc_uri *uri; size_t scheme_begin = 0; size_t scheme_end = NOT_SET; diff --git a/src/core/ext/filters/client_channel/uri_parser.h b/src/core/ext/filters/client_channel/uri_parser.h index 2698d448d8e..b889040b167 100644 --- a/src/core/ext/filters/client_channel/uri_parser.h +++ b/src/core/ext/filters/client_channel/uri_parser.h @@ -53,7 +53,7 @@ typedef struct { /** parse a uri, return NULL on failure */ grpc_uri *grpc_uri_parse(grpc_exec_ctx *exec_ctx, const char *uri_text, - int suppress_errors); + bool suppress_errors); /** return the part of a query string after the '=' in "?key=xxx&...", or NULL * if key is not present */ diff --git a/src/core/lib/channel/channel_args.c b/src/core/lib/channel/channel_args.c index 3de31d99da1..238d176dfae 100644 --- a/src/core/lib/channel/channel_args.c +++ b/src/core/lib/channel/channel_args.c @@ -31,17 +31,18 @@ * */ -#include "src/core/lib/channel/channel_args.h" -#include -#include "src/core/lib/support/string.h" +#include +#include #include +#include #include #include #include #include -#include +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/support/string.h" static grpc_arg copy_arg(const grpc_arg *src) { grpc_arg dst; @@ -330,7 +331,7 @@ const grpc_arg *grpc_channel_args_find(const grpc_channel_args *args, } int grpc_channel_arg_get_integer(const grpc_arg *arg, - grpc_integer_options options) { + const grpc_integer_options options) { if (arg == NULL) return options.default_value; if (arg->type != GRPC_ARG_INTEGER) { gpr_log(GPR_ERROR, "%s ignored: it must be an integer", arg->key); diff --git a/src/core/lib/channel/channel_args.h b/src/core/lib/channel/channel_args.h index 5ffcacb7fd9..f0f603e2514 100644 --- a/src/core/lib/channel/channel_args.h +++ b/src/core/lib/channel/channel_args.h @@ -120,9 +120,10 @@ typedef struct grpc_integer_options { int min_value; int max_value; } grpc_integer_options; + /** Returns the value of \a arg, subject to the contraints in \a options. */ int grpc_channel_arg_get_integer(const grpc_arg *arg, - grpc_integer_options options); + const grpc_integer_options options); bool grpc_channel_arg_get_bool(const grpc_arg *arg, bool default_value); diff --git a/src/core/lib/iomgr/sockaddr_utils.h b/src/core/lib/iomgr/sockaddr_utils.h index 2b22f11b49d..be3ea2038f1 100644 --- a/src/core/lib/iomgr/sockaddr_utils.h +++ b/src/core/lib/iomgr/sockaddr_utils.h @@ -50,7 +50,7 @@ int grpc_sockaddr_to_v4mapped(const grpc_resolved_address *addr, grpc_resolved_address *addr6_out); /* If addr is ::, 0.0.0.0, or ::ffff:0.0.0.0, writes the port number to - *port_out (if not NULL) and returns true, otherwise returns false. */ + *port_out (if not NULL) and returns true, otherwise returns false. */ int grpc_sockaddr_is_wildcard(const grpc_resolved_address *addr, int *port_out); /* Writes 0.0.0.0:port and [::]:port to separate sockaddrs. */ diff --git a/src/core/lib/security/credentials/fake/fake_credentials.c b/src/core/lib/security/credentials/fake/fake_credentials.c index 68636ba2088..3fdb67fb912 100644 --- a/src/core/lib/security/credentials/fake/fake_credentials.c +++ b/src/core/lib/security/credentials/fake/fake_credentials.c @@ -39,11 +39,15 @@ #include #include +#include "src/core/lib/channel/channel_args.h" #include "src/core/lib/iomgr/executor.h" #include "src/core/lib/support/string.h" /* -- Fake transport security credentials. -- */ +#define GRPC_ARG_FAKE_SECURITY_EXPECTED_TARGETS \ + "grpc.fake_security.expected_targets" + static grpc_security_status fake_transport_security_create_security_connector( grpc_exec_ctx *exec_ctx, grpc_channel_credentials *c, grpc_call_credentials *call_creds, const char *target, @@ -88,6 +92,25 @@ grpc_server_credentials *grpc_fake_transport_security_server_credentials_create( return c; } +grpc_arg grpc_fake_transport_expected_targets_arg(char *expected_targets) { + grpc_arg arg; + arg.type = GRPC_ARG_STRING; + arg.key = GRPC_ARG_FAKE_SECURITY_EXPECTED_TARGETS; + arg.value.string = expected_targets; + return arg; +} + +const char *grpc_fake_transport_get_expected_targets( + const grpc_channel_args *args) { + const grpc_arg *expected_target_arg = + grpc_channel_args_find(args, GRPC_ARG_FAKE_SECURITY_EXPECTED_TARGETS); + if (expected_target_arg != NULL && + expected_target_arg->type == GRPC_ARG_STRING) { + return expected_target_arg->value.string; + } + return NULL; +} + /* -- Metadata-only test credentials. -- */ static void md_only_test_destruct(grpc_exec_ctx *exec_ctx, diff --git a/src/core/lib/security/credentials/fake/fake_credentials.h b/src/core/lib/security/credentials/fake/fake_credentials.h index 0fe98417c6c..a28b545a676 100644 --- a/src/core/lib/security/credentials/fake/fake_credentials.h +++ b/src/core/lib/security/credentials/fake/fake_credentials.h @@ -38,10 +38,17 @@ /* -- Fake transport security credentials. -- */ +/* Creates a fake transport security credentials object for testing. */ +grpc_channel_credentials *grpc_fake_transport_security_credentials_create(void); + +/* Creates a fake server transport security credentials object for testing. */ +grpc_server_credentials *grpc_fake_transport_security_server_credentials_create( + void); + /* Used to verify the target names given to the fake transport security * connector. * - * Its syntax by example: + * The syntax of \a expected_targets by example: * For LB channels: * "backend_target_1,backend_target_2,...;lb_target_1,lb_target_2,..." * For regular channels: @@ -50,15 +57,11 @@ * That is to say, LB channels have a heading list of LB targets separated from * the list of backend targets by a semicolon. For non-LB channels, only the * latter is present. */ -#define GRPC_ARG_FAKE_SECURITY_EXPECTED_TARGETS \ - "grpc.test_only.fake_security.expected_target" +grpc_arg grpc_fake_transport_expected_targets_arg(char *expected_targets); -/* Creates a fake transport security credentials object for testing. */ -grpc_channel_credentials *grpc_fake_transport_security_credentials_create(void); - -/* Creates a fake server transport security credentials object for testing. */ -grpc_server_credentials *grpc_fake_transport_security_server_credentials_create( - void); +/* Return the value associated with the expected targets channel arg or NULL */ +const char *grpc_fake_transport_get_expected_targets( + const grpc_channel_args *args); /* -- Metadata-only Test credentials. -- */ diff --git a/src/core/lib/security/transport/security_connector.c b/src/core/lib/security/transport/security_connector.c index dbe3263f92a..b15196e6770 100644 --- a/src/core/lib/security/transport/security_connector.c +++ b/src/core/lib/security/transport/security_connector.c @@ -423,12 +423,8 @@ grpc_channel_security_connector *grpc_fake_channel_security_connector_create( c->base.check_call_host = fake_channel_check_call_host; c->base.add_handshakers = fake_channel_add_handshakers; c->target = gpr_strdup(target); - const grpc_arg *expected_target_arg = - grpc_channel_args_find(args, GRPC_ARG_FAKE_SECURITY_EXPECTED_TARGETS); - if (expected_target_arg != NULL) { - GPR_ASSERT(expected_target_arg->type == GRPC_ARG_STRING); - c->expected_targets = gpr_strdup(expected_target_arg->value.string); - } + const char *expected_targets = grpc_fake_transport_get_expected_targets(args); + c->expected_targets = gpr_strdup(expected_targets); c->is_lb_channel = (grpc_lb_targets_info_find_in_args(args) != NULL); return &c->base; } diff --git a/test/core/client_channel/parse_address_test.c b/test/core/client_channel/parse_address_test.c index 629cdb001f9..802e41e5dee 100644 --- a/test/core/client_channel/parse_address_test.c +++ b/test/core/client_channel/parse_address_test.c @@ -47,12 +47,12 @@ #ifdef GRPC_HAVE_UNIX_SOCKET -static void test_parse_unix(const char *uri_text, const char *pathname) { +static void test_grpc_parse_unix(const char *uri_text, const char *pathname) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_uri *uri = grpc_uri_parse(&exec_ctx, uri_text, 0); grpc_resolved_address addr; - GPR_ASSERT(1 == parse_unix(uri, &addr)); + GPR_ASSERT(1 == grpc_parse_unix(uri, &addr)); struct sockaddr_un *addr_un = (struct sockaddr_un *)addr.addr; GPR_ASSERT(AF_UNIX == addr_un->sun_family); GPR_ASSERT(0 == strcmp(addr_un->sun_path, pathname)); @@ -63,18 +63,18 @@ static void test_parse_unix(const char *uri_text, const char *pathname) { #else /* GRPC_HAVE_UNIX_SOCKET */ -static void test_parse_unix(const char *uri_text, const char *pathname) {} +static void test_grpc_parse_unix(const char *uri_text, const char *pathname) {} #endif /* GRPC_HAVE_UNIX_SOCKET */ -static void test_parse_ipv4(const char *uri_text, const char *host, - unsigned short port) { +static void test_grpc_parse_ipv4(const char *uri_text, const char *host, + unsigned short port) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_uri *uri = grpc_uri_parse(&exec_ctx, uri_text, 0); grpc_resolved_address addr; char ntop_buf[INET_ADDRSTRLEN]; - GPR_ASSERT(1 == parse_ipv4(uri, &addr)); + GPR_ASSERT(1 == grpc_parse_ipv4(uri, &addr)); struct sockaddr_in *addr_in = (struct sockaddr_in *)addr.addr; GPR_ASSERT(AF_INET == addr_in->sin_family); GPR_ASSERT(NULL != grpc_inet_ntop(AF_INET, &addr_in->sin_addr, ntop_buf, @@ -86,14 +86,14 @@ static void test_parse_ipv4(const char *uri_text, const char *host, grpc_exec_ctx_finish(&exec_ctx); } -static void test_parse_ipv6(const char *uri_text, const char *host, - unsigned short port, uint32_t scope_id) { +static void test_grpc_parse_ipv6(const char *uri_text, const char *host, + unsigned short port, uint32_t scope_id) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_uri *uri = grpc_uri_parse(&exec_ctx, uri_text, 0); grpc_resolved_address addr; char ntop_buf[INET6_ADDRSTRLEN]; - GPR_ASSERT(1 == parse_ipv6(uri, &addr)); + GPR_ASSERT(1 == grpc_parse_ipv6(uri, &addr)); struct sockaddr_in6 *addr_in6 = (struct sockaddr_in6 *)addr.addr; GPR_ASSERT(AF_INET6 == addr_in6->sin6_family); GPR_ASSERT(NULL != grpc_inet_ntop(AF_INET6, &addr_in6->sin6_addr, ntop_buf, @@ -109,8 +109,8 @@ static void test_parse_ipv6(const char *uri_text, const char *host, int main(int argc, char **argv) { grpc_test_init(argc, argv); - test_parse_unix("unix:/path/name", "/path/name"); - test_parse_ipv4("ipv4:192.0.2.1:12345", "192.0.2.1", 12345); - test_parse_ipv6("ipv6:[2001:db8::1]:12345", "2001:db8::1", 12345, 0); - test_parse_ipv6("ipv6:[2001:db8::1%252]:12345", "2001:db8::1", 12345, 2); + test_grpc_parse_unix("unix:/path/name", "/path/name"); + test_grpc_parse_ipv4("ipv4:192.0.2.1:12345", "192.0.2.1", 12345); + test_grpc_parse_ipv6("ipv6:[2001:db8::1]:12345", "2001:db8::1", 12345, 0); + test_grpc_parse_ipv6("ipv6:[2001:db8::1%252]:12345", "2001:db8::1", 12345, 2); } diff --git a/test/core/client_channel/resolvers/BUILD b/test/core/client_channel/resolvers/BUILD index af37072e3a4..e8361cdef63 100644 --- a/test/core/client_channel/resolvers/BUILD +++ b/test/core/client_channel/resolvers/BUILD @@ -32,20 +32,48 @@ licenses(["notice"]) # 3-clause BSD cc_test( name = "dns_resolver_connectivity_test", srcs = ["dns_resolver_connectivity_test.c"], - deps = ["//:grpc", "//test/core/util:grpc_test_util", "//:gpr", "//test/core/util:gpr_test_util"], - copts = ['-std=c99'] + copts = ["-std=c99"], + deps = [ + "//:gpr", + "//:grpc", + "//test/core/util:gpr_test_util", + "//test/core/util:grpc_test_util", + ], ) cc_test( name = "dns_resolver_test", srcs = ["dns_resolver_test.c"], - deps = ["//:grpc", "//test/core/util:grpc_test_util", "//:gpr", "//test/core/util:gpr_test_util"], - copts = ['-std=c99'] + copts = ["-std=c99"], + deps = [ + "//:gpr", + "//:grpc", + "//test/core/util:gpr_test_util", + "//test/core/util:grpc_test_util", + ], ) cc_test( name = "sockaddr_resolver_test", srcs = ["sockaddr_resolver_test.c"], - deps = ["//:grpc", "//test/core/util:grpc_test_util", "//:gpr", "//test/core/util:gpr_test_util"], - copts = ['-std=c99'] + copts = ["-std=c99"], + deps = [ + "//:gpr", + "//:grpc", + "//test/core/util:gpr_test_util", + "//test/core/util:grpc_test_util", + ], +) + +cc_test( + name = "fake_resolver_test", + srcs = ["fake_resolver_test.c"], + copts = ["-std=c99"], + deps = [ + "//:gpr", + "//:grpc", + "//test/core/end2end:fake_resolver", + "//test/core/util:gpr_test_util", + "//test/core/util:grpc_test_util", + ], ) diff --git a/test/core/client_channel/resolvers/fake_resolver_test.c b/test/core/client_channel/resolvers/fake_resolver_test.c new file mode 100644 index 00000000000..861918fbd65 --- /dev/null +++ b/test/core/client_channel/resolvers/fake_resolver_test.c @@ -0,0 +1,187 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include + +#include +#include +#include + +#include "src/core/ext/filters/client_channel/lb_policy_factory.h" +#include "src/core/ext/filters/client_channel/parse_address.h" +#include "src/core/ext/filters/client_channel/resolver_registry.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/iomgr/combiner.h" +#include "src/core/lib/security/credentials/fake/fake_credentials.h" + +#include "test/core/end2end/fake_resolver.h" +#include "test/core/util/test_config.h" + +static grpc_resolver *build_fake_resolver( + grpc_exec_ctx *exec_ctx, grpc_combiner *combiner, + grpc_fake_resolver_response_generator *response_generator) { + grpc_resolver_factory *factory = grpc_resolver_factory_lookup("test"); + grpc_arg generator_arg = + grpc_fake_resolver_response_generator_arg(response_generator); + grpc_resolver_args args; + memset(&args, 0, sizeof(args)); + grpc_channel_args channel_args = {1, &generator_arg}; + args.args = &channel_args; + args.combiner = combiner; + grpc_resolver *resolver = + grpc_resolver_factory_create_resolver(exec_ctx, factory, &args); + grpc_resolver_factory_unref(factory); + return resolver; +} + +typedef struct on_resolution_arg { + grpc_channel_args *resolver_result; + grpc_channel_args *expected_resolver_result; + bool was_called; +} on_resolution_arg; + +void on_resolution_cb(grpc_exec_ctx *exec_ctx, void *arg, grpc_error *error) { + on_resolution_arg *res = arg; + res->was_called = true; + // We only check the addresses channel arg because that's the only one + // explicitly set by the test via + // grpc_fake_resolver_response_generator_set_response. + const grpc_lb_addresses *actual_lb_addresses = + grpc_lb_addresses_find_channel_arg(res->resolver_result); + const grpc_lb_addresses *expected_lb_addresses = + grpc_lb_addresses_find_channel_arg(res->expected_resolver_result); + GPR_ASSERT( + grpc_lb_addresses_cmp(actual_lb_addresses, expected_lb_addresses) == 0); + grpc_channel_args_destroy(exec_ctx, res->resolver_result); + grpc_channel_args_destroy(exec_ctx, res->expected_resolver_result); +} + +static void test_fake_resolver() { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_combiner *combiner = grpc_combiner_create(NULL); + // Create resolver. + grpc_fake_resolver_response_generator *response_generator = + grpc_fake_resolver_response_generator_create(); + grpc_resolver *resolver = + build_fake_resolver(&exec_ctx, combiner, response_generator); + GPR_ASSERT(resolver != NULL); + + // Setup expectations. + grpc_uri *uris[] = {grpc_uri_parse(&exec_ctx, "ipv4:10.2.1.1:1234", true), + grpc_uri_parse(&exec_ctx, "ipv4:127.0.0.1:4321", true)}; + char *balancer_names[] = {"name1", "name2"}; + const bool is_balancer[] = {true, false}; + grpc_lb_addresses *addresses = grpc_lb_addresses_create(3, NULL); + for (size_t i = 0; i < GPR_ARRAY_SIZE(uris); ++i) { + grpc_lb_addresses_set_address_from_uri( + addresses, i, uris[i], is_balancer[i], balancer_names[i], NULL); + grpc_uri_destroy(uris[i]); + } + const grpc_arg addresses_arg = + grpc_lb_addresses_create_channel_arg(addresses); + grpc_channel_args *results = + grpc_channel_args_copy_and_add(NULL, &addresses_arg, 1); + grpc_lb_addresses_destroy(&exec_ctx, addresses); + on_resolution_arg on_res_arg; + memset(&on_res_arg, 0, sizeof(on_res_arg)); + on_res_arg.expected_resolver_result = results; + grpc_closure *on_resolution = grpc_closure_create( + on_resolution_cb, &on_res_arg, grpc_combiner_scheduler(combiner, false)); + + // Set resolver results and trigger first resolution. on_resolution_cb + // performs the checks. + grpc_fake_resolver_response_generator_set_response( + &exec_ctx, response_generator, results); + grpc_resolver_next_locked(&exec_ctx, resolver, &on_res_arg.resolver_result, + on_resolution); + grpc_exec_ctx_flush(&exec_ctx); + GPR_ASSERT(on_res_arg.was_called); + + // Setup update. + grpc_uri *uris_update[] = { + grpc_uri_parse(&exec_ctx, "ipv4:192.168.1.0:31416", true)}; + char *balancer_names_update[] = {"name3"}; + const bool is_balancer_update[] = {false}; + grpc_lb_addresses *addresses_update = grpc_lb_addresses_create(1, NULL); + for (size_t i = 0; i < GPR_ARRAY_SIZE(uris_update); ++i) { + grpc_lb_addresses_set_address_from_uri(addresses_update, i, uris_update[i], + is_balancer_update[i], + balancer_names_update[i], NULL); + grpc_uri_destroy(uris_update[i]); + } + + grpc_arg addresses_update_arg = + grpc_lb_addresses_create_channel_arg(addresses_update); + grpc_channel_args *results_update = + grpc_channel_args_copy_and_add(NULL, &addresses_update_arg, 1); + grpc_lb_addresses_destroy(&exec_ctx, addresses_update); + + // Setup expectations for the update. + on_resolution_arg on_res_arg_update; + memset(&on_res_arg_update, 0, sizeof(on_res_arg_update)); + on_res_arg_update.expected_resolver_result = results_update; + on_resolution = grpc_closure_create(on_resolution_cb, &on_res_arg_update, + grpc_combiner_scheduler(combiner, false)); + + // Set updated resolver results and trigger a second resolution. + grpc_fake_resolver_response_generator_set_response( + &exec_ctx, response_generator, results_update); + grpc_resolver_next_locked(&exec_ctx, resolver, + &on_res_arg_update.resolver_result, on_resolution); + grpc_exec_ctx_flush(&exec_ctx); + GPR_ASSERT(on_res_arg.was_called); + + // Requesting a new resolution without re-senting the response shouldn't + // trigger the resolution callback. + memset(&on_res_arg, 0, sizeof(on_res_arg)); + grpc_resolver_next_locked(&exec_ctx, resolver, &on_res_arg.resolver_result, + on_resolution); + grpc_exec_ctx_flush(&exec_ctx); + GPR_ASSERT(!on_res_arg.was_called); + + GRPC_COMBINER_UNREF(&exec_ctx, combiner, "test_fake_resolver"); + GRPC_RESOLVER_UNREF(&exec_ctx, resolver, "test_fake_resolver"); + grpc_exec_ctx_finish(&exec_ctx); + grpc_fake_resolver_response_generator_unref(response_generator); +} + +int main(int argc, char **argv) { + grpc_test_init(argc, argv); + grpc_fake_resolver_init(); // Registers the "test" scheme. + grpc_init(); + + test_fake_resolver(); + + grpc_shutdown(); + return 0; +} diff --git a/test/core/end2end/BUILD b/test/core/end2end/BUILD index 0cef7aa01df..ffea1cc4e8e 100644 --- a/test/core/end2end/BUILD +++ b/test/core/end2end/BUILD @@ -32,49 +32,66 @@ licenses(["notice"]) # 3-clause BSD load(":generate_tests.bzl", "grpc_end2end_tests") cc_library( - name = 'cq_verifier', - srcs = ['cq_verifier.c'], - hdrs = ['cq_verifier.h'], - deps = ['//:gpr', '//:grpc', '//test/core/util:grpc_test_util'], - copts = ['-std=c99'], - visibility = ["//test:__subpackages__"], + name = "cq_verifier", + srcs = ["cq_verifier.c"], + hdrs = ["cq_verifier.h"], + copts = ["-std=c99"], + visibility = ["//test:__subpackages__"], + deps = [ + "//:gpr", + "//:grpc", + "//test/core/util:grpc_test_util", + ], ) cc_library( - name = 'ssl_test_data', - visibility = ["//test:__subpackages__"], - hdrs = ['data/ssl_test_data.h'], - copts = ['-std=c99'], - srcs = [ - "data/client_certs.c", - "data/server1_cert.c", - "data/server1_key.c", - "data/test_root_cert.c", - ] + name = "ssl_test_data", + srcs = [ + "data/client_certs.c", + "data/server1_cert.c", + "data/server1_key.c", + "data/test_root_cert.c", + ], + hdrs = ["data/ssl_test_data.h"], + copts = ["-std=c99"], + visibility = ["//test:__subpackages__"], ) cc_library( - name = 'fake_resolver', - hdrs = ['fake_resolver.h'], - srcs = ['fake_resolver.c'], - copts = ['-std=c99'], - deps = ['//:gpr', '//:grpc', '//test/core/util:grpc_test_util'] + name = "fake_resolver", + srcs = ["fake_resolver.c"], + hdrs = ["fake_resolver.h"], + copts = ["-std=c99"], + visibility = ["//test:__subpackages__"], + deps = [ + "//:gpr", + "//:grpc", + "//test/core/util:grpc_test_util", + ], ) cc_library( - name = 'http_proxy', - hdrs = ['fixtures/http_proxy_fixture.h'], - srcs = ['fixtures/http_proxy_fixture.c'], - copts = ['-std=c99'], - deps = ['//:gpr', '//:grpc', '//test/core/util:grpc_test_util'] + name = "http_proxy", + srcs = ["fixtures/http_proxy_fixture.c"], + hdrs = ["fixtures/http_proxy_fixture.h"], + copts = ["-std=c99"], + deps = [ + "//:gpr", + "//:grpc", + "//test/core/util:grpc_test_util", + ], ) cc_library( - name = 'proxy', - hdrs = ['fixtures/proxy.h'], - srcs = ['fixtures/proxy.c'], - copts = ['-std=c99'], - deps = ['//:gpr', '//:grpc', '//test/core/util:grpc_test_util'] + name = "proxy", + srcs = ["fixtures/proxy.c"], + hdrs = ["fixtures/proxy.h"], + copts = ["-std=c99"], + deps = [ + "//:gpr", + "//:grpc", + "//test/core/util:grpc_test_util", + ], ) grpc_end2end_tests() diff --git a/test/core/end2end/fake_resolver.c b/test/core/end2end/fake_resolver.c index 1c7dd1339c1..6a71c20b804 100644 --- a/test/core/end2end/fake_resolver.c +++ b/test/core/end2end/fake_resolver.c @@ -32,6 +32,7 @@ // This is similar to the sockaddr resolver, except that it supports a // bunch of query args that are useful for dependency injection in tests. +#include #include #include #include @@ -46,12 +47,18 @@ #include "src/core/ext/filters/client_channel/parse_address.h" #include "src/core/ext/filters/client_channel/resolver_registry.h" #include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/iomgr/combiner.h" #include "src/core/lib/iomgr/resolve_address.h" #include "src/core/lib/iomgr/unix_sockets_posix.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_string_helpers.h" #include "src/core/lib/support/string.h" +#include "test/core/end2end/fake_resolver.h" + +#define GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR \ + "grpc.fake_resolver.response_generator" + // // fake_resolver // @@ -62,12 +69,11 @@ typedef struct { // passed-in parameters grpc_channel_args* channel_args; - grpc_lb_addresses* addresses; - // mutex guarding the rest of the state - gpr_mu mu; - // have we published? - bool published; + // If not NULL, the next set of resolution results to be returned to + // grpc_resolver_next_locked()'s closure. + grpc_channel_args* next_results; + // pending next completion, or NULL grpc_closure* next_completion; // target result address for next completion @@ -76,60 +82,137 @@ typedef struct { static void fake_resolver_destroy(grpc_exec_ctx* exec_ctx, grpc_resolver* gr) { fake_resolver* r = (fake_resolver*)gr; - gpr_mu_destroy(&r->mu); + grpc_channel_args_destroy(exec_ctx, r->next_results); grpc_channel_args_destroy(exec_ctx, r->channel_args); - grpc_lb_addresses_destroy(exec_ctx, r->addresses); gpr_free(r); } -static void fake_resolver_shutdown(grpc_exec_ctx* exec_ctx, - grpc_resolver* resolver) { +static void fake_resolver_shutdown_locked(grpc_exec_ctx* exec_ctx, + grpc_resolver* resolver) { fake_resolver* r = (fake_resolver*)resolver; - gpr_mu_lock(&r->mu); if (r->next_completion != NULL) { *r->target_result = NULL; grpc_closure_sched(exec_ctx, r->next_completion, GRPC_ERROR_NONE); r->next_completion = NULL; } - gpr_mu_unlock(&r->mu); } static void fake_resolver_maybe_finish_next_locked(grpc_exec_ctx* exec_ctx, fake_resolver* r) { - if (r->next_completion != NULL && !r->published) { - r->published = true; - grpc_arg arg = grpc_lb_addresses_create_channel_arg(r->addresses); + if (r->next_completion != NULL && r->next_results != NULL) { *r->target_result = - grpc_channel_args_copy_and_add(r->channel_args, &arg, 1); + grpc_channel_args_merge(r->channel_args, r->next_results); + grpc_channel_args_destroy(exec_ctx, r->next_results); grpc_closure_sched(exec_ctx, r->next_completion, GRPC_ERROR_NONE); r->next_completion = NULL; + r->next_results = NULL; } } -static void fake_resolver_channel_saw_error(grpc_exec_ctx* exec_ctx, - grpc_resolver* resolver) { +static void fake_resolver_channel_saw_error_locked(grpc_exec_ctx* exec_ctx, + grpc_resolver* resolver) { fake_resolver* r = (fake_resolver*)resolver; - gpr_mu_lock(&r->mu); - r->published = false; fake_resolver_maybe_finish_next_locked(exec_ctx, r); - gpr_mu_unlock(&r->mu); } -static void fake_resolver_next(grpc_exec_ctx* exec_ctx, grpc_resolver* resolver, - grpc_channel_args** target_result, - grpc_closure* on_complete) { +static void fake_resolver_next_locked(grpc_exec_ctx* exec_ctx, + grpc_resolver* resolver, + grpc_channel_args** target_result, + grpc_closure* on_complete) { fake_resolver* r = (fake_resolver*)resolver; - gpr_mu_lock(&r->mu); GPR_ASSERT(!r->next_completion); r->next_completion = on_complete; r->target_result = target_result; fake_resolver_maybe_finish_next_locked(exec_ctx, r); - gpr_mu_unlock(&r->mu); } static const grpc_resolver_vtable fake_resolver_vtable = { - fake_resolver_destroy, fake_resolver_shutdown, - fake_resolver_channel_saw_error, fake_resolver_next}; + fake_resolver_destroy, fake_resolver_shutdown_locked, + fake_resolver_channel_saw_error_locked, fake_resolver_next_locked}; + +struct grpc_fake_resolver_response_generator { + fake_resolver* resolver; // Set by the fake_resolver constructor to itself. + grpc_channel_args* next_response; + gpr_refcount refcount; +}; + +grpc_fake_resolver_response_generator* +grpc_fake_resolver_response_generator_create() { + grpc_fake_resolver_response_generator* generator = + gpr_zalloc(sizeof(*generator)); + gpr_ref_init(&generator->refcount, 1); + return generator; +} + +grpc_fake_resolver_response_generator* +grpc_fake_resolver_response_generator_ref( + grpc_fake_resolver_response_generator* generator) { + gpr_ref(&generator->refcount); + return generator; +} + +void grpc_fake_resolver_response_generator_unref( + grpc_fake_resolver_response_generator* generator) { + if (gpr_unref(&generator->refcount)) { + gpr_free(generator); + } +} + +static void set_response_cb(grpc_exec_ctx* exec_ctx, void* arg, + grpc_error* error) { + grpc_fake_resolver_response_generator* generator = arg; + fake_resolver* r = generator->resolver; + if (r->next_results != NULL) { + grpc_channel_args_destroy(exec_ctx, r->next_results); + } + r->next_results = generator->next_response; + fake_resolver_maybe_finish_next_locked(exec_ctx, r); +} + +void grpc_fake_resolver_response_generator_set_response( + grpc_exec_ctx* exec_ctx, grpc_fake_resolver_response_generator* generator, + grpc_channel_args* next_response) { + GPR_ASSERT(generator->resolver != NULL); + generator->next_response = grpc_channel_args_copy(next_response); + grpc_closure_sched( + exec_ctx, + grpc_closure_create( + set_response_cb, generator, + grpc_combiner_scheduler(generator->resolver->base.combiner, false)), + GRPC_ERROR_NONE); +} + +static void* response_generator_arg_copy(void* p) { + return grpc_fake_resolver_response_generator_ref(p); +} + +static void response_generator_arg_destroy(grpc_exec_ctx* exec_ctx, void* p) { + grpc_fake_resolver_response_generator_unref(p); +} + +static int response_generator_cmp(void* a, void* b) { return GPR_ICMP(a, b); } + +static const grpc_arg_pointer_vtable response_generator_arg_vtable = { + response_generator_arg_copy, response_generator_arg_destroy, + response_generator_cmp}; + +grpc_arg grpc_fake_resolver_response_generator_arg( + grpc_fake_resolver_response_generator* generator) { + grpc_arg arg; + arg.type = GRPC_ARG_POINTER; + arg.key = GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR; + arg.value.pointer.p = generator; + arg.value.pointer.vtable = &response_generator_arg_vtable; + return arg; +} + +grpc_fake_resolver_response_generator* +grpc_fake_resolver_get_response_generator(const grpc_channel_args* args) { + const grpc_arg* arg = + grpc_channel_args_find(args, GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR); + if (arg == NULL || arg->type != GRPC_ARG_POINTER) return NULL; + return arg->value.pointer.p; +} // // fake_resolver_factory @@ -139,81 +222,15 @@ static void fake_resolver_factory_ref(grpc_resolver_factory* factory) {} static void fake_resolver_factory_unref(grpc_resolver_factory* factory) {} -static void do_nothing(void* ignored) {} - static grpc_resolver* fake_resolver_create(grpc_exec_ctx* exec_ctx, grpc_resolver_factory* factory, grpc_resolver_args* args) { - if (0 != strcmp(args->uri->authority, "")) { - gpr_log(GPR_ERROR, "authority based uri's not supported by the %s scheme", - args->uri->scheme); - return NULL; - } - // Get lb_enabled arg. Anything other than "0" is interpreted as true. - const char* lb_enabled_qpart = - grpc_uri_get_query_arg(args->uri, "lb_enabled"); - const bool lb_enabled = - lb_enabled_qpart != NULL && strcmp("0", lb_enabled_qpart) != 0; - - // Get the balancer's names. - const char* balancer_names = - grpc_uri_get_query_arg(args->uri, "balancer_names"); - grpc_slice_buffer balancer_names_parts; - grpc_slice_buffer_init(&balancer_names_parts); - if (balancer_names != NULL) { - const grpc_slice balancer_names_slice = - grpc_slice_from_copied_string(balancer_names); - grpc_slice_split(balancer_names_slice, ",", &balancer_names_parts); - grpc_slice_unref(balancer_names_slice); - } - - // Construct addresses. - grpc_slice path_slice = - grpc_slice_new(args->uri->path, strlen(args->uri->path), do_nothing); - grpc_slice_buffer path_parts; - grpc_slice_buffer_init(&path_parts); - grpc_slice_split(path_slice, ",", &path_parts); - if (balancer_names_parts.count > 0 && - path_parts.count != balancer_names_parts.count) { - gpr_log(GPR_ERROR, - "Balancer names present but mismatched with number of addresses: " - "%lu balancer names != %lu addresses", - (unsigned long)balancer_names_parts.count, - (unsigned long)path_parts.count); - return NULL; - } - grpc_lb_addresses* addresses = - grpc_lb_addresses_create(path_parts.count, NULL /* user_data_vtable */); - bool errors_found = false; - for (size_t i = 0; i < addresses->num_addresses; i++) { - grpc_uri ith_uri = *args->uri; - char* part_str = grpc_slice_to_c_string(path_parts.slices[i]); - ith_uri.path = part_str; - if (!parse_ipv4(&ith_uri, &addresses->addresses[i].address)) { - errors_found = true; - } - gpr_free(part_str); - if (errors_found) break; - addresses->addresses[i].is_balancer = lb_enabled; - addresses->addresses[i].balancer_name = - balancer_names_parts.count > 0 - ? grpc_dump_slice(balancer_names_parts.slices[i], GPR_DUMP_ASCII) - : NULL; - } - grpc_slice_buffer_destroy_internal(exec_ctx, &path_parts); - grpc_slice_buffer_destroy_internal(exec_ctx, &balancer_names_parts); - grpc_slice_unref(path_slice); - if (errors_found) { - grpc_lb_addresses_destroy(exec_ctx, addresses); - return NULL; - } - // Instantiate resolver. - fake_resolver* r = gpr_malloc(sizeof(fake_resolver)); - memset(r, 0, sizeof(*r)); + fake_resolver* r = gpr_zalloc(sizeof(*r)); r->channel_args = grpc_channel_args_copy(args->args); - r->addresses = addresses; - gpr_mu_init(&r->mu); grpc_resolver_init(&r->base, &fake_resolver_vtable, args->combiner); + grpc_fake_resolver_response_generator* response_generator = + grpc_fake_resolver_get_response_generator(args->args); + if (response_generator != NULL) response_generator->resolver = r; return &r->base; } diff --git a/test/core/end2end/fake_resolver.h b/test/core/end2end/fake_resolver.h index 7a30347f301..447289adefa 100644 --- a/test/core/end2end/fake_resolver.h +++ b/test/core/end2end/fake_resolver.h @@ -32,8 +32,42 @@ #ifndef GRPC_TEST_CORE_END2END_FAKE_RESOLVER_H #define GRPC_TEST_CORE_END2END_FAKE_RESOLVER_H +#include "src/core/ext/filters/client_channel/lb_policy_factory.h" +#include "src/core/ext/filters/client_channel/uri_parser.h" +#include "src/core/lib/channel/channel_args.h" + #include "test/core/util/test_config.h" void grpc_fake_resolver_init(); +// Instances of \a grpc_fake_resolver_response_generator are passed to the +// fake resolver in a channel argument (see \a +// grpc_fake_resolver_response_generator_arg) in order to inject and trigger +// custom resolutions. See also \a +// grpc_fake_resolver_response_generator_set_response. +typedef struct grpc_fake_resolver_response_generator + grpc_fake_resolver_response_generator; +grpc_fake_resolver_response_generator* +grpc_fake_resolver_response_generator_create(); + +// Instruct the fake resolver associated with the \a response_generator instance +// to trigger a new resolution for \a uri and \a args. +void grpc_fake_resolver_response_generator_set_response( + grpc_exec_ctx* exec_ctx, grpc_fake_resolver_response_generator* generator, + grpc_channel_args* next_response); + +// Return a \a grpc_arg for a \a grpc_fake_resolver_response_generator instance. +grpc_arg grpc_fake_resolver_response_generator_arg( + grpc_fake_resolver_response_generator* generator); +// Return the \a grpc_fake_resolver_response_generator instance in \a args or +// NULL. +grpc_fake_resolver_response_generator* +grpc_fake_resolver_get_response_generator(const grpc_channel_args* args); + +grpc_fake_resolver_response_generator* +grpc_fake_resolver_response_generator_ref( + grpc_fake_resolver_response_generator* generator); +void grpc_fake_resolver_response_generator_unref( + grpc_fake_resolver_response_generator* generator); + #endif /* GRPC_TEST_CORE_END2END_FAKE_RESOLVER_H */ diff --git a/test/cpp/grpclb/grpclb_test.cc b/test/cpp/grpclb/grpclb_test.cc index a80431f46a1..997a8391eb7 100644 --- a/test/cpp/grpclb/grpclb_test.cc +++ b/test/cpp/grpclb/grpclb_test.cc @@ -573,34 +573,51 @@ static void perform_request(client_fixture *cf) { static void setup_client(const server_fixture *lb_server, const server_fixture *backends, client_fixture *cf) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - char *lb_uri; - // The grpclb LB policy will be automatically selected by virtue of - // the fact that the returned addresses are balancer addresses. - gpr_asprintf(&lb_uri, "test:///%s?lb_enabled=1&balancer_names=%s", - lb_server->servers_hostport, lb_server->balancer_name); - - grpc_arg expected_target_arg; - expected_target_arg.type = GRPC_ARG_STRING; - expected_target_arg.key = - const_cast(GRPC_ARG_FAKE_SECURITY_EXPECTED_TARGETS); char *expected_target_names = NULL; const char *backends_name = lb_server->servers_hostport; gpr_asprintf(&expected_target_names, "%s;%s", backends_name, BALANCERS_NAME); - expected_target_arg.value.string = const_cast(expected_target_names); + grpc_fake_resolver_response_generator *response_generator = + grpc_fake_resolver_response_generator_create(); + + grpc_lb_addresses *addresses = grpc_lb_addresses_create(1, NULL); + char *lb_uri_str; + gpr_asprintf(&lb_uri_str, "ipv4:%s", lb_server->servers_hostport); + grpc_uri *lb_uri = grpc_uri_parse(&exec_ctx, lb_uri_str, true); + GPR_ASSERT(lb_uri != NULL); + grpc_lb_addresses_set_address_from_uri(addresses, 0, lb_uri, true, + lb_server->balancer_name, NULL); + grpc_uri_destroy(lb_uri); + gpr_free(lb_uri_str); + + gpr_asprintf(&cf->server_uri, "test:///%s", lb_server->servers_hostport); + const grpc_arg fake_addresses = + grpc_lb_addresses_create_channel_arg(addresses); + grpc_channel_args *fake_result = + grpc_channel_args_copy_and_add(NULL, &fake_addresses, 1); + grpc_lb_addresses_destroy(&exec_ctx, addresses); + + const grpc_arg new_args[] = { + grpc_fake_transport_expected_targets_arg(expected_target_names), + grpc_fake_resolver_response_generator_arg(response_generator)}; + grpc_channel_args *args = - grpc_channel_args_copy_and_add(NULL, &expected_target_arg, 1); + grpc_channel_args_copy_and_add(NULL, new_args, GPR_ARRAY_SIZE(new_args)); gpr_free(expected_target_names); cf->cq = grpc_completion_queue_create_for_next(NULL); - cf->server_uri = lb_uri; grpc_channel_credentials *fake_creds = grpc_fake_transport_security_credentials_create(); cf->client = grpc_secure_channel_create(fake_creds, cf->server_uri, args, NULL); + grpc_fake_resolver_response_generator_set_response( + &exec_ctx, response_generator, fake_result); + grpc_channel_args_destroy(&exec_ctx, fake_result); grpc_channel_credentials_unref(&exec_ctx, fake_creds); grpc_channel_args_destroy(&exec_ctx, args); + grpc_fake_resolver_response_generator_unref(response_generator); + grpc_exec_ctx_finish(&exec_ctx); } static void teardown_client(client_fixture *cf) { @@ -787,8 +804,8 @@ TEST(GrpclbTest, InvalidAddressInServerlist) {} int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); - grpc_test_init(argc, argv); grpc_fake_resolver_init(); + grpc_test_init(argc, argv); grpc_init(); const auto result = RUN_ALL_TESTS(); grpc_shutdown(); diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 95d5649d000..aa4869e1c8a 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -470,6 +470,23 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c", + "name": "fake_resolver_test", + "src": [ + "test/core/client_channel/resolvers/fake_resolver_test.c" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index dde006737f2..6338ea70127 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -577,6 +577,28 @@ "linux" ] }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c", + "name": "fake_resolver_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, { "args": [], "ci_platforms": [ diff --git a/vsprojects/buildtests_c.sln b/vsprojects/buildtests_c.sln index 1e8336c54ef..6ae8bfa74de 100644 --- a/vsprojects/buildtests_c.sln +++ b/vsprojects/buildtests_c.sln @@ -317,6 +317,17 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "error_test", "vcxproj\test\ {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} EndProjectSection EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fake_resolver_test", "vcxproj\test\fake_resolver_test\fake_resolver_test.vcxproj", "{2F59EB2E-CEF9-A291-9480-1F737C3E5E57}" + ProjectSection(myProperties) = preProject + lib = "False" + EndProjectSection + ProjectSection(ProjectDependencies) = postProject + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} = {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + {29D16885-7228-4C31-81ED-5F9187C7F2A9} = {29D16885-7228-4C31-81ED-5F9187C7F2A9} + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} = {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + EndProjectSection +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fling_client", "vcxproj\test\fling_client\fling_client.vcxproj", "{0647D598-9611-F659-EA36-DF995C9F736B}" ProjectSection(myProperties) = preProject lib = "False" @@ -2116,6 +2127,22 @@ Global {42720233-A6D4-66BC-CCA2-06B57261D0B3}.Release-DLL|Win32.Build.0 = Release|Win32 {42720233-A6D4-66BC-CCA2-06B57261D0B3}.Release-DLL|x64.ActiveCfg = Release|x64 {42720233-A6D4-66BC-CCA2-06B57261D0B3}.Release-DLL|x64.Build.0 = Release|x64 + {2F59EB2E-CEF9-A291-9480-1F737C3E5E57}.Debug|Win32.ActiveCfg = Debug|Win32 + {2F59EB2E-CEF9-A291-9480-1F737C3E5E57}.Debug|x64.ActiveCfg = Debug|x64 + {2F59EB2E-CEF9-A291-9480-1F737C3E5E57}.Release|Win32.ActiveCfg = Release|Win32 + {2F59EB2E-CEF9-A291-9480-1F737C3E5E57}.Release|x64.ActiveCfg = Release|x64 + {2F59EB2E-CEF9-A291-9480-1F737C3E5E57}.Debug|Win32.Build.0 = Debug|Win32 + {2F59EB2E-CEF9-A291-9480-1F737C3E5E57}.Debug|x64.Build.0 = Debug|x64 + {2F59EB2E-CEF9-A291-9480-1F737C3E5E57}.Release|Win32.Build.0 = Release|Win32 + {2F59EB2E-CEF9-A291-9480-1F737C3E5E57}.Release|x64.Build.0 = Release|x64 + {2F59EB2E-CEF9-A291-9480-1F737C3E5E57}.Debug-DLL|Win32.ActiveCfg = Debug|Win32 + {2F59EB2E-CEF9-A291-9480-1F737C3E5E57}.Debug-DLL|Win32.Build.0 = Debug|Win32 + {2F59EB2E-CEF9-A291-9480-1F737C3E5E57}.Debug-DLL|x64.ActiveCfg = Debug|x64 + {2F59EB2E-CEF9-A291-9480-1F737C3E5E57}.Debug-DLL|x64.Build.0 = Debug|x64 + {2F59EB2E-CEF9-A291-9480-1F737C3E5E57}.Release-DLL|Win32.ActiveCfg = Release|Win32 + {2F59EB2E-CEF9-A291-9480-1F737C3E5E57}.Release-DLL|Win32.Build.0 = Release|Win32 + {2F59EB2E-CEF9-A291-9480-1F737C3E5E57}.Release-DLL|x64.ActiveCfg = Release|x64 + {2F59EB2E-CEF9-A291-9480-1F737C3E5E57}.Release-DLL|x64.Build.0 = Release|x64 {0647D598-9611-F659-EA36-DF995C9F736B}.Debug|Win32.ActiveCfg = Debug|Win32 {0647D598-9611-F659-EA36-DF995C9F736B}.Debug|x64.ActiveCfg = Debug|x64 {0647D598-9611-F659-EA36-DF995C9F736B}.Release|Win32.ActiveCfg = Release|Win32 diff --git a/vsprojects/vcxproj/test/fake_resolver_test/fake_resolver_test.vcxproj b/vsprojects/vcxproj/test/fake_resolver_test/fake_resolver_test.vcxproj new file mode 100644 index 00000000000..8ea721dd7b4 --- /dev/null +++ b/vsprojects/vcxproj/test/fake_resolver_test/fake_resolver_test.vcxproj @@ -0,0 +1,199 @@ + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {2F59EB2E-CEF9-A291-9480-1F737C3E5E57} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + Application + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + fake_resolver_test + static + Debug + static + Debug + + + fake_resolver_test + static + Release + static + Release + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + + + + + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + + + {29D16885-7228-4C31-81ED-5F9187C7F2A9} + + + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + + + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + diff --git a/vsprojects/vcxproj/test/fake_resolver_test/fake_resolver_test.vcxproj.filters b/vsprojects/vcxproj/test/fake_resolver_test/fake_resolver_test.vcxproj.filters new file mode 100644 index 00000000000..353cae2b078 --- /dev/null +++ b/vsprojects/vcxproj/test/fake_resolver_test/fake_resolver_test.vcxproj.filters @@ -0,0 +1,24 @@ + + + + + test\core\client_channel\resolvers + + + + + + {05f58640-4b7c-c233-bf86-f119fe270ba1} + + + {caeef88d-baf6-603c-83b0-84b1009be480} + + + {c5bba556-fd85-f7b4-99b7-8b1eeb4dfcb2} + + + {eff74a86-6a4c-b68c-0330-6901d992c5c7} + + + + From a7a1098d2f4a88a038e61a949ee98bd2e5444b7a Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Fri, 14 Apr 2017 14:39:37 -0700 Subject: [PATCH 54/60] Updated load_balancer.proto --- .../proto/grpc/lb/v1/load_balancer.pb.c | 27 ++++-- .../proto/grpc/lb/v1/load_balancer.pb.h | 97 ++++++++++++------- src/proto/grpc/lb/v1/load_balancer.proto | 57 +++++++++-- test/cpp/grpclb/grpclb_api_test.cc | 12 ++- 4 files changed, 137 insertions(+), 56 deletions(-) diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c index e9adf98711e..fb119c7fc88 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.c @@ -16,6 +16,12 @@ const pb_field_t grpc_lb_v1_Duration_fields[3] = { PB_LAST_FIELD }; +const pb_field_t grpc_lb_v1_Timestamp_fields[3] = { + PB_FIELD( 1, INT64 , OPTIONAL, STATIC , FIRST, grpc_lb_v1_Timestamp, seconds, seconds, 0), + PB_FIELD( 2, INT32 , OPTIONAL, STATIC , OTHER, grpc_lb_v1_Timestamp, nanos, seconds, 0), + PB_LAST_FIELD +}; + const pb_field_t grpc_lb_v1_LoadBalanceRequest_fields[3] = { PB_FIELD( 1, MESSAGE , OPTIONAL, STATIC , FIRST, grpc_lb_v1_LoadBalanceRequest, initial_request, initial_request, &grpc_lb_v1_InitialLoadBalanceRequest_fields), PB_FIELD( 2, MESSAGE , OPTIONAL, STATIC , OTHER, grpc_lb_v1_LoadBalanceRequest, client_stats, initial_request, &grpc_lb_v1_ClientStats_fields), @@ -27,10 +33,14 @@ const pb_field_t grpc_lb_v1_InitialLoadBalanceRequest_fields[2] = { PB_LAST_FIELD }; -const pb_field_t grpc_lb_v1_ClientStats_fields[4] = { - PB_FIELD( 1, INT64 , OPTIONAL, STATIC , FIRST, grpc_lb_v1_ClientStats, total_requests, total_requests, 0), - PB_FIELD( 2, INT64 , OPTIONAL, STATIC , OTHER, grpc_lb_v1_ClientStats, client_rpc_errors, total_requests, 0), - PB_FIELD( 3, INT64 , OPTIONAL, STATIC , OTHER, grpc_lb_v1_ClientStats, dropped_requests, client_rpc_errors, 0), +const pb_field_t grpc_lb_v1_ClientStats_fields[8] = { + PB_FIELD( 1, MESSAGE , OPTIONAL, STATIC , FIRST, grpc_lb_v1_ClientStats, timestamp, timestamp, &grpc_lb_v1_Timestamp_fields), + PB_FIELD( 2, INT64 , OPTIONAL, STATIC , OTHER, grpc_lb_v1_ClientStats, num_calls_started, timestamp, 0), + PB_FIELD( 3, INT64 , OPTIONAL, STATIC , OTHER, grpc_lb_v1_ClientStats, num_calls_finished, num_calls_started, 0), + PB_FIELD( 4, INT64 , OPTIONAL, STATIC , OTHER, grpc_lb_v1_ClientStats, num_calls_finished_with_drop_for_rate_limiting, num_calls_finished, 0), + PB_FIELD( 5, INT64 , OPTIONAL, STATIC , OTHER, grpc_lb_v1_ClientStats, num_calls_finished_with_drop_for_load_balancing, num_calls_finished_with_drop_for_rate_limiting, 0), + PB_FIELD( 6, INT64 , OPTIONAL, STATIC , OTHER, grpc_lb_v1_ClientStats, num_calls_finished_with_client_failed_to_send, num_calls_finished_with_drop_for_load_balancing, 0), + PB_FIELD( 7, INT64 , OPTIONAL, STATIC , OTHER, grpc_lb_v1_ClientStats, num_calls_finished_known_received, num_calls_finished_with_client_failed_to_send, 0), PB_LAST_FIELD }; @@ -52,11 +62,12 @@ const pb_field_t grpc_lb_v1_ServerList_fields[3] = { PB_LAST_FIELD }; -const pb_field_t grpc_lb_v1_Server_fields[5] = { +const pb_field_t grpc_lb_v1_Server_fields[6] = { PB_FIELD( 1, BYTES , OPTIONAL, STATIC , FIRST, grpc_lb_v1_Server, ip_address, ip_address, 0), PB_FIELD( 2, INT32 , OPTIONAL, STATIC , OTHER, grpc_lb_v1_Server, port, ip_address, 0), PB_FIELD( 3, STRING , OPTIONAL, STATIC , OTHER, grpc_lb_v1_Server, load_balance_token, port, 0), - PB_FIELD( 4, BOOL , OPTIONAL, STATIC , OTHER, grpc_lb_v1_Server, drop_request, load_balance_token, 0), + PB_FIELD( 4, BOOL , OPTIONAL, STATIC , OTHER, grpc_lb_v1_Server, drop_for_rate_limiting, load_balance_token, 0), + PB_FIELD( 5, BOOL , OPTIONAL, STATIC , OTHER, grpc_lb_v1_Server, drop_for_load_balancing, drop_for_rate_limiting, 0), PB_LAST_FIELD }; @@ -70,7 +81,7 @@ const pb_field_t grpc_lb_v1_Server_fields[5] = { * numbers or field sizes that are larger than what can fit in 8 or 16 bit * field descriptors. */ -PB_STATIC_ASSERT((pb_membersize(grpc_lb_v1_LoadBalanceRequest, initial_request) < 65536 && pb_membersize(grpc_lb_v1_LoadBalanceRequest, client_stats) < 65536 && pb_membersize(grpc_lb_v1_LoadBalanceResponse, initial_response) < 65536 && pb_membersize(grpc_lb_v1_LoadBalanceResponse, server_list) < 65536 && pb_membersize(grpc_lb_v1_InitialLoadBalanceResponse, client_stats_report_interval) < 65536 && pb_membersize(grpc_lb_v1_ServerList, servers) < 65536 && pb_membersize(grpc_lb_v1_ServerList, expiration_interval) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_grpc_lb_v1_Duration_grpc_lb_v1_LoadBalanceRequest_grpc_lb_v1_InitialLoadBalanceRequest_grpc_lb_v1_ClientStats_grpc_lb_v1_LoadBalanceResponse_grpc_lb_v1_InitialLoadBalanceResponse_grpc_lb_v1_ServerList_grpc_lb_v1_Server) +PB_STATIC_ASSERT((pb_membersize(grpc_lb_v1_LoadBalanceRequest, initial_request) < 65536 && pb_membersize(grpc_lb_v1_LoadBalanceRequest, client_stats) < 65536 && pb_membersize(grpc_lb_v1_ClientStats, timestamp) < 65536 && pb_membersize(grpc_lb_v1_LoadBalanceResponse, initial_response) < 65536 && pb_membersize(grpc_lb_v1_LoadBalanceResponse, server_list) < 65536 && pb_membersize(grpc_lb_v1_InitialLoadBalanceResponse, client_stats_report_interval) < 65536 && pb_membersize(grpc_lb_v1_ServerList, servers) < 65536 && pb_membersize(grpc_lb_v1_ServerList, expiration_interval) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_grpc_lb_v1_Duration_grpc_lb_v1_Timestamp_grpc_lb_v1_LoadBalanceRequest_grpc_lb_v1_InitialLoadBalanceRequest_grpc_lb_v1_ClientStats_grpc_lb_v1_LoadBalanceResponse_grpc_lb_v1_InitialLoadBalanceResponse_grpc_lb_v1_ServerList_grpc_lb_v1_Server) #endif #if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT) @@ -81,7 +92,7 @@ PB_STATIC_ASSERT((pb_membersize(grpc_lb_v1_LoadBalanceRequest, initial_request) * numbers or field sizes that are larger than what can fit in the default * 8 bit descriptors. */ -PB_STATIC_ASSERT((pb_membersize(grpc_lb_v1_LoadBalanceRequest, initial_request) < 256 && pb_membersize(grpc_lb_v1_LoadBalanceRequest, client_stats) < 256 && pb_membersize(grpc_lb_v1_LoadBalanceResponse, initial_response) < 256 && pb_membersize(grpc_lb_v1_LoadBalanceResponse, server_list) < 256 && pb_membersize(grpc_lb_v1_InitialLoadBalanceResponse, client_stats_report_interval) < 256 && pb_membersize(grpc_lb_v1_ServerList, servers) < 256 && pb_membersize(grpc_lb_v1_ServerList, expiration_interval) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_grpc_lb_v1_Duration_grpc_lb_v1_LoadBalanceRequest_grpc_lb_v1_InitialLoadBalanceRequest_grpc_lb_v1_ClientStats_grpc_lb_v1_LoadBalanceResponse_grpc_lb_v1_InitialLoadBalanceResponse_grpc_lb_v1_ServerList_grpc_lb_v1_Server) +PB_STATIC_ASSERT((pb_membersize(grpc_lb_v1_LoadBalanceRequest, initial_request) < 256 && pb_membersize(grpc_lb_v1_LoadBalanceRequest, client_stats) < 256 && pb_membersize(grpc_lb_v1_ClientStats, timestamp) < 256 && pb_membersize(grpc_lb_v1_LoadBalanceResponse, initial_response) < 256 && pb_membersize(grpc_lb_v1_LoadBalanceResponse, server_list) < 256 && pb_membersize(grpc_lb_v1_InitialLoadBalanceResponse, client_stats_report_interval) < 256 && pb_membersize(grpc_lb_v1_ServerList, servers) < 256 && pb_membersize(grpc_lb_v1_ServerList, expiration_interval) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_grpc_lb_v1_Duration_grpc_lb_v1_Timestamp_grpc_lb_v1_LoadBalanceRequest_grpc_lb_v1_InitialLoadBalanceRequest_grpc_lb_v1_ClientStats_grpc_lb_v1_LoadBalanceResponse_grpc_lb_v1_InitialLoadBalanceResponse_grpc_lb_v1_ServerList_grpc_lb_v1_Server) #endif diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h b/src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h index 725aa7e386f..d3ae919ec27 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h @@ -14,16 +14,6 @@ extern "C" { #endif /* Struct definitions */ -typedef struct _grpc_lb_v1_ClientStats { - bool has_total_requests; - int64_t total_requests; - bool has_client_rpc_errors; - int64_t client_rpc_errors; - bool has_dropped_requests; - int64_t dropped_requests; -/* @@protoc_insertion_point(struct:grpc_lb_v1_ClientStats) */ -} grpc_lb_v1_ClientStats; - typedef struct _grpc_lb_v1_Duration { bool has_seconds; int64_t seconds; @@ -46,11 +36,39 @@ typedef struct _grpc_lb_v1_Server { int32_t port; bool has_load_balance_token; char load_balance_token[50]; - bool has_drop_request; - bool drop_request; + bool has_drop_for_rate_limiting; + bool drop_for_rate_limiting; + bool has_drop_for_load_balancing; + bool drop_for_load_balancing; /* @@protoc_insertion_point(struct:grpc_lb_v1_Server) */ } grpc_lb_v1_Server; +typedef struct _grpc_lb_v1_Timestamp { + bool has_seconds; + int64_t seconds; + bool has_nanos; + int32_t nanos; +/* @@protoc_insertion_point(struct:grpc_lb_v1_Timestamp) */ +} grpc_lb_v1_Timestamp; + +typedef struct _grpc_lb_v1_ClientStats { + bool has_timestamp; + grpc_lb_v1_Timestamp timestamp; + bool has_num_calls_started; + int64_t num_calls_started; + bool has_num_calls_finished; + int64_t num_calls_finished; + bool has_num_calls_finished_with_drop_for_rate_limiting; + int64_t num_calls_finished_with_drop_for_rate_limiting; + bool has_num_calls_finished_with_drop_for_load_balancing; + int64_t num_calls_finished_with_drop_for_load_balancing; + bool has_num_calls_finished_with_client_failed_to_send; + int64_t num_calls_finished_with_client_failed_to_send; + bool has_num_calls_finished_known_received; + int64_t num_calls_finished_known_received; +/* @@protoc_insertion_point(struct:grpc_lb_v1_ClientStats) */ +} grpc_lb_v1_ClientStats; + typedef struct _grpc_lb_v1_InitialLoadBalanceResponse { bool has_load_balancer_delegate; char load_balancer_delegate[64]; @@ -59,6 +77,13 @@ typedef struct _grpc_lb_v1_InitialLoadBalanceResponse { /* @@protoc_insertion_point(struct:grpc_lb_v1_InitialLoadBalanceResponse) */ } grpc_lb_v1_InitialLoadBalanceResponse; +typedef struct _grpc_lb_v1_ServerList { + pb_callback_t servers; + bool has_expiration_interval; + grpc_lb_v1_Duration expiration_interval; +/* @@protoc_insertion_point(struct:grpc_lb_v1_ServerList) */ +} grpc_lb_v1_ServerList; + typedef struct _grpc_lb_v1_LoadBalanceRequest { bool has_initial_request; grpc_lb_v1_InitialLoadBalanceRequest initial_request; @@ -67,13 +92,6 @@ typedef struct _grpc_lb_v1_LoadBalanceRequest { /* @@protoc_insertion_point(struct:grpc_lb_v1_LoadBalanceRequest) */ } grpc_lb_v1_LoadBalanceRequest; -typedef struct _grpc_lb_v1_ServerList { - pb_callback_t servers; - bool has_expiration_interval; - grpc_lb_v1_Duration expiration_interval; -/* @@protoc_insertion_point(struct:grpc_lb_v1_ServerList) */ -} grpc_lb_v1_ServerList; - typedef struct _grpc_lb_v1_LoadBalanceResponse { bool has_initial_response; grpc_lb_v1_InitialLoadBalanceResponse initial_response; @@ -86,61 +104,72 @@ typedef struct _grpc_lb_v1_LoadBalanceResponse { /* Initializer values for message structs */ #define grpc_lb_v1_Duration_init_default {false, 0, false, 0} +#define grpc_lb_v1_Timestamp_init_default {false, 0, false, 0} #define grpc_lb_v1_LoadBalanceRequest_init_default {false, grpc_lb_v1_InitialLoadBalanceRequest_init_default, false, grpc_lb_v1_ClientStats_init_default} #define grpc_lb_v1_InitialLoadBalanceRequest_init_default {false, ""} -#define grpc_lb_v1_ClientStats_init_default {false, 0, false, 0, false, 0} +#define grpc_lb_v1_ClientStats_init_default {false, grpc_lb_v1_Timestamp_init_default, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0} #define grpc_lb_v1_LoadBalanceResponse_init_default {false, grpc_lb_v1_InitialLoadBalanceResponse_init_default, false, grpc_lb_v1_ServerList_init_default} #define grpc_lb_v1_InitialLoadBalanceResponse_init_default {false, "", false, grpc_lb_v1_Duration_init_default} #define grpc_lb_v1_ServerList_init_default {{{NULL}, NULL}, false, grpc_lb_v1_Duration_init_default} -#define grpc_lb_v1_Server_init_default {false, {0, {0}}, false, 0, false, "", false, 0} +#define grpc_lb_v1_Server_init_default {false, {0, {0}}, false, 0, false, "", false, 0, false, 0} #define grpc_lb_v1_Duration_init_zero {false, 0, false, 0} +#define grpc_lb_v1_Timestamp_init_zero {false, 0, false, 0} #define grpc_lb_v1_LoadBalanceRequest_init_zero {false, grpc_lb_v1_InitialLoadBalanceRequest_init_zero, false, grpc_lb_v1_ClientStats_init_zero} #define grpc_lb_v1_InitialLoadBalanceRequest_init_zero {false, ""} -#define grpc_lb_v1_ClientStats_init_zero {false, 0, false, 0, false, 0} +#define grpc_lb_v1_ClientStats_init_zero {false, grpc_lb_v1_Timestamp_init_zero, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0} #define grpc_lb_v1_LoadBalanceResponse_init_zero {false, grpc_lb_v1_InitialLoadBalanceResponse_init_zero, false, grpc_lb_v1_ServerList_init_zero} #define grpc_lb_v1_InitialLoadBalanceResponse_init_zero {false, "", false, grpc_lb_v1_Duration_init_zero} #define grpc_lb_v1_ServerList_init_zero {{{NULL}, NULL}, false, grpc_lb_v1_Duration_init_zero} -#define grpc_lb_v1_Server_init_zero {false, {0, {0}}, false, 0, false, "", false, 0} +#define grpc_lb_v1_Server_init_zero {false, {0, {0}}, false, 0, false, "", false, 0, false, 0} /* Field tags (for use in manual encoding/decoding) */ -#define grpc_lb_v1_ClientStats_total_requests_tag 1 -#define grpc_lb_v1_ClientStats_client_rpc_errors_tag 2 -#define grpc_lb_v1_ClientStats_dropped_requests_tag 3 #define grpc_lb_v1_Duration_seconds_tag 1 #define grpc_lb_v1_Duration_nanos_tag 2 #define grpc_lb_v1_InitialLoadBalanceRequest_name_tag 1 #define grpc_lb_v1_Server_ip_address_tag 1 #define grpc_lb_v1_Server_port_tag 2 #define grpc_lb_v1_Server_load_balance_token_tag 3 -#define grpc_lb_v1_Server_drop_request_tag 4 +#define grpc_lb_v1_Server_drop_for_rate_limiting_tag 4 +#define grpc_lb_v1_Server_drop_for_load_balancing_tag 5 +#define grpc_lb_v1_Timestamp_seconds_tag 1 +#define grpc_lb_v1_Timestamp_nanos_tag 2 +#define grpc_lb_v1_ClientStats_timestamp_tag 1 +#define grpc_lb_v1_ClientStats_num_calls_started_tag 2 +#define grpc_lb_v1_ClientStats_num_calls_finished_tag 3 +#define grpc_lb_v1_ClientStats_num_calls_finished_with_drop_for_rate_limiting_tag 4 +#define grpc_lb_v1_ClientStats_num_calls_finished_with_drop_for_load_balancing_tag 5 +#define grpc_lb_v1_ClientStats_num_calls_finished_with_client_failed_to_send_tag 6 +#define grpc_lb_v1_ClientStats_num_calls_finished_known_received_tag 7 #define grpc_lb_v1_InitialLoadBalanceResponse_load_balancer_delegate_tag 1 #define grpc_lb_v1_InitialLoadBalanceResponse_client_stats_report_interval_tag 2 -#define grpc_lb_v1_LoadBalanceRequest_initial_request_tag 1 -#define grpc_lb_v1_LoadBalanceRequest_client_stats_tag 2 #define grpc_lb_v1_ServerList_servers_tag 1 #define grpc_lb_v1_ServerList_expiration_interval_tag 3 +#define grpc_lb_v1_LoadBalanceRequest_initial_request_tag 1 +#define grpc_lb_v1_LoadBalanceRequest_client_stats_tag 2 #define grpc_lb_v1_LoadBalanceResponse_initial_response_tag 1 #define grpc_lb_v1_LoadBalanceResponse_server_list_tag 2 /* Struct field encoding specification for nanopb */ extern const pb_field_t grpc_lb_v1_Duration_fields[3]; +extern const pb_field_t grpc_lb_v1_Timestamp_fields[3]; extern const pb_field_t grpc_lb_v1_LoadBalanceRequest_fields[3]; extern const pb_field_t grpc_lb_v1_InitialLoadBalanceRequest_fields[2]; -extern const pb_field_t grpc_lb_v1_ClientStats_fields[4]; +extern const pb_field_t grpc_lb_v1_ClientStats_fields[8]; extern const pb_field_t grpc_lb_v1_LoadBalanceResponse_fields[3]; extern const pb_field_t grpc_lb_v1_InitialLoadBalanceResponse_fields[3]; extern const pb_field_t grpc_lb_v1_ServerList_fields[3]; -extern const pb_field_t grpc_lb_v1_Server_fields[5]; +extern const pb_field_t grpc_lb_v1_Server_fields[6]; /* Maximum encoded size of messages (where known) */ #define grpc_lb_v1_Duration_size 22 -#define grpc_lb_v1_LoadBalanceRequest_size 169 +#define grpc_lb_v1_Timestamp_size 22 +#define grpc_lb_v1_LoadBalanceRequest_size 226 #define grpc_lb_v1_InitialLoadBalanceRequest_size 131 -#define grpc_lb_v1_ClientStats_size 33 +#define grpc_lb_v1_ClientStats_size 90 #define grpc_lb_v1_LoadBalanceResponse_size (98 + grpc_lb_v1_ServerList_size) #define grpc_lb_v1_InitialLoadBalanceResponse_size 90 /* grpc_lb_v1_ServerList_size depends on runtime parameters */ -#define grpc_lb_v1_Server_size 83 +#define grpc_lb_v1_Server_size 85 /* Message IDs (where set with "msgid" option) */ #ifdef PB_MSGID diff --git a/src/proto/grpc/lb/v1/load_balancer.proto b/src/proto/grpc/lb/v1/load_balancer.proto index 44a5150a7e0..a2502fb284a 100644 --- a/src/proto/grpc/lb/v1/load_balancer.proto +++ b/src/proto/grpc/lb/v1/load_balancer.proto @@ -45,6 +45,20 @@ message Duration { int32 nanos = 2; } +message Timestamp { + + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + int64 seconds = 1; + + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + int32 nanos = 2; +} + service LoadBalancer { // Bidirectional rpc to get a list of servers. rpc BalanceLoad(stream LoadBalanceRequest) @@ -63,22 +77,37 @@ message LoadBalanceRequest { } message InitialLoadBalanceRequest { - // Name of load balanced service (IE, service.grpc.gslb.google.com). Its + // Name of load balanced service (IE, balancer.service.com) // length should be less than 256 bytes. string name = 1; } // Contains client level statistics that are useful to load balancing. Each -// count should be reset to zero after reporting the stats. +// count except the timestamp should be reset to zero after reporting the stats. message ClientStats { - // The total number of requests sent by the client since the last report. - int64 total_requests = 1; + // The timestamp of generating the report. + Timestamp timestamp = 1; - // The number of client rpc errors since the last report. - int64 client_rpc_errors = 2; + // The total number of RPCs that started. + int64 num_calls_started = 2; - // The number of dropped requests since the last report. - int64 dropped_requests = 3; + // The total number of RPCs that finished. + int64 num_calls_finished = 3; + + // The total number of RPCs that were dropped by the client because of rate + // limiting. + int64 num_calls_finished_with_drop_for_rate_limiting = 4; + + // The total number of RPCs that were dropped by the client because of load + // balancing. + int64 num_calls_finished_with_drop_for_load_balancing = 5; + + // The total number of RPCs that failed to reach a server except dropped RPCs. + int64 num_calls_finished_with_client_failed_to_send = 6; + + // The total number of RPCs that finished and are known to have been received + // by a server. + int64 num_calls_finished_known_received = 7; } message LoadBalanceResponse { @@ -120,6 +149,10 @@ message ServerList { Duration expiration_interval = 3; } +// Contains server information. When none of the [drop_for_*] fields are true, +// use the other fields. When drop_for_rate_limiting is true, ignore all other +// fields. Use drop_for_load_balancing only when it is true and +// drop_for_rate_limiting is false. message Server { // A resolved address for the server, serialized in network-byte-order. It may // either be an IPv4 or IPv6 address. @@ -137,6 +170,10 @@ message Server { string load_balance_token = 3; // Indicates whether this particular request should be dropped by the client - // when this server is chosen from the list. - bool drop_request = 4; + // for rate limiting. + bool drop_for_rate_limiting = 4; + + // Indicates whether this particular request should be dropped by the client + // for load balancing. + bool drop_for_load_balancing = 5; } diff --git a/test/cpp/grpclb/grpclb_api_test.cc b/test/cpp/grpclb/grpclb_api_test.cc index d9df2bb6735..0b569f14157 100644 --- a/test/cpp/grpclb/grpclb_api_test.cc +++ b/test/cpp/grpclb/grpclb_api_test.cc @@ -106,11 +106,13 @@ TEST_F(GrpclbTest, ParseResponseServerList) { auto* server = serverlist->add_servers(); server->set_ip_address(Ip4ToPackedString("127.0.0.1")); server->set_port(12345); - server->set_drop_request(true); + server->set_drop_for_rate_limiting(true); + server->set_drop_for_load_balancing(false); server = response.mutable_server_list()->add_servers(); server->set_ip_address(Ip4ToPackedString("10.0.0.1")); server->set_port(54321); - server->set_drop_request(false); + server->set_drop_for_rate_limiting(false); + server->set_drop_for_load_balancing(true); auto* expiration_interval = serverlist->mutable_expiration_interval(); expiration_interval->set_seconds(888); expiration_interval->set_nanos(999); @@ -125,12 +127,14 @@ TEST_F(GrpclbTest, ParseResponseServerList) { EXPECT_EQ(PackedStringToIp(c_serverlist->servers[0]->ip_address), "127.0.0.1"); EXPECT_EQ(c_serverlist->servers[0]->port, 12345); - EXPECT_TRUE(c_serverlist->servers[0]->drop_request); + EXPECT_TRUE(c_serverlist->servers[0]->drop_for_rate_limiting); + EXPECT_FALSE(c_serverlist->servers[0]->drop_for_load_balancing); EXPECT_TRUE(c_serverlist->servers[1]->has_ip_address); EXPECT_EQ(PackedStringToIp(c_serverlist->servers[1]->ip_address), "10.0.0.1"); EXPECT_EQ(c_serverlist->servers[1]->port, 54321); - EXPECT_FALSE(c_serverlist->servers[1]->drop_request); + EXPECT_FALSE(c_serverlist->servers[1]->drop_for_rate_limiting); + EXPECT_TRUE(c_serverlist->servers[1]->drop_for_load_balancing); EXPECT_TRUE(c_serverlist->expiration_interval.has_seconds); EXPECT_EQ(c_serverlist->expiration_interval.seconds, 888); From a818f72c0c84f386d8e415e12f215453d35eeb21 Mon Sep 17 00:00:00 2001 From: David Garcia Quintas Date: Fri, 14 Apr 2017 16:01:14 -0700 Subject: [PATCH 55/60] Load Reporting back to using metadata --- grpc.def | 1 - include/grpc/grpc.h | 6 - include/grpc/load_reporting.h | 12 +- .../filters/load_reporting/load_reporting.c | 17 -- .../load_reporting/load_reporting_filter.c | 27 ++- src/core/lib/channel/context.h | 3 - src/core/lib/transport/static_metadata.c | 195 +++++++++--------- src/core/lib/transport/static_metadata.h | 97 ++++----- src/cpp/server/server_context.cc | 12 +- src/ruby/ext/grpc/rb_grpc_imports.generated.c | 2 - src/ruby/ext/grpc/rb_grpc_imports.generated.h | 3 - test/core/end2end/fuzzers/hpack.dictionary | 2 + test/core/end2end/tests/load_reporting_hook.c | 22 +- tools/codegen/core/gen_static_metadata.py | 1 + 14 files changed, 201 insertions(+), 199 deletions(-) diff --git a/grpc.def b/grpc.def index 1589316a588..841d55f341d 100644 --- a/grpc.def +++ b/grpc.def @@ -72,7 +72,6 @@ EXPORTS grpc_channel_create_registered_call grpc_call_start_batch grpc_call_get_peer - grpc_call_set_load_reporting_cost_context grpc_census_call_set_context grpc_census_call_get_context grpc_channel_get_target diff --git a/include/grpc/grpc.h b/include/grpc/grpc.h index 42cf201da44..9a65d34ce34 100644 --- a/include/grpc/grpc.h +++ b/include/grpc/grpc.h @@ -296,12 +296,6 @@ GRPCAPI grpc_call_error grpc_call_start_batch(grpc_call *call, functionality. Instead, use grpc_auth_context. */ GRPCAPI char *grpc_call_get_peer(grpc_call *call); -struct grpc_load_reporting_cost_context; - -/* Associate costs contained in \a cost_context to \a call. */ -GRPCAPI void grpc_call_set_load_reporting_cost_context( - grpc_call *call, struct grpc_load_reporting_cost_context *context); - struct census_context; /** Set census context for a call; Must be called before first call to diff --git a/include/grpc/load_reporting.h b/include/grpc/load_reporting.h index c833ce5c63b..a334a113507 100644 --- a/include/grpc/load_reporting.h +++ b/include/grpc/load_reporting.h @@ -35,7 +35,6 @@ #define GRPC_LOAD_REPORTING_H #include -#include #ifdef __cplusplus extern "C" { @@ -50,11 +49,12 @@ extern "C" { * gRPC LB system. */ #define GRPC_LB_TOKEN_MD_KEY "lb-token" -/** A sequence of values for load reporting purposes */ -typedef struct grpc_load_reporting_cost_context { - grpc_slice *values; - size_t values_count; -} grpc_load_reporting_cost_context; +/** Metadata key for gRPC LB cost reporting. + * + * The value corresponding to this key is an opaque binary blob reported by the + * backend as part of its trailing metadata containing cost information for the + * call. */ +#define GRPC_LB_COST_MD_KEY "lb-cost-bin" #ifdef __cplusplus } diff --git a/src/core/ext/filters/load_reporting/load_reporting.c b/src/core/ext/filters/load_reporting/load_reporting.c index 3b5d2103c1f..4e9d0937ae0 100644 --- a/src/core/ext/filters/load_reporting/load_reporting.c +++ b/src/core/ext/filters/load_reporting/load_reporting.c @@ -47,23 +47,6 @@ #include "src/core/lib/surface/call.h" #include "src/core/lib/surface/channel_init.h" -static void destroy_lr_cost_context(void *c) { - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_load_reporting_cost_context *cost_ctx = c; - for (size_t i = 0; i < cost_ctx->values_count; ++i) { - grpc_slice_unref_internal(&exec_ctx, cost_ctx->values[i]); - } - grpc_exec_ctx_finish(&exec_ctx); - gpr_free(cost_ctx->values); - gpr_free(cost_ctx); -} - -void grpc_call_set_load_reporting_cost_context( - grpc_call *call, grpc_load_reporting_cost_context *ctx) { - grpc_call_context_set(call, GRPC_CONTEXT_LR_COST, ctx, - destroy_lr_cost_context); -} - static bool is_load_reporting_enabled(const grpc_channel_args *a) { return grpc_channel_arg_get_bool( grpc_channel_args_find(a, GRPC_ARG_ENABLE_LOAD_REPORTING), false); diff --git a/src/core/ext/filters/load_reporting/load_reporting_filter.c b/src/core/ext/filters/load_reporting/load_reporting_filter.c index 57b25d06518..75a9a566871 100644 --- a/src/core/ext/filters/load_reporting/load_reporting_filter.c +++ b/src/core/ext/filters/load_reporting/load_reporting_filter.c @@ -48,6 +48,8 @@ typedef struct call_data { intptr_t id; /**< an id unique to the call */ + bool have_trailing_md_string; + grpc_slice trailing_md_string; bool have_initial_md_string; grpc_slice initial_md_string; bool have_service_method; @@ -140,6 +142,9 @@ static void destroy_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, if (calld->have_initial_md_string) { grpc_slice_unref_internal(exec_ctx, calld->initial_md_string); } + if (calld->have_trailing_md_string) { + grpc_slice_unref_internal(exec_ctx, calld->trailing_md_string); + } if (calld->have_service_method) { grpc_slice_unref_internal(exec_ctx, calld->service_method); } @@ -183,6 +188,18 @@ static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, */ } +static grpc_filtered_mdelem lr_trailing_md_filter(grpc_exec_ctx *exec_ctx, + void *user_data, + grpc_mdelem md) { + grpc_call_element *elem = user_data; + call_data *calld = elem->call_data; + if (grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDSTR_LB_COST_BIN)) { + calld->trailing_md_string = GRPC_MDVALUE(md); + return GRPC_FILTERED_REMOVE(); + } + return GRPC_FILTERED_MDELEM(md); +} + static void lr_start_transport_stream_op_batch( grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_transport_stream_op_batch *op) { @@ -190,13 +207,21 @@ static void lr_start_transport_stream_op_batch( call_data *calld = elem->call_data; if (op->recv_initial_metadata) { + /* substitute our callback for the higher callback */ calld->recv_initial_metadata = op->payload->recv_initial_metadata.recv_initial_metadata; - /* substitute our callback for the higher callback */ calld->ops_recv_initial_metadata_ready = op->payload->recv_initial_metadata.recv_initial_metadata_ready; op->payload->recv_initial_metadata.recv_initial_metadata_ready = &calld->on_initial_md_ready; + } else if (op->send_trailing_metadata) { + GRPC_LOG_IF_ERROR( + "grpc_metadata_batch_filter", + grpc_metadata_batch_filter( + exec_ctx, + op->payload->send_trailing_metadata.send_trailing_metadata, + lr_trailing_md_filter, elem, + "LR trailing metadata filtering error")); } grpc_call_next_op(exec_ctx, elem, op); diff --git a/src/core/lib/channel/context.h b/src/core/lib/channel/context.h index 2c1174ce7a1..6c931ad28ae 100644 --- a/src/core/lib/channel/context.h +++ b/src/core/lib/channel/context.h @@ -50,9 +50,6 @@ typedef enum { /// Reserved for traffic_class_context. GRPC_CONTEXT_TRAFFIC, - /// Costs for Load Reporting. - GRPC_CONTEXT_LR_COST, - GRPC_CONTEXT_COUNT } grpc_context_index; diff --git a/src/core/lib/transport/static_metadata.c b/src/core/lib/transport/static_metadata.c index 599d4ad1171..862cdaa8e09 100644 --- a/src/core/lib/transport/static_metadata.c +++ b/src/core/lib/transport/static_metadata.c @@ -97,23 +97,23 @@ static uint8_t g_bytes[] = { 101, 105, 102, 45, 110, 111, 110, 101, 45, 109, 97, 116, 99, 104, 105, 102, 45, 114, 97, 110, 103, 101, 105, 102, 45, 117, 110, 109, 111, 100, 105, 102, 105, 101, 100, 45, 115, 105, 110, 99, 101, 108, 97, 115, 116, - 45, 109, 111, 100, 105, 102, 105, 101, 100, 108, 105, 110, 107, 108, 111, - 99, 97, 116, 105, 111, 110, 109, 97, 120, 45, 102, 111, 114, 119, 97, - 114, 100, 115, 112, 114, 111, 120, 121, 45, 97, 117, 116, 104, 101, 110, - 116, 105, 99, 97, 116, 101, 112, 114, 111, 120, 121, 45, 97, 117, 116, - 104, 111, 114, 105, 122, 97, 116, 105, 111, 110, 114, 97, 110, 103, 101, - 114, 101, 102, 101, 114, 101, 114, 114, 101, 102, 114, 101, 115, 104, 114, - 101, 116, 114, 121, 45, 97, 102, 116, 101, 114, 115, 101, 114, 118, 101, - 114, 115, 101, 116, 45, 99, 111, 111, 107, 105, 101, 115, 116, 114, 105, - 99, 116, 45, 116, 114, 97, 110, 115, 112, 111, 114, 116, 45, 115, 101, - 99, 117, 114, 105, 116, 121, 116, 114, 97, 110, 115, 102, 101, 114, 45, - 101, 110, 99, 111, 100, 105, 110, 103, 118, 97, 114, 121, 118, 105, 97, - 119, 119, 119, 45, 97, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, - 101, 105, 100, 101, 110, 116, 105, 116, 121, 44, 100, 101, 102, 108, 97, - 116, 101, 105, 100, 101, 110, 116, 105, 116, 121, 44, 103, 122, 105, 112, - 100, 101, 102, 108, 97, 116, 101, 44, 103, 122, 105, 112, 105, 100, 101, - 110, 116, 105, 116, 121, 44, 100, 101, 102, 108, 97, 116, 101, 44, 103, - 122, 105, 112}; + 45, 109, 111, 100, 105, 102, 105, 101, 100, 108, 98, 45, 99, 111, 115, + 116, 45, 98, 105, 110, 108, 105, 110, 107, 108, 111, 99, 97, 116, 105, + 111, 110, 109, 97, 120, 45, 102, 111, 114, 119, 97, 114, 100, 115, 112, + 114, 111, 120, 121, 45, 97, 117, 116, 104, 101, 110, 116, 105, 99, 97, + 116, 101, 112, 114, 111, 120, 121, 45, 97, 117, 116, 104, 111, 114, 105, + 122, 97, 116, 105, 111, 110, 114, 97, 110, 103, 101, 114, 101, 102, 101, + 114, 101, 114, 114, 101, 102, 114, 101, 115, 104, 114, 101, 116, 114, 121, + 45, 97, 102, 116, 101, 114, 115, 101, 114, 118, 101, 114, 115, 101, 116, + 45, 99, 111, 111, 107, 105, 101, 115, 116, 114, 105, 99, 116, 45, 116, + 114, 97, 110, 115, 112, 111, 114, 116, 45, 115, 101, 99, 117, 114, 105, + 116, 121, 116, 114, 97, 110, 115, 102, 101, 114, 45, 101, 110, 99, 111, + 100, 105, 110, 103, 118, 97, 114, 121, 118, 105, 97, 119, 119, 119, 45, + 97, 117, 116, 104, 101, 110, 116, 105, 99, 97, 116, 101, 105, 100, 101, + 110, 116, 105, 116, 121, 44, 100, 101, 102, 108, 97, 116, 101, 105, 100, + 101, 110, 116, 105, 116, 121, 44, 103, 122, 105, 112, 100, 101, 102, 108, + 97, 116, 101, 44, 103, 122, 105, 112, 105, 100, 101, 110, 116, 105, 116, + 121, 44, 100, 101, 102, 108, 97, 116, 101, 44, 103, 122, 105, 112}; static void static_ref(void *unused) {} static void static_unref(grpc_exec_ctx *exec_ctx, void *unused) {} @@ -223,6 +223,7 @@ grpc_slice_refcount grpc_static_metadata_refcounts[GRPC_STATIC_MDSTR_COUNT] = { {&grpc_static_metadata_vtable, &static_sub_refcnt}, {&grpc_static_metadata_vtable, &static_sub_refcnt}, {&grpc_static_metadata_vtable, &static_sub_refcnt}, + {&grpc_static_metadata_vtable, &static_sub_refcnt}, }; const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT] = { @@ -383,64 +384,67 @@ const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT] = { {.refcount = &grpc_static_metadata_refcounts[77], .data.refcounted = {g_bytes + 791, 13}}, {.refcount = &grpc_static_metadata_refcounts[78], - .data.refcounted = {g_bytes + 804, 4}}, + .data.refcounted = {g_bytes + 804, 11}}, {.refcount = &grpc_static_metadata_refcounts[79], - .data.refcounted = {g_bytes + 808, 8}}, + .data.refcounted = {g_bytes + 815, 4}}, {.refcount = &grpc_static_metadata_refcounts[80], - .data.refcounted = {g_bytes + 816, 12}}, + .data.refcounted = {g_bytes + 819, 8}}, {.refcount = &grpc_static_metadata_refcounts[81], - .data.refcounted = {g_bytes + 828, 18}}, + .data.refcounted = {g_bytes + 827, 12}}, {.refcount = &grpc_static_metadata_refcounts[82], - .data.refcounted = {g_bytes + 846, 19}}, + .data.refcounted = {g_bytes + 839, 18}}, {.refcount = &grpc_static_metadata_refcounts[83], - .data.refcounted = {g_bytes + 865, 5}}, + .data.refcounted = {g_bytes + 857, 19}}, {.refcount = &grpc_static_metadata_refcounts[84], - .data.refcounted = {g_bytes + 870, 7}}, + .data.refcounted = {g_bytes + 876, 5}}, {.refcount = &grpc_static_metadata_refcounts[85], - .data.refcounted = {g_bytes + 877, 7}}, + .data.refcounted = {g_bytes + 881, 7}}, {.refcount = &grpc_static_metadata_refcounts[86], - .data.refcounted = {g_bytes + 884, 11}}, + .data.refcounted = {g_bytes + 888, 7}}, {.refcount = &grpc_static_metadata_refcounts[87], - .data.refcounted = {g_bytes + 895, 6}}, + .data.refcounted = {g_bytes + 895, 11}}, {.refcount = &grpc_static_metadata_refcounts[88], - .data.refcounted = {g_bytes + 901, 10}}, + .data.refcounted = {g_bytes + 906, 6}}, {.refcount = &grpc_static_metadata_refcounts[89], - .data.refcounted = {g_bytes + 911, 25}}, + .data.refcounted = {g_bytes + 912, 10}}, {.refcount = &grpc_static_metadata_refcounts[90], - .data.refcounted = {g_bytes + 936, 17}}, + .data.refcounted = {g_bytes + 922, 25}}, {.refcount = &grpc_static_metadata_refcounts[91], - .data.refcounted = {g_bytes + 953, 4}}, + .data.refcounted = {g_bytes + 947, 17}}, {.refcount = &grpc_static_metadata_refcounts[92], - .data.refcounted = {g_bytes + 957, 3}}, + .data.refcounted = {g_bytes + 964, 4}}, {.refcount = &grpc_static_metadata_refcounts[93], - .data.refcounted = {g_bytes + 960, 16}}, + .data.refcounted = {g_bytes + 968, 3}}, {.refcount = &grpc_static_metadata_refcounts[94], - .data.refcounted = {g_bytes + 976, 16}}, + .data.refcounted = {g_bytes + 971, 16}}, {.refcount = &grpc_static_metadata_refcounts[95], - .data.refcounted = {g_bytes + 992, 13}}, + .data.refcounted = {g_bytes + 987, 16}}, {.refcount = &grpc_static_metadata_refcounts[96], - .data.refcounted = {g_bytes + 1005, 12}}, + .data.refcounted = {g_bytes + 1003, 13}}, {.refcount = &grpc_static_metadata_refcounts[97], - .data.refcounted = {g_bytes + 1017, 21}}, + .data.refcounted = {g_bytes + 1016, 12}}, + {.refcount = &grpc_static_metadata_refcounts[98], + .data.refcounted = {g_bytes + 1028, 21}}, }; uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 4, 6, 6, 8, 8}; + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4, 4, 6, 6, 8, 8}; static const int8_t elems_r[] = { - 10, 8, -3, 0, 9, 21, -76, 22, 0, 10, -7, 0, 0, 0, 14, 0, - 13, 12, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, -50, -51, 15, -53, -54, -55, -56, -56, -57, -58, 0, 37, 36, 35, 34, - 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, - 17, 16, 15, 14, 13, 12, 11, 10, 13, 12, 11, 10, 9, 8, 7, 0}; + 10, 8, -3, 0, 9, 21, -77, 22, 0, 10, -7, 0, 0, 0, + 14, 0, 13, 12, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, -50, -51, 16, -53, -54, -55, -56, + -56, -57, -58, -59, 0, 37, 36, 35, 34, 33, 32, 31, 30, 29, + 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, + 14, 13, 12, 11, 10, 13, 12, 11, 10, 9, 8, 7, 0}; static uint32_t elems_phash(uint32_t i) { i -= 42; - uint32_t x = i % 96; - uint32_t y = i / 96; + uint32_t x = i % 97; + uint32_t y = i / 97; uint32_t h = x; if (y < GPR_ARRAY_SIZE(elems_r)) { uint32_t delta = (uint32_t)elems_r[y]; @@ -450,29 +454,30 @@ static uint32_t elems_phash(uint32_t i) { } static const uint16_t elem_keys[] = { - 1009, 1010, 1011, 240, 241, 242, 243, 244, 138, 139, 42, 43, - 429, 430, 431, 911, 912, 913, 712, 713, 1392, 522, 714, 1588, - 1686, 1784, 4822, 4920, 4951, 5116, 5214, 5312, 5410, 1405, 5508, 5606, - 5704, 5802, 5900, 5998, 6096, 6194, 6292, 6390, 6488, 6586, 6684, 6782, - 6880, 6978, 7076, 7174, 7272, 7370, 7468, 7566, 7664, 7762, 7860, 7958, - 8056, 8154, 8252, 8350, 8448, 1074, 1075, 1076, 1077, 8546, 8644, 8742, - 8840, 8938, 9036, 9134, 0, 314, 0, 0, 0, 0, 0, 0, + 1019, 1020, 1021, 242, 243, 244, 245, 246, 139, 140, 42, 43, + 433, 434, 435, 920, 921, 922, 719, 720, 1406, 527, 721, 1604, + 1703, 1802, 4871, 4970, 5001, 5168, 5267, 5366, 5465, 1419, 5564, 5663, + 5762, 5861, 5960, 6059, 6158, 6257, 6356, 6455, 6554, 6653, 6752, 6851, + 6950, 7049, 7148, 7247, 7346, 7445, 7544, 7643, 7742, 7841, 7940, 8039, + 8138, 8237, 8336, 8435, 8534, 8633, 1085, 1086, 1087, 1088, 8732, 8831, + 8930, 9029, 9128, 9227, 9326, 0, 317, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 133, 233, 234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 132, 231, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + 0}; static const uint8_t elem_idxs[] = { - 73, 76, 74, 19, 20, 21, 22, 23, 15, 16, 17, 18, 11, 12, 13, - 3, 4, 5, 0, 1, 41, 6, 2, 69, 48, 55, 24, 25, 26, 27, + 74, 77, 75, 19, 20, 21, 22, 23, 15, 16, 17, 18, 11, 12, 13, + 3, 4, 5, 0, 1, 41, 6, 2, 70, 48, 55, 24, 25, 26, 27, 28, 29, 30, 7, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 42, 43, 44, 45, 46, 47, 49, 50, 51, 52, 53, 54, 56, 57, 58, 59, - 60, 61, 62, 63, 64, 75, 77, 78, 79, 65, 66, 67, 68, 70, 71, - 72, 255, 14, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 8, 9, 10}; + 60, 61, 62, 63, 64, 65, 76, 78, 79, 80, 66, 67, 68, 69, 71, + 72, 73, 255, 14, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 8, 9, 10}; grpc_mdelem grpc_static_mdelem_for_static_strings(int a, int b) { if (a == -1 || b == -1) return GRPC_MDNULL; - uint32_t k = (uint32_t)(a * 98 + b); + uint32_t k = (uint32_t)(a * 99 + b); uint32_t h = elems_phash(k); return h < GPR_ARRAY_SIZE(elem_keys) && elem_keys[h] == k ? GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[elem_idxs[h]], @@ -706,71 +711,75 @@ grpc_mdelem_data grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT] = { {.refcount = &grpc_static_metadata_refcounts[20], .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[78], - .data.refcounted = {g_bytes + 804, 4}}, + .data.refcounted = {g_bytes + 804, 11}}, {.refcount = &grpc_static_metadata_refcounts[20], .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[79], - .data.refcounted = {g_bytes + 808, 8}}, + .data.refcounted = {g_bytes + 815, 4}}, {.refcount = &grpc_static_metadata_refcounts[20], .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[80], - .data.refcounted = {g_bytes + 816, 12}}, + .data.refcounted = {g_bytes + 819, 8}}, {.refcount = &grpc_static_metadata_refcounts[20], .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[81], - .data.refcounted = {g_bytes + 828, 18}}, + .data.refcounted = {g_bytes + 827, 12}}, {.refcount = &grpc_static_metadata_refcounts[20], .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[82], - .data.refcounted = {g_bytes + 846, 19}}, + .data.refcounted = {g_bytes + 839, 18}}, {.refcount = &grpc_static_metadata_refcounts[20], .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[83], - .data.refcounted = {g_bytes + 865, 5}}, + .data.refcounted = {g_bytes + 857, 19}}, {.refcount = &grpc_static_metadata_refcounts[20], .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[84], - .data.refcounted = {g_bytes + 870, 7}}, + .data.refcounted = {g_bytes + 876, 5}}, {.refcount = &grpc_static_metadata_refcounts[20], .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[85], - .data.refcounted = {g_bytes + 877, 7}}, + .data.refcounted = {g_bytes + 881, 7}}, {.refcount = &grpc_static_metadata_refcounts[20], .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[86], - .data.refcounted = {g_bytes + 884, 11}}, + .data.refcounted = {g_bytes + 888, 7}}, {.refcount = &grpc_static_metadata_refcounts[20], .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[87], - .data.refcounted = {g_bytes + 895, 6}}, + .data.refcounted = {g_bytes + 895, 11}}, {.refcount = &grpc_static_metadata_refcounts[20], .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[88], - .data.refcounted = {g_bytes + 901, 10}}, + .data.refcounted = {g_bytes + 906, 6}}, {.refcount = &grpc_static_metadata_refcounts[20], .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[89], - .data.refcounted = {g_bytes + 911, 25}}, + .data.refcounted = {g_bytes + 912, 10}}, {.refcount = &grpc_static_metadata_refcounts[20], .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[90], - .data.refcounted = {g_bytes + 936, 17}}, + .data.refcounted = {g_bytes + 922, 25}}, {.refcount = &grpc_static_metadata_refcounts[20], .data.refcounted = {g_bytes + 234, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[16], - .data.refcounted = {g_bytes + 200, 10}}, + {{.refcount = &grpc_static_metadata_refcounts[91], + .data.refcounted = {g_bytes + 947, 17}}, {.refcount = &grpc_static_metadata_refcounts[20], .data.refcounted = {g_bytes + 234, 0}}}, - {{.refcount = &grpc_static_metadata_refcounts[91], - .data.refcounted = {g_bytes + 953, 4}}, + {{.refcount = &grpc_static_metadata_refcounts[16], + .data.refcounted = {g_bytes + 200, 10}}, {.refcount = &grpc_static_metadata_refcounts[20], .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[92], - .data.refcounted = {g_bytes + 957, 3}}, + .data.refcounted = {g_bytes + 964, 4}}, {.refcount = &grpc_static_metadata_refcounts[20], .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[93], - .data.refcounted = {g_bytes + 960, 16}}, + .data.refcounted = {g_bytes + 968, 3}}, + {.refcount = &grpc_static_metadata_refcounts[20], + .data.refcounted = {g_bytes + 234, 0}}}, + {{.refcount = &grpc_static_metadata_refcounts[94], + .data.refcounted = {g_bytes + 971, 16}}, {.refcount = &grpc_static_metadata_refcounts[20], .data.refcounted = {g_bytes + 234, 0}}}, {{.refcount = &grpc_static_metadata_refcounts[10], @@ -783,24 +792,24 @@ grpc_mdelem_data grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT] = { .data.refcounted = {g_bytes + 377, 7}}}, {{.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[94], - .data.refcounted = {g_bytes + 976, 16}}}, + {.refcount = &grpc_static_metadata_refcounts[95], + .data.refcounted = {g_bytes + 987, 16}}}, {{.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, {.refcount = &grpc_static_metadata_refcounts[30], .data.refcounted = {g_bytes + 373, 4}}}, - {{.refcount = &grpc_static_metadata_refcounts[10], - .data.refcounted = {g_bytes + 90, 20}}, - {.refcount = &grpc_static_metadata_refcounts[95], - .data.refcounted = {g_bytes + 992, 13}}}, {{.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, {.refcount = &grpc_static_metadata_refcounts[96], - .data.refcounted = {g_bytes + 1005, 12}}}, + .data.refcounted = {g_bytes + 1003, 13}}}, {{.refcount = &grpc_static_metadata_refcounts[10], .data.refcounted = {g_bytes + 90, 20}}, {.refcount = &grpc_static_metadata_refcounts[97], - .data.refcounted = {g_bytes + 1017, 21}}}, + .data.refcounted = {g_bytes + 1016, 12}}}, + {{.refcount = &grpc_static_metadata_refcounts[10], + .data.refcounted = {g_bytes + 90, 20}}, + {.refcount = &grpc_static_metadata_refcounts[98], + .data.refcounted = {g_bytes + 1028, 21}}}, }; -const uint8_t grpc_static_accept_encoding_metadata[8] = {0, 73, 74, 75, - 76, 77, 78, 79}; +const uint8_t grpc_static_accept_encoding_metadata[8] = {0, 74, 75, 76, + 77, 78, 79, 80}; diff --git a/src/core/lib/transport/static_metadata.h b/src/core/lib/transport/static_metadata.h index dcbe418edc5..84fb316fd66 100644 --- a/src/core/lib/transport/static_metadata.h +++ b/src/core/lib/transport/static_metadata.h @@ -44,7 +44,7 @@ #include "src/core/lib/transport/metadata.h" -#define GRPC_STATIC_MDSTR_COUNT 98 +#define GRPC_STATIC_MDSTR_COUNT 99 extern const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT]; /* ":path" */ #define GRPC_MDSTR_PATH (grpc_static_slice_table[0]) @@ -205,47 +205,49 @@ extern const grpc_slice grpc_static_slice_table[GRPC_STATIC_MDSTR_COUNT]; #define GRPC_MDSTR_IF_UNMODIFIED_SINCE (grpc_static_slice_table[76]) /* "last-modified" */ #define GRPC_MDSTR_LAST_MODIFIED (grpc_static_slice_table[77]) +/* "lb-cost-bin" */ +#define GRPC_MDSTR_LB_COST_BIN (grpc_static_slice_table[78]) /* "link" */ -#define GRPC_MDSTR_LINK (grpc_static_slice_table[78]) +#define GRPC_MDSTR_LINK (grpc_static_slice_table[79]) /* "location" */ -#define GRPC_MDSTR_LOCATION (grpc_static_slice_table[79]) +#define GRPC_MDSTR_LOCATION (grpc_static_slice_table[80]) /* "max-forwards" */ -#define GRPC_MDSTR_MAX_FORWARDS (grpc_static_slice_table[80]) +#define GRPC_MDSTR_MAX_FORWARDS (grpc_static_slice_table[81]) /* "proxy-authenticate" */ -#define GRPC_MDSTR_PROXY_AUTHENTICATE (grpc_static_slice_table[81]) +#define GRPC_MDSTR_PROXY_AUTHENTICATE (grpc_static_slice_table[82]) /* "proxy-authorization" */ -#define GRPC_MDSTR_PROXY_AUTHORIZATION (grpc_static_slice_table[82]) +#define GRPC_MDSTR_PROXY_AUTHORIZATION (grpc_static_slice_table[83]) /* "range" */ -#define GRPC_MDSTR_RANGE (grpc_static_slice_table[83]) +#define GRPC_MDSTR_RANGE (grpc_static_slice_table[84]) /* "referer" */ -#define GRPC_MDSTR_REFERER (grpc_static_slice_table[84]) +#define GRPC_MDSTR_REFERER (grpc_static_slice_table[85]) /* "refresh" */ -#define GRPC_MDSTR_REFRESH (grpc_static_slice_table[85]) +#define GRPC_MDSTR_REFRESH (grpc_static_slice_table[86]) /* "retry-after" */ -#define GRPC_MDSTR_RETRY_AFTER (grpc_static_slice_table[86]) +#define GRPC_MDSTR_RETRY_AFTER (grpc_static_slice_table[87]) /* "server" */ -#define GRPC_MDSTR_SERVER (grpc_static_slice_table[87]) +#define GRPC_MDSTR_SERVER (grpc_static_slice_table[88]) /* "set-cookie" */ -#define GRPC_MDSTR_SET_COOKIE (grpc_static_slice_table[88]) +#define GRPC_MDSTR_SET_COOKIE (grpc_static_slice_table[89]) /* "strict-transport-security" */ -#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (grpc_static_slice_table[89]) +#define GRPC_MDSTR_STRICT_TRANSPORT_SECURITY (grpc_static_slice_table[90]) /* "transfer-encoding" */ -#define GRPC_MDSTR_TRANSFER_ENCODING (grpc_static_slice_table[90]) +#define GRPC_MDSTR_TRANSFER_ENCODING (grpc_static_slice_table[91]) /* "vary" */ -#define GRPC_MDSTR_VARY (grpc_static_slice_table[91]) +#define GRPC_MDSTR_VARY (grpc_static_slice_table[92]) /* "via" */ -#define GRPC_MDSTR_VIA (grpc_static_slice_table[92]) +#define GRPC_MDSTR_VIA (grpc_static_slice_table[93]) /* "www-authenticate" */ -#define GRPC_MDSTR_WWW_AUTHENTICATE (grpc_static_slice_table[93]) +#define GRPC_MDSTR_WWW_AUTHENTICATE (grpc_static_slice_table[94]) /* "identity,deflate" */ -#define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE (grpc_static_slice_table[94]) +#define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE (grpc_static_slice_table[95]) /* "identity,gzip" */ -#define GRPC_MDSTR_IDENTITY_COMMA_GZIP (grpc_static_slice_table[95]) +#define GRPC_MDSTR_IDENTITY_COMMA_GZIP (grpc_static_slice_table[96]) /* "deflate,gzip" */ -#define GRPC_MDSTR_DEFLATE_COMMA_GZIP (grpc_static_slice_table[96]) +#define GRPC_MDSTR_DEFLATE_COMMA_GZIP (grpc_static_slice_table[97]) /* "identity,deflate,gzip" */ #define GRPC_MDSTR_IDENTITY_COMMA_DEFLATE_COMMA_GZIP \ - (grpc_static_slice_table[97]) + (grpc_static_slice_table[98]) extern const grpc_slice_refcount_vtable grpc_static_metadata_vtable; extern grpc_slice_refcount @@ -257,7 +259,7 @@ extern grpc_slice_refcount #define GRPC_STATIC_METADATA_INDEX(static_slice) \ ((int)((static_slice).refcount - grpc_static_metadata_refcounts)) -#define GRPC_STATIC_MDELEM_COUNT 80 +#define GRPC_STATIC_MDELEM_COUNT 81 extern grpc_mdelem_data grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT]; extern uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT]; /* "grpc-status": "0" */ @@ -428,78 +430,81 @@ extern uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT]; /* "lb-token": "" */ #define GRPC_MDELEM_LB_TOKEN_EMPTY \ (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[55], GRPC_MDELEM_STORAGE_STATIC)) +/* "lb-cost-bin": "" */ +#define GRPC_MDELEM_LB_COST_BIN_EMPTY \ + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[56], GRPC_MDELEM_STORAGE_STATIC)) /* "link": "" */ #define GRPC_MDELEM_LINK_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[56], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[57], GRPC_MDELEM_STORAGE_STATIC)) /* "location": "" */ #define GRPC_MDELEM_LOCATION_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[57], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[58], GRPC_MDELEM_STORAGE_STATIC)) /* "max-forwards": "" */ #define GRPC_MDELEM_MAX_FORWARDS_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[58], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[59], GRPC_MDELEM_STORAGE_STATIC)) /* "proxy-authenticate": "" */ #define GRPC_MDELEM_PROXY_AUTHENTICATE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[59], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[60], GRPC_MDELEM_STORAGE_STATIC)) /* "proxy-authorization": "" */ #define GRPC_MDELEM_PROXY_AUTHORIZATION_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[60], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[61], GRPC_MDELEM_STORAGE_STATIC)) /* "range": "" */ #define GRPC_MDELEM_RANGE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[61], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[62], GRPC_MDELEM_STORAGE_STATIC)) /* "referer": "" */ #define GRPC_MDELEM_REFERER_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[62], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[63], GRPC_MDELEM_STORAGE_STATIC)) /* "refresh": "" */ #define GRPC_MDELEM_REFRESH_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[63], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[64], GRPC_MDELEM_STORAGE_STATIC)) /* "retry-after": "" */ #define GRPC_MDELEM_RETRY_AFTER_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[64], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[65], GRPC_MDELEM_STORAGE_STATIC)) /* "server": "" */ #define GRPC_MDELEM_SERVER_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[65], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[66], GRPC_MDELEM_STORAGE_STATIC)) /* "set-cookie": "" */ #define GRPC_MDELEM_SET_COOKIE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[66], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[67], GRPC_MDELEM_STORAGE_STATIC)) /* "strict-transport-security": "" */ #define GRPC_MDELEM_STRICT_TRANSPORT_SECURITY_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[67], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[68], GRPC_MDELEM_STORAGE_STATIC)) /* "transfer-encoding": "" */ #define GRPC_MDELEM_TRANSFER_ENCODING_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[68], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[69], GRPC_MDELEM_STORAGE_STATIC)) /* "user-agent": "" */ #define GRPC_MDELEM_USER_AGENT_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[69], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[70], GRPC_MDELEM_STORAGE_STATIC)) /* "vary": "" */ #define GRPC_MDELEM_VARY_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[70], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[71], GRPC_MDELEM_STORAGE_STATIC)) /* "via": "" */ #define GRPC_MDELEM_VIA_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[71], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[72], GRPC_MDELEM_STORAGE_STATIC)) /* "www-authenticate": "" */ #define GRPC_MDELEM_WWW_AUTHENTICATE_EMPTY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[72], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[73], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "identity" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[73], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[74], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "deflate" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_DEFLATE \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[74], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[75], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "identity,deflate" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY_COMMA_DEFLATE \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[75], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[76], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "gzip" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_GZIP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[76], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[77], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "identity,gzip" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY_COMMA_GZIP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[77], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[78], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "deflate,gzip" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_DEFLATE_COMMA_GZIP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[78], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[79], GRPC_MDELEM_STORAGE_STATIC)) /* "grpc-accept-encoding": "identity,deflate,gzip" */ #define GRPC_MDELEM_GRPC_ACCEPT_ENCODING_IDENTITY_COMMA_DEFLATE_COMMA_GZIP \ - (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[79], GRPC_MDELEM_STORAGE_STATIC)) + (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[80], GRPC_MDELEM_STORAGE_STATIC)) grpc_mdelem grpc_static_mdelem_for_static_strings(int a, int b); typedef enum { diff --git a/src/cpp/server/server_context.cc b/src/cpp/server/server_context.cc index afc9873c148..33ad0ae1423 100644 --- a/src/cpp/server/server_context.cc +++ b/src/cpp/server/server_context.cc @@ -229,17 +229,9 @@ const struct census_context* ServerContext::census_context() const { void ServerContext::SetLoadReportingCosts( const std::vector& cost_data) { if (call_ == nullptr) return; - grpc_load_reporting_cost_context* cost_ctx = - static_cast( - gpr_malloc(sizeof(*cost_ctx))); - cost_ctx->values_count = cost_data.size(); - cost_ctx->values = static_cast( - gpr_malloc(sizeof(*cost_ctx->values) * cost_ctx->values_count)); - for (size_t i = 0; i < cost_ctx->values_count; ++i) { - cost_ctx->values[i] = - grpc_slice_from_copied_buffer(cost_data[i].data(), cost_data[i].size()); + for (const auto& cost_datum : cost_data) { + AddTrailingMetadata(GRPC_LB_COST_MD_KEY, cost_datum); } - grpc_call_set_load_reporting_cost_context(call_, cost_ctx); } } // namespace grpc diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index 063f92114c0..f189e88198e 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -110,7 +110,6 @@ grpc_channel_register_call_type grpc_channel_register_call_import; grpc_channel_create_registered_call_type grpc_channel_create_registered_call_import; grpc_call_start_batch_type grpc_call_start_batch_import; grpc_call_get_peer_type grpc_call_get_peer_import; -grpc_call_set_load_reporting_cost_context_type grpc_call_set_load_reporting_cost_context_import; grpc_census_call_set_context_type grpc_census_call_set_context_import; grpc_census_call_get_context_type grpc_census_call_get_context_import; grpc_channel_get_target_type grpc_channel_get_target_import; @@ -407,7 +406,6 @@ void grpc_rb_load_imports(HMODULE library) { grpc_channel_create_registered_call_import = (grpc_channel_create_registered_call_type) GetProcAddress(library, "grpc_channel_create_registered_call"); grpc_call_start_batch_import = (grpc_call_start_batch_type) GetProcAddress(library, "grpc_call_start_batch"); grpc_call_get_peer_import = (grpc_call_get_peer_type) GetProcAddress(library, "grpc_call_get_peer"); - grpc_call_set_load_reporting_cost_context_import = (grpc_call_set_load_reporting_cost_context_type) GetProcAddress(library, "grpc_call_set_load_reporting_cost_context"); grpc_census_call_set_context_import = (grpc_census_call_set_context_type) GetProcAddress(library, "grpc_census_call_set_context"); grpc_census_call_get_context_import = (grpc_census_call_get_context_type) GetProcAddress(library, "grpc_census_call_get_context"); grpc_channel_get_target_import = (grpc_channel_get_target_type) GetProcAddress(library, "grpc_channel_get_target"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index e036ff7bd63..dcdc8063a22 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -281,9 +281,6 @@ extern grpc_call_start_batch_type grpc_call_start_batch_import; typedef char *(*grpc_call_get_peer_type)(grpc_call *call); extern grpc_call_get_peer_type grpc_call_get_peer_import; #define grpc_call_get_peer grpc_call_get_peer_import -typedef void(*grpc_call_set_load_reporting_cost_context_type)(grpc_call *call, struct grpc_load_reporting_cost_context *context); -extern grpc_call_set_load_reporting_cost_context_type grpc_call_set_load_reporting_cost_context_import; -#define grpc_call_set_load_reporting_cost_context grpc_call_set_load_reporting_cost_context_import typedef void(*grpc_census_call_set_context_type)(grpc_call *call, struct census_context *context); extern grpc_census_call_set_context_type grpc_census_call_set_context_import; #define grpc_census_call_set_context grpc_census_call_set_context_import diff --git a/test/core/end2end/fuzzers/hpack.dictionary b/test/core/end2end/fuzzers/hpack.dictionary index 4282c612146..2bb9de34c56 100644 --- a/test/core/end2end/fuzzers/hpack.dictionary +++ b/test/core/end2end/fuzzers/hpack.dictionary @@ -77,6 +77,7 @@ "\x08if-range" "\x13if-unmodified-since" "\x0Dlast-modified" +"\x0Blb-cost-bin" "\x04link" "\x08location" "\x0Cmax-forwards" @@ -153,6 +154,7 @@ "\x00\x13if-unmodified-since\x00" "\x00\x0Dlast-modified\x00" "\x00\x08lb-token\x00" +"\x00\x0Blb-cost-bin\x00" "\x00\x04link\x00" "\x00\x08location\x00" "\x00\x0Cmax-forwards\x00" diff --git a/test/core/end2end/tests/load_reporting_hook.c b/test/core/end2end/tests/load_reporting_hook.c index 72e9db6e214..c021bc07ff0 100644 --- a/test/core/end2end/tests/load_reporting_hook.c +++ b/test/core/end2end/tests/load_reporting_hook.c @@ -129,8 +129,7 @@ static void end_test(grpc_end2end_test_fixture *f) { static void request_response_with_payload( grpc_end2end_test_config config, grpc_end2end_test_fixture f, const char *method_name, const char *request_msg, const char *response_msg, - grpc_metadata *initial_lr_metadata, - grpc_load_reporting_cost_context *cost_ctx) { + grpc_metadata *initial_lr_metadata, grpc_metadata *trailing_lr_metadata) { grpc_slice request_payload_slice = grpc_slice_from_static_string(request_msg); grpc_slice response_payload_slice = grpc_slice_from_static_string(response_msg); @@ -243,8 +242,9 @@ static void request_response_with_payload( op->reserved = NULL; op++; op->op = GRPC_OP_SEND_STATUS_FROM_SERVER; - GPR_ASSERT(cost_ctx != NULL); - grpc_call_set_load_reporting_cost_context(s, cost_ctx); + GPR_ASSERT(trailing_lr_metadata != NULL); + op->data.send_status_from_server.trailing_metadata_count = 1; + op->data.send_status_from_server.trailing_metadata = trailing_lr_metadata; op->data.send_status_from_server.status = GRPC_STATUS_OK; grpc_slice status_details = grpc_slice_from_static_string("xyz"); op->data.send_status_from_server.status_details = &status_details; @@ -298,21 +298,21 @@ static void test_load_reporting_hook(grpc_end2end_test_config config) { const char *response_msg = "... and the response from the server"; grpc_metadata initial_lr_metadata; + grpc_metadata trailing_lr_metadata; initial_lr_metadata.key = GRPC_MDSTR_LB_TOKEN; initial_lr_metadata.value = grpc_slice_from_static_string("client-token"); memset(&initial_lr_metadata.internal_data, 0, sizeof(initial_lr_metadata.internal_data)); - grpc_load_reporting_cost_context *cost_ctx = gpr_malloc(sizeof(*cost_ctx)); - memset(cost_ctx, 0, sizeof(*cost_ctx)); - cost_ctx->values_count = 1; - cost_ctx->values = - gpr_malloc(sizeof(*cost_ctx->values) * cost_ctx->values_count); - cost_ctx->values[0] = grpc_slice_from_static_string("cost-token"); + trailing_lr_metadata.key = GRPC_MDSTR_LB_COST_BIN; + trailing_lr_metadata.value = grpc_slice_from_static_string("server-token"); + memset(&trailing_lr_metadata.internal_data, 0, + sizeof(trailing_lr_metadata.internal_data)); request_response_with_payload(config, f, method_name, request_msg, - response_msg, &initial_lr_metadata, cost_ctx); + response_msg, &initial_lr_metadata, + &trailing_lr_metadata); end_test(&f); { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; diff --git a/tools/codegen/core/gen_static_metadata.py b/tools/codegen/core/gen_static_metadata.py index ead57b2bf6d..360b0f56e92 100755 --- a/tools/codegen/core/gen_static_metadata.py +++ b/tools/codegen/core/gen_static_metadata.py @@ -124,6 +124,7 @@ CONFIG = [ ('if-unmodified-since', ''), ('last-modified', ''), ('lb-token', ''), + ('lb-cost-bin', ''), ('link', ''), ('location', ''), ('max-forwards', ''), From a1f76425596c05a12513ef9656983cdf0613aeb3 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 17 Apr 2017 12:04:46 -0700 Subject: [PATCH 56/60] Public functions must be prefixed with grpc_ --- .../ext/transport/chttp2/transport/chttp2_transport.c | 4 ++-- src/core/ext/transport/chttp2/transport/frame_data.c | 10 ++++------ src/core/ext/transport/chttp2/transport/frame_data.h | 10 ++++------ 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index 9269611dd9d..5c5175f5668 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1679,7 +1679,7 @@ void grpc_chttp2_maybe_complete_recv_message(grpc_exec_ctx *exec_ctx, grpc_slice_buffer_swap(&s->unprocessed_incoming_frames_buffer, &s->frame_storage); } - error = deframe_unprocessed_incoming_frames( + error = grpc_deframe_unprocessed_incoming_frames( exec_ctx, &s->data_parser, s, &s->unprocessed_incoming_frames_buffer, NULL, s->recv_message); if (error != GRPC_ERROR_NONE) { @@ -2627,7 +2627,7 @@ static grpc_error *incoming_byte_stream_pull(grpc_exec_ctx *exec_ctx, grpc_chttp2_stream *s = bs->stream; if (s->unprocessed_incoming_frames_buffer.length > 0) { - grpc_error *error = deframe_unprocessed_incoming_frames( + grpc_error *error = grpc_deframe_unprocessed_incoming_frames( exec_ctx, &s->data_parser, s, &s->unprocessed_incoming_frames_buffer, slice, NULL); if (error != GRPC_ERROR_NONE) { diff --git a/src/core/ext/transport/chttp2/transport/frame_data.c b/src/core/ext/transport/chttp2/transport/frame_data.c index 5d382d80a8b..9aba3ea445d 100644 --- a/src/core/ext/transport/chttp2/transport/frame_data.c +++ b/src/core/ext/transport/chttp2/transport/frame_data.c @@ -112,12 +112,10 @@ void grpc_chttp2_encode_data(uint32_t id, grpc_slice_buffer *inbuf, stats->data_bytes += write_bytes; } -grpc_error *deframe_unprocessed_incoming_frames(grpc_exec_ctx *exec_ctx, - grpc_chttp2_data_parser *p, - grpc_chttp2_stream *s, - grpc_slice_buffer *slices, - grpc_slice *slice_out, - grpc_byte_stream **stream_out) { +grpc_error *grpc_deframe_unprocessed_incoming_frames( + grpc_exec_ctx *exec_ctx, grpc_chttp2_data_parser *p, grpc_chttp2_stream *s, + grpc_slice_buffer *slices, grpc_slice *slice_out, + grpc_byte_stream **stream_out) { grpc_error *error = GRPC_ERROR_NONE; grpc_chttp2_transport *t = s->t; diff --git a/src/core/ext/transport/chttp2/transport/frame_data.h b/src/core/ext/transport/chttp2/transport/frame_data.h index 2fb8983c38e..9ed4ad0f218 100644 --- a/src/core/ext/transport/chttp2/transport/frame_data.h +++ b/src/core/ext/transport/chttp2/transport/frame_data.h @@ -90,11 +90,9 @@ void grpc_chttp2_encode_data(uint32_t id, grpc_slice_buffer *inbuf, grpc_transport_one_way_stats *stats, grpc_slice_buffer *outbuf); -grpc_error *deframe_unprocessed_incoming_frames(grpc_exec_ctx *exec_ctx, - grpc_chttp2_data_parser *p, - grpc_chttp2_stream *s, - grpc_slice_buffer *slices, - grpc_slice *slice_out, - grpc_byte_stream **stream_out); +grpc_error *grpc_deframe_unprocessed_incoming_frames( + grpc_exec_ctx *exec_ctx, grpc_chttp2_data_parser *p, grpc_chttp2_stream *s, + grpc_slice_buffer *slices, grpc_slice *slice_out, + grpc_byte_stream **stream_out); #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_FRAME_DATA_H */ From 7cc31f1d69bee3dc451d736a0a22f17b07e029e2 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 17 Apr 2017 15:11:05 -0700 Subject: [PATCH 57/60] Fix python artifact build --- PYTHON-MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PYTHON-MANIFEST.in b/PYTHON-MANIFEST.in index 846530532df..bb76b38f9fb 100644 --- a/PYTHON-MANIFEST.in +++ b/PYTHON-MANIFEST.in @@ -7,7 +7,7 @@ graft include/grpc graft third_party/boringssl graft third_party/nanopb graft third_party/zlib -graft third_party/c-ares +graft third_party/cares include src/python/grpcio/_spawn_patch.py include src/python/grpcio/commands.py include src/python/grpcio/grpc_version.py From a22a19fd64fdf4c5c2a26f8989e86907b9e219af Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 18 Apr 2017 12:31:49 +0200 Subject: [PATCH 58/60] fix markdown --- doc/server_side_auth.md | 4 ++-- examples/cpp/README.md | 2 +- examples/cpp/cpptutorial.md | 2 +- examples/csharp/route_guide/README.md | 2 +- .../dynamic_codegen/route_guide/README.md | 2 +- .../node/static_codegen/route_guide/README.md | 2 +- examples/objective-c/auth_sample/README.md | 2 +- examples/objective-c/helloworld/README.md | 2 +- examples/objective-c/route_guide/README.md | 2 +- examples/php/route_guide/README.md | 2 +- .../ruby/errors_and_cancellation/README.md | 2 +- examples/ruby/route_guide/README.md | 2 +- tools/grift/README.md | 12 ++++------ tools/run_tests/README.md | 24 +++++++++---------- vsprojects/README.md | 6 ++--- 15 files changed, 33 insertions(+), 35 deletions(-) diff --git a/doc/server_side_auth.md b/doc/server_side_auth.md index 288c6e9cb38..d260237928d 100644 --- a/doc/server_side_auth.md +++ b/doc/server_side_auth.md @@ -13,7 +13,7 @@ The contents of the *auth properties* are populated by an *auth interceptor*. Th WARNING: AuthContext is the only reliable source of truth when it comes to authenticating RPCs. Using any other call/context properties for authentication purposes is wrong and inherently unsafe. -####Example AuthContext contents +#### Example AuthContext contents For secure channel using mutual TLS authentication with both client and server certificates (test certificates from this repository are used). @@ -45,7 +45,7 @@ gRPC comes with some basic "interceptors" already built-in. WARNING: While there is a public API that allows anyone to write their own custom interceptor, please think twice before using it. There are legitimate uses for custom interceptors but you should keep in mind that as auth interceptors essentially decide which RPCs are authenticated and which are not, their code is very sensitive from the security perspective and getting things wrong might have serious consequences. If unsure, we strongly recommend to rely on official & proven interceptors that come with gRPC. -####Available auth interceptors +#### Available auth interceptors - TLS/SSL certificate authentication (built into gRPC's security layer, automatically used whenever you use a secure connection) - (coming soon) JWT auth token authentication - more will be added over time diff --git a/examples/cpp/README.md b/examples/cpp/README.md index 783935cd53d..8e2bb5d13b1 100644 --- a/examples/cpp/README.md +++ b/examples/cpp/README.md @@ -1,4 +1,4 @@ -#gRPC in 3 minutes (C++) +# gRPC in 3 minutes (C++) ## Installation diff --git a/examples/cpp/cpptutorial.md b/examples/cpp/cpptutorial.md index ae84aba9163..7d98367da43 100644 --- a/examples/cpp/cpptutorial.md +++ b/examples/cpp/cpptutorial.md @@ -1,4 +1,4 @@ -#gRPC Basics: C++ +# gRPC Basics: C++ This tutorial provides a basic C++ programmer's introduction to working with gRPC. By walking through this example you'll learn how to: diff --git a/examples/csharp/route_guide/README.md b/examples/csharp/route_guide/README.md index a9aa87a83df..98073b02963 100644 --- a/examples/csharp/route_guide/README.md +++ b/examples/csharp/route_guide/README.md @@ -1,4 +1,4 @@ -#gRPC Basics: C# sample code +# gRPC Basics: C# sample code The files in this folder are the samples used in [gRPC Basics: C#][], a detailed tutorial for using gRPC in C#. diff --git a/examples/node/dynamic_codegen/route_guide/README.md b/examples/node/dynamic_codegen/route_guide/README.md index 22bcf789863..7ec519c76bb 100644 --- a/examples/node/dynamic_codegen/route_guide/README.md +++ b/examples/node/dynamic_codegen/route_guide/README.md @@ -1,4 +1,4 @@ -#gRPC Basics: Node.js sample code +# gRPC Basics: Node.js sample code The files in this folder are the samples used in [gRPC Basics: Node.js][], a detailed tutorial for using gRPC in Node.js. diff --git a/examples/node/static_codegen/route_guide/README.md b/examples/node/static_codegen/route_guide/README.md index 22bcf789863..7ec519c76bb 100644 --- a/examples/node/static_codegen/route_guide/README.md +++ b/examples/node/static_codegen/route_guide/README.md @@ -1,4 +1,4 @@ -#gRPC Basics: Node.js sample code +# gRPC Basics: Node.js sample code The files in this folder are the samples used in [gRPC Basics: Node.js][], a detailed tutorial for using gRPC in Node.js. diff --git a/examples/objective-c/auth_sample/README.md b/examples/objective-c/auth_sample/README.md index c560b7af65b..b75dcab2de4 100644 --- a/examples/objective-c/auth_sample/README.md +++ b/examples/objective-c/auth_sample/README.md @@ -1,3 +1,3 @@ -#OAuth2 on gRPC: Objective-C +# OAuth2 on gRPC: Objective-C This is the supporting code for the tutorial "[OAuth2 on gRPC: Objective-C](http://www.grpc.io/docs/tutorials/auth/oauth2-objective-c.html)." diff --git a/examples/objective-c/helloworld/README.md b/examples/objective-c/helloworld/README.md index fdff938a978..365bea14228 100644 --- a/examples/objective-c/helloworld/README.md +++ b/examples/objective-c/helloworld/README.md @@ -1,4 +1,4 @@ -#gRPC in 3 minutes (Objective-C) +# gRPC in 3 minutes (Objective-C) ## Installation diff --git a/examples/objective-c/route_guide/README.md b/examples/objective-c/route_guide/README.md index 6a6f7c0d338..b99bf546fb7 100644 --- a/examples/objective-c/route_guide/README.md +++ b/examples/objective-c/route_guide/README.md @@ -1,4 +1,4 @@ -#gRPC Basics: Objective-C +# gRPC Basics: Objective-C This is the supporting code for the tutorial "[gRPC Basics: Objective-C](http://www.grpc.io/docs/tutorials/basic/objective-c.html)." diff --git a/examples/php/route_guide/README.md b/examples/php/route_guide/README.md index 26f1704f122..5b2cc05efc7 100644 --- a/examples/php/route_guide/README.md +++ b/examples/php/route_guide/README.md @@ -1,4 +1,4 @@ -#gRPC Basics: PHP sample code +# gRPC Basics: PHP sample code The files in this folder are the samples used in [gRPC Basics: PHP][], a detailed tutorial for using gRPC in PHP. diff --git a/examples/ruby/errors_and_cancellation/README.md b/examples/ruby/errors_and_cancellation/README.md index 126518c4aab..661bd84792f 100644 --- a/examples/ruby/errors_and_cancellation/README.md +++ b/examples/ruby/errors_and_cancellation/README.md @@ -1,4 +1,4 @@ -#Errors and Cancelletion code samples for grpc-ruby +# Errors and Cancelletion code samples for grpc-ruby The examples in this directory show use of grpc errors. diff --git a/examples/ruby/route_guide/README.md b/examples/ruby/route_guide/README.md index 3c353d1d976..b2514630a96 100644 --- a/examples/ruby/route_guide/README.md +++ b/examples/ruby/route_guide/README.md @@ -1,4 +1,4 @@ -#gRPC Basics: Ruby sample code +# gRPC Basics: Ruby sample code The files in this folder are the samples used in [gRPC Basics: Ruby][], a detailed tutorial for using gRPC in Ruby. diff --git a/tools/grift/README.md b/tools/grift/README.md index 7cbbdc567bf..2b5fa5ae14b 100644 --- a/tools/grift/README.md +++ b/tools/grift/README.md @@ -1,6 +1,4 @@ -Copyright 2016 Google Inc. - -#Documentation +# Documentation grift is integration of [Apache Thrift](https://github.com/apache/thrift.git) Serializer with gRPC. @@ -8,19 +6,19 @@ This integration allows you to use grpc to send thrift messages in C++ and java. grift uses Compact Protocol to serialize thrift messages. -##generating grpc plugins for thrift services +## generating grpc plugins for thrift services -###CPP +### C++ ```sh $ thrift --gen cpp ``` -###JAVA +### Java ```sh $ thrift --gen java ``` -#Installation +# Installation Before Installing thrift make sure to apply this [patch](grpc_plugins_generator.patch) to third_party/thrift. Go to third_party/thrift and follow the [INSTALLATION](https://github.com/apache/thrift.git) instructions to install thrift with commit id bcad91771b7f0bff28a1cac1981d7ef2b9bcef3c. \ No newline at end of file diff --git a/tools/run_tests/README.md b/tools/run_tests/README.md index e709ddd2c02..05d33fd6b1c 100644 --- a/tools/run_tests/README.md +++ b/tools/run_tests/README.md @@ -1,44 +1,44 @@ -#Overview +# Overview This directory contains scripts that facilitate building and running tests. We are using python scripts as entrypoint for our tests because that gives us the opportunity to run tests using the same commandline regardless of the platform you are using. -#Unit tests (run_tests.py) +# Unit tests (run_tests.py) Builds gRPC in given language and runs unit tests. Use `tools/run_tests/run_tests.py --help` for more help. -######Example +###### Example `tools/run_tests/run_tests.py -l csharp -c dbg` -######Useful options (among many others) +###### Useful options (among many others) - `--use_docker` Builds a docker container containing all the prerequisites for given language and runs the tests under that container. - `--build_only` Only build, do not run the tests. -#Interop tests (run_interop_tests.py) +# Interop tests (run_interop_tests.py) Runs tests for cross-platform/cross-language interoperability. For more details, see [Interop tests descriptions](/doc/interop-test-descriptions.md) The script is also capable of running interop tests for grpc-java and grpc-go, using sources checked out alongside the ones of the grpc repository. -######Example +###### Example `tools/run_tests/run_interop_tests.py -l csharp -s c++ --use_docker` (run interop tests with C# client and C++ server) Note: if you see an error like `no space left on device` when running the interop tests using Docker, make sure that Docker is building the image files in a location with sufficient disk space. -#Performance benchmarks (run_performance_tests.py) +# Performance benchmarks (run_performance_tests.py) Runs predefined benchmark scenarios for given languages. Besides the simple configuration of running all the scenarios locally, the script also supports orchestrating test runs with client and server running on different machines and uploading the results to BigQuery. -######Example +###### Example `tools/run_tests/run_peformance_tests.py -l c++ node` -######Useful options +###### Useful options - `--regex` use regex to select particular scenarios to run. -#Stress tests (run_stress_tests.py) +# Stress tests (run_stress_tests.py) Runs modified interop tests clients and servers under heavy load for an extended period of time to discover potential stability issues. The tests are internally using Kubernetes to run the client and server on GKE and upload statistics to BigQuery. @@ -47,10 +47,10 @@ The tests are internally using Kubernetes to run the client and server on GKE an The directory `tools/run_tests/stress_test/configs/` contains the config files for several scenarios -#Artifacts & Packages (task_runner.py) +# Artifacts & Packages (task_runner.py) A generalized framework for running predefined tasks based on their labels. We use this to building binary artifacts & distrib packages and testing them) -######Example +###### Example `tools/run_tests/task_runner.py -f python artifact linux x64` (build tasks with labels `python`, `artifact`, `linux`, and `x64`) diff --git a/vsprojects/README.md b/vsprojects/README.md index f1663d25485..1f0cdc24ba7 100644 --- a/vsprojects/README.md +++ b/vsprojects/README.md @@ -1,10 +1,10 @@ -#Pre-generated MS Visual Studio project & solution files +# Pre-generated MS Visual Studio project & solution files Versions 2013 and 2015 are both supported. You can use [their respective community editions](https://www.visualstudio.com/en-us/downloads/download-visual-studio-vs.aspx). -#Building +# Building We are using [NuGet](http://www.nuget.org) to pull zlib and openssl dependencies. If you don't have Visual Studio NuGet plugin installed, you'll need to download nuget.exe from the web and manually restore the NuGet packages. @@ -19,7 +19,7 @@ After that, you can build the solution using one of these options: 1. open `grpc.sln` with Visual Studio and hit "Build". 2. build from commandline using `msbuild grpc.sln /p:Configuration=Debug` -#C/C++ Test Dependencies +# C/C++ Test Dependencies * gtest isn't available as a git repo like the other dependencies. download it and add it to `/third_party/gtest/` (the folder will end up with `/build-aux/`, `/cmake/`, `/codegear/`, etc. folders in it). * if using vs2013: open/import the gtest solution in `/msvc/`, and save over the first solution (you will have to change it from read-only). change all projects to use `/MDd` (Property Pages - C/C++ - Code Generation - Runtime Library) and build. This is a "multithreaded debug" setting and it needs to match grpc. * build all From ed94e39ee8f5efd65330d72eb0bc5603efe12091 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 18 Apr 2017 14:25:17 +0200 Subject: [PATCH 59/60] setting handler is a race --- .../Grpc.Core.Tests/AppDomainUnloadTest.cs | 8 ++++---- .../Grpc.Core.Tests/UserAgentStringTest.cs | 19 ++++++++++--------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/csharp/Grpc.Core.Tests/AppDomainUnloadTest.cs b/src/csharp/Grpc.Core.Tests/AppDomainUnloadTest.cs index d7ebdb4201e..7858e77b278 100644 --- a/src/csharp/Grpc.Core.Tests/AppDomainUnloadTest.cs +++ b/src/csharp/Grpc.Core.Tests/AppDomainUnloadTest.cs @@ -72,10 +72,6 @@ namespace Grpc.Core.Tests public AppDomainTestClass() { var helper = new MockServiceHelper(Host); - var server = helper.GetServer(); - server.Start(); - var channel = helper.GetChannel(); - var readyToShutdown = new TaskCompletionSource(); helper.DuplexStreamingHandler = new DuplexStreamingServerMethod(async (requestStream, responseStream, context) => { @@ -83,6 +79,10 @@ namespace Grpc.Core.Tests await requestStream.ToListAsync(); }); + var server = helper.GetServer(); + server.Start(); + var channel = helper.GetChannel(); + var call = Calls.AsyncDuplexStreamingCall(helper.CreateDuplexStreamingCall()); readyToShutdown.Task.Wait(); // make sure handler is running } diff --git a/src/csharp/Grpc.Core.Tests/UserAgentStringTest.cs b/src/csharp/Grpc.Core.Tests/UserAgentStringTest.cs index cc830086a67..74b4997f69e 100644 --- a/src/csharp/Grpc.Core.Tests/UserAgentStringTest.cs +++ b/src/csharp/Grpc.Core.Tests/UserAgentStringTest.cs @@ -63,10 +63,6 @@ namespace Grpc.Core.Tests public void DefaultUserAgentString() { helper = new MockServiceHelper(Host); - server = helper.GetServer(); - server.Start(); - channel = helper.GetChannel(); - helper.UnaryHandler = new UnaryServerMethod((request, context) => { var userAgentString = context.RequestHeaders.First(m => (m.Key == "user-agent")).Value; @@ -75,6 +71,11 @@ namespace Grpc.Core.Tests Assert.IsTrue(parts[1].StartsWith("grpc-c/")); return Task.FromResult("PASS"); }); + + server = helper.GetServer(); + server.Start(); + channel = helper.GetChannel(); + Assert.AreEqual("PASS", Calls.BlockingUnaryCall(helper.CreateUnaryCall(), "")); } @@ -83,11 +84,6 @@ namespace Grpc.Core.Tests { helper = new MockServiceHelper(Host, channelOptions: new[] { new ChannelOption(ChannelOptions.PrimaryUserAgentString, "XYZ") }); - server = helper.GetServer(); - server.Start(); - channel = helper.GetChannel(); - - channel = helper.GetChannel(); helper.UnaryHandler = new UnaryServerMethod((request, context) => { var userAgentString = context.RequestHeaders.First(m => (m.Key == "user-agent")).Value; @@ -95,6 +91,11 @@ namespace Grpc.Core.Tests Assert.AreEqual("XYZ", parts[0]); return Task.FromResult("PASS"); }); + + server = helper.GetServer(); + server.Start(); + channel = helper.GetChannel(); + Assert.AreEqual("PASS", Calls.BlockingUnaryCall(helper.CreateUnaryCall(), "")); } } From 68f6a6732771e0265bb656d189de4b76d8268033 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Wed, 12 Apr 2017 14:19:49 -0700 Subject: [PATCH 60/60] Always generate body for python code elements Since #... class Service: class NextService: #... without a body or `pass` under `Service` is invalid Python, we ensure a `pass` statement is always emitted to make the generated code valid. --- src/compiler/python_generator.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/compiler/python_generator.cc b/src/compiler/python_generator.cc index 2649c1688d4..50ee54abffe 100644 --- a/src/compiler/python_generator.cc +++ b/src/compiler/python_generator.cc @@ -101,6 +101,14 @@ PrivateGenerator::PrivateGenerator(const GeneratorConfiguration& config, void PrivateGenerator::PrintAllComments(StringVector comments, grpc_generator::Printer* out) { if (comments.empty()) { + // Python requires code structures like class and def to have + // a body, even if it is just "pass" or a docstring. We need + // to ensure not to generate empty bodies. We could do something + // smarter and more sophisticated, but at the moment, if there is + // no docstring to print, we simply emit "pass" to ensure validity + // of the generated code. + out->Print("# missing associated documentation comment in .proto file\n"); + out->Print("pass\n"); return; } out->Print("\"\"\"");