Chiebot-Mirror
/
grpc
mirror of https://github.com/grpc/grpc.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

[experiments] Allow specifying experiments defaults to be different for debug/release builds (#30895)

Browse Source 
* cpp-codegen

* bazel-gen

* Automated change: Fix sanity tests

* Update grpc_build_system.bzl

Co-authored-by: ctiller <ctiller@users.noreply.github.com>
pull/30904/head
Craig Tiller 2 years ago committed by GitHub
parent 260a02d1ad
commit bc5db5395a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 127 additions and 82 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 45
      bazel/experiments.bzl
  2. 81
      bazel/grpc_build_system.bzl
  3. 8
      src/core/lib/experiments/experiments.yaml
  4. 72
      tools/codegen/core/gen_experiments.py
  5. 3
      tools/remote_build/linux.bazelrc

45
bazel/experiments.bzl
Unescape View File

@ -17,24 +17,29 @@
"""Dictionary of tags to experiments so we know when to test different experiments."""
EXPERIMENTS = {
"endpoint_test": [
"tcp_frame_size_tuning",
"tcp_rcv_lowat",
"tcp_read_chunks",
],
"flow_control_test": [
"flow_control_fixes",
"peer_state_based_framing",
"tcp_frame_size_tuning",
"tcp_rcv_lowat",
"tcp_read_chunks",
],
"resource_quota_test": [
"memory_pressure_controller",
"periodic_resource_quota_reclamation",
"unconstrained_max_quota_buffer_size",
],
}
NEGATED_EXPERIMENTS = {
"dbg": {
},
"off": {
"endpoint_test": [
"tcp_frame_size_tuning",
"tcp_rcv_lowat",
"tcp_read_chunks",
],
"flow_control_test": [
"flow_control_fixes",
"peer_state_based_framing",
"tcp_frame_size_tuning",
"tcp_rcv_lowat",
"tcp_read_chunks",
],
"resource_quota_test": [
"memory_pressure_controller",
"periodic_resource_quota_reclamation",
"unconstrained_max_quota_buffer_size",
],
},
"on": {
},
"opt": {
},
}

81
bazel/grpc_build_system.bzl
Unescape View File

@ -29,7 +29,7 @@ Contains macros used throughout the repo.
load("//bazel:cc_grpc_library.bzl", "cc_grpc_library")
load("//bazel:copts.bzl", "GRPC_DEFAULT_COPTS")
load("//bazel:experiments.bzl", "EXPERIMENTS", "NEGATED_EXPERIMENTS")
load("//bazel:experiments.bzl", "EXPERIMENTS")
load("@upb//bazel:upb_proto_library.bzl", "upb_proto_library", "upb_proto_reflection_library")
load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test")
load("@build_bazel_rules_apple//apple/testing/default_runner:ios_test_runner.bzl", "ios_test_runner")
@ -341,20 +341,22 @@ def expand_tests(name, srcs, deps, tags, args, exclude_pollers, uses_polling, us
})
experiments = {}
for tag in tags:
if tag not in EXPERIMENTS:
continue
for experiment in EXPERIMENTS[tag]:
experiments[experiment] = 1
experiments = list(experiments.keys())
negated_experiments = {}
for tag in tags:
if tag not in NEGATED_EXPERIMENTS:
continue
for experiment in NEGATED_EXPERIMENTS[tag]:
negated_experiments[experiment] = 1
negated_experiments = list(negated_experiments.keys())
for mode, tag_to_experiments in EXPERIMENTS.items():
experiments[mode] = {}
for tag in tags:
if tag not in tag_to_experiments:
continue
for experiment in tag_to_experiments[tag]:
experiments[mode][experiment] = 1
experiments[mode] = list(experiments[mode].keys())
mode_config = {
# format: <mode>: (enabled_target_tags, disabled_target_tags)
"dbg": (["noopt"], ["nodbg"]),
"opt": (["nodbg"], ["noopt"]),
"on": ([], None),
"off": (None, []),
}
must_have_tags = [
# We don't run experiments on cmake builds
@ -367,29 +369,32 @@ def expand_tests(name, srcs, deps, tags, args, exclude_pollers, uses_polling, us
"no_arm64",
]
experiment_config = list(poller_config)
for experiment in experiments:
for config in poller_config:
config = dict(config)
config["name"] = config["name"] + "@experiment=" + experiment
config["args"] = config["args"] + ["--experiment=" + experiment]
tags = config["tags"]
for tag in must_have_tags:
if tag not in tags:
tags = tags + [tag]
config["tags"] = tags
experiment_config.append(config)
for experiment in negated_experiments:
for config in poller_config:
config = dict(config)
config["name"] = config["name"] + "@experiment=no_" + experiment
config["args"] = config["args"] + ["--experiment=-" + experiment]
tags = config["tags"]
for tag in must_have_tags:
if tag not in tags:
tags = tags + [tag]
config["tags"] = tags
experiment_config.append(config)
for mode, config in mode_config.items():
enabled_tags, disabled_tags = config
if enabled_tags != None:
for experiment in experiments[mode]:
for config in poller_config:
config = dict(config)
config["name"] = config["name"] + "@experiment=" + experiment
config["args"] = config["args"] + ["--experiment=" + experiment]
tags = config["tags"]
for tag in must_have_tags + enabled_tags:
if tag not in tags:
tags = tags + [tag]
config["tags"] = tags
experiment_config.append(config)
if disabled_tags != None:
for experiment in experiments[mode]:
for config in poller_config:
config = dict(config)
config["name"] = config["name"] + "@experiment=no_" + experiment
config["args"] = config["args"] + ["--experiment=-" + experiment]
tags = config["tags"]
for tag in must_have_tags + disabled_tags:
if tag not in tags:
tags = tags + [tag]
config["tags"] = tags
experiment_config.append(config)
return experiment_config
def grpc_cc_test(name, srcs = [], deps = [], external_deps = [], args = [], data = [], uses_polling = True, language = "C++", size = "medium", timeout = None, tags = [], exec_compatible_with = [], exec_properties = {}, shard_count = None, flaky = None, copts = [], linkstatic = None, exclude_pollers = [], uses_event_engine = True):

8
src/core/lib/experiments/experiments.yaml
Unescape View File

@ -15,7 +15,13 @@
# Format of each entry:
# name: name of the experiment
# description: description of the experiment
# default: boolean - does this experiment default on
# default: one of:
# - false - the experiment defaults to off
# - debug - the experiment defaults to on in debug,
# off in release
# - release - the experiment defaults to on in release,
# off in debug
# - true - the experiment defaults to on
# expiry: when is the next time this experiment *must* be updated
# (date, YYYY/MM/DD)
# test_tags: a set of bazel tags, that if a test declares them signals

72
tools/codegen/core/gen_experiments.py
Unescape View File

@ -36,6 +36,20 @@ import yaml
with open('src/core/lib/experiments/experiments.yaml') as f:
attrs = yaml.load(f.read(), Loader=yaml.FullLoader)
DEFAULTS = {
False: 'false',
True: 'true',
'debug': 'kDefaultForDebugOnly',
'release': 'kDefaultForReleaseOnly',
}
BZL_LIST_FOR_DEFAULTS = {
False: 'off',
True: 'on',
'debug': 'dbg',
'release': 'opt',
}
error = False
today = datetime.date.today()
two_quarters_from_now = today + datetime.timedelta(days=180)
@ -50,6 +64,10 @@ for attr in attrs:
if 'default' not in attr:
print("no default for experiment %s" % attr['name'])
error = True
if attr['default'] not in DEFAULTS:
print("invalid default for experiment %s: %r" %
(attr['name'], attr['default']))
error = True
if 'expiry' not in attr:
print("no expiry for experiment %s" % attr['name'])
error = True
@ -172,6 +190,19 @@ with open('src/core/lib/experiments/experiments.cc', 'w') as C:
print("const char* const description_%s = %s;" %
(attr['name'], c_str(attr['description'])),
file=C)
have_defaults = set(DEFAULTS[attr['default']] for attr in attrs)
if 'kDefaultForDebugOnly' in have_defaults or 'kDefaultForReleaseOnly' in have_defaults:
print("#ifdef NDEBUG", file=C)
if 'kDefaultForDebugOnly' in have_defaults:
print("const bool kDefaultForDebugOnly = false;", file=C)
if 'kDefaultForReleaseOnly' in have_defaults:
print("const bool kDefaultForReleaseOnly = true;", file=C)
print("#else", file=C)
if 'kDefaultForDebugOnly' in have_defaults:
print("const bool kDefaultForDebugOnly = true;", file=C)
if 'kDefaultForReleaseOnly' in have_defaults:
print("const bool kDefaultForReleaseOnly = false;", file=C)
print("#endif", file=C)
print("}", file=C)
print(file=C)
print("namespace grpc_core {", file=C)
@ -179,22 +210,18 @@ with open('src/core/lib/experiments/experiments.cc', 'w') as C:
print("const ExperimentMetadata g_experiment_metadata[] = {", file=C)
for attr in attrs:
print(" {%s, description_%s, %s}," %
(c_str(attr['name']), attr['name'],
'true' if attr['default'] else 'false'),
(c_str(attr['name']), attr['name'], DEFAULTS[attr['default']]),
file=C)
print("};", file=C)
print(file=C)
print("} // namespace grpc_core", file=C)
tags_to_experiments = collections.defaultdict(list)
tags_to_negated_experiments = collections.defaultdict(list)
bzl_to_tags_to_experiments = dict((key, collections.defaultdict(list))
for key in BZL_LIST_FOR_DEFAULTS.keys())
for attr in attrs:
if attr['default']:
for tag in attr['test_tags']:
tags_to_negated_experiments[tag].append(attr['name'])
else:
for tag in attr['test_tags']:
tags_to_experiments[tag].append(attr['name'])
for tag in attr['test_tags']:
bzl_to_tags_to_experiments[attr['default']][tag].append(attr['name'])
with open('bazel/experiments.bzl', 'w') as B:
put_copyright(B, "#")
@ -208,19 +235,18 @@ with open('bazel/experiments.bzl', 'w') as B:
"\"\"\"Dictionary of tags to experiments so we know when to test different experiments.\"\"\"",
file=B)
bzl_to_tags_to_experiments = sorted(
(BZL_LIST_FOR_DEFAULTS[default], tags_to_experiments)
for default, tags_to_experiments in bzl_to_tags_to_experiments.items())
print(file=B)
print("EXPERIMENTS = {", file=B)
for tag, experiments in sorted(tags_to_experiments.items()):
print(" \"%s\": [" % tag, file=B)
for experiment in sorted(experiments):
print(" \"%s\"," % experiment, file=B)
print(" ],", file=B)
print("}", file=B)
print(file=B)
print("NEGATED_EXPERIMENTS = {", file=B)
for tag, experiments in sorted(tags_to_negated_experiments.items()):
print(" \"%s\": [" % tag, file=B)
for experiment in sorted(experiments):
print(" \"%s\"," % experiment, file=B)
print(" ],", file=B)
for key, tags_to_experiments in bzl_to_tags_to_experiments:
print(" \"%s\": {" % key, file=B)
for tag, experiments in sorted(tags_to_experiments.items()):
print(" \"%s\": [" % tag, file=B)
for experiment in sorted(experiments):
print(" \"%s\"," % experiment, file=B)
print(" ],", file=B)
print(" },", file=B)
print("}", file=B)

3
tools/remote_build/linux.bazelrc
Unescape View File

@ -34,6 +34,9 @@ import %workspace%/tools/remote_build/include/test_config_common.bazelrc
build --jobs=100
build:opt --test_tag_filters=-noopt
build:dbg --test_tag_filters=-nodbg
# address sanitizer: most settings are already in %workspace%/.bazelrc
# we only need a few additional ones that are Foundry specific
build:asan --copt=-gmlt

Write Preview
Loading…
Cancel
Save