diff --git a/bazel/experiments.bzl b/bazel/experiments.bzl index 6d9663b7772..0c0b83d2237 100644 --- a/bazel/experiments.bzl +++ b/bazel/experiments.bzl @@ -1,4 +1,4 @@ -# Copyright 2022 gRPC authors. +# Copyright 2023 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Automatically generated by tools/codegen/core/gen_experiments.py +# Auto generated by tools/codegen/core/gen_experiments.py """Dictionary of tags to experiments so we know when to test different experiments.""" diff --git a/src/core/lib/experiments/experiments.cc b/src/core/lib/experiments/experiments.cc index b86837118ee..dc48e9b928c 100644 --- a/src/core/lib/experiments/experiments.cc +++ b/src/core/lib/experiments/experiments.cc @@ -1,4 +1,4 @@ -// Copyright 2022 gRPC authors. +// Copyright 2023 gRPC authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Automatically generated by tools/codegen/core/gen_experiments.py +// Auto generated by tools/codegen/core/gen_experiments.py #include @@ -25,65 +25,67 @@ const char* const description_tcp_frame_size_tuning = "would not indicate completion of a read operation until a specified " "number of bytes have been read over the socket. Buffers are also " "allocated according to estimated RPC sizes."; -const char* const additional_constraints_tcp_frame_size_tuning = ""; +const char* const additional_constraints_tcp_frame_size_tuning = "{}"; const char* const description_tcp_rcv_lowat = "Use SO_RCVLOWAT to avoid wakeups on the read path."; -const char* const additional_constraints_tcp_rcv_lowat = ""; +const char* const additional_constraints_tcp_rcv_lowat = "{}"; const char* const description_peer_state_based_framing = "If set, the max sizes of frames sent to lower layers is controlled based " "on the peer's memory pressure which is reflected in its max http2 frame " "size."; -const char* const additional_constraints_peer_state_based_framing = ""; +const char* const additional_constraints_peer_state_based_framing = "{}"; const char* const description_memory_pressure_controller = "New memory pressure controller"; -const char* const additional_constraints_memory_pressure_controller = ""; +const char* const additional_constraints_memory_pressure_controller = "{}"; const char* const description_unconstrained_max_quota_buffer_size = "Discard the cap on the max free pool size for one memory allocator"; const char* const additional_constraints_unconstrained_max_quota_buffer_size = - ""; + "{}"; const char* const description_event_engine_client = "Use EventEngine clients instead of iomgr's grpc_tcp_client"; -const char* const additional_constraints_event_engine_client = ""; +const char* const additional_constraints_event_engine_client = "{}"; const char* const description_monitoring_experiment = "Placeholder experiment to prove/disprove our monitoring is working"; -const char* const additional_constraints_monitoring_experiment = ""; +const char* const additional_constraints_monitoring_experiment = "{}"; const char* const description_promise_based_client_call = "If set, use the new gRPC promise based call code when it's appropriate " "(ie when all filters in a stack are promise based)"; -const char* const additional_constraints_promise_based_client_call = ""; +const char* const additional_constraints_promise_based_client_call = "{}"; const char* const description_free_large_allocator = "If set, return all free bytes from a \042big\042 allocator"; -const char* const additional_constraints_free_large_allocator = ""; +const char* const additional_constraints_free_large_allocator = "{}"; const char* const description_promise_based_server_call = "If set, use the new gRPC promise based call code when it's appropriate " "(ie when all filters in a stack are promise based)"; -const char* const additional_constraints_promise_based_server_call = ""; +const char* const additional_constraints_promise_based_server_call = "{}"; const char* const description_transport_supplies_client_latency = "If set, use the transport represented value for client latency in " "opencensus"; -const char* const additional_constraints_transport_supplies_client_latency = ""; +const char* const additional_constraints_transport_supplies_client_latency = + "{}"; const char* const description_event_engine_listener = "Use EventEngine listeners instead of iomgr's grpc_tcp_server"; -const char* const additional_constraints_event_engine_listener = ""; +const char* const additional_constraints_event_engine_listener = "{}"; const char* const description_schedule_cancellation_over_write = "Allow cancellation op to be scheduled over a write"; -const char* const additional_constraints_schedule_cancellation_over_write = ""; +const char* const additional_constraints_schedule_cancellation_over_write = + "{}"; const char* const description_trace_record_callops = "Enables tracing of call batch initiation and completion."; -const char* const additional_constraints_trace_record_callops = ""; +const char* const additional_constraints_trace_record_callops = "{}"; const char* const description_event_engine_dns = "If set, use EventEngine DNSResolver for client channel resolution"; -const char* const additional_constraints_event_engine_dns = ""; +const char* const additional_constraints_event_engine_dns = "{}"; const char* const description_work_stealing = "If set, use a work stealing thread pool implementation in EventEngine"; -const char* const additional_constraints_work_stealing = ""; +const char* const additional_constraints_work_stealing = "{}"; const char* const description_client_privacy = "If set, client privacy"; -const char* const additional_constraints_client_privacy = ""; +const char* const additional_constraints_client_privacy = "{}"; const char* const description_canary_client_privacy = "If set, canary client privacy"; -const char* const additional_constraints_canary_client_privacy = ""; +const char* const additional_constraints_canary_client_privacy = "{}"; const char* const description_server_privacy = "If set, server privacy"; -const char* const additional_constraints_server_privacy = ""; +const char* const additional_constraints_server_privacy = "{}"; } // namespace namespace grpc_core { diff --git a/src/core/lib/experiments/experiments.h b/src/core/lib/experiments/experiments.h index e43e20c7fbc..ced23dd2b66 100644 --- a/src/core/lib/experiments/experiments.h +++ b/src/core/lib/experiments/experiments.h @@ -1,4 +1,4 @@ -// Copyright 2022 gRPC authors. +// Copyright 2023 gRPC authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Automatically generated by tools/codegen/core/gen_experiments.py +// Auto generated by tools/codegen/core/gen_experiments.py // // This file contains the autogenerated parts of the experiments API. // diff --git a/tools/codegen/core/experiments_compiler.py b/tools/codegen/core/experiments_compiler.py new file mode 100644 index 00000000000..8d9a7aeaf62 --- /dev/null +++ b/tools/codegen/core/experiments_compiler.py @@ -0,0 +1,396 @@ +#!/usr/bin/env python3 + +# Copyright 2023 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +A module to assist in generating experiment related code and artifacts. +""" + +from __future__ import print_function + +import collections +import ctypes +import datetime +import json +import math +import os +import re +import sys + +import yaml + +_CODEGEN_PLACEHOLDER_TEXT = """ +This file contains the autogenerated parts of the experiments API. + +It generates two symbols for each experiment. + +For the experiment named new_car_project, it generates: + +- a function IsNewCarProjectEnabled() that returns true if the experiment + should be enabled at runtime. + +- a macro GRPC_EXPERIMENT_IS_INCLUDED_NEW_CAR_PROJECT that is defined if the + experiment *could* be enabled at runtime. + +The function is used to determine whether to run the experiment or +non-experiment code path. + +If the experiment brings significant bloat, the macro can be used to avoid +including the experiment code path in the binary for binaries that are size +sensitive. + +By default that includes our iOS and Android builds. + +Finally, a small array is included that contains the metadata for each +experiment. + +A macro, GRPC_EXPERIMENTS_ARE_FINAL, controls whether we fix experiment +configuration at build time (if it's defined) or allow it to be tuned at +runtime (if it's disabled). + +If you are using the Bazel build system, that macro can be configured with +--define=grpc_experiments_are_final=true +""" + + +def ToCStr(s, encoding='ascii'): + if isinstance(s, str): + s = s.encode(encoding) + result = '' + for c in s: + c = chr(c) if isinstance(c, int) else c + if not (32 <= ord(c) < 127) or c in ('\\', '"'): + result += '\\%03o' % ord(c) + else: + result += c + return '"' + result + '"' + + +def SnakeToPascal(s): + return ''.join(x.capitalize() for x in s.split('_')) + + +def PutBanner(files, banner, prefix): + # Print a big comment block into a set of files + for f in files: + for line in banner: + if not line: + print(prefix, file=f) + else: + print('%s %s' % (prefix, line), file=f) + print(file=f) + + +def PutCopyright(file, prefix): + # copy-paste copyright notice from this file + with open(__file__) as my_source: + copyright = [] + for line in my_source: + if line[0] != '#': + break + for line in my_source: + if line[0] == '#': + copyright.append(line) + break + for line in my_source: + if line[0] != '#': + break + copyright.append(line) + PutBanner([file], [line[2:].rstrip() for line in copyright], prefix) + + +class ExperimentDefinition(object): + + def __init__(self, attributes): + self._error = False + if 'name' not in attributes: + print("ERROR: experiment with no name: %r" % attributes) + self._error = True + if 'description' not in attributes: + print("ERROR: no description for experiment %s" % + attributes['name']) + self._error = True + if 'owner' not in attributes: + print("ERROR: no owner for experiment %s" % attributes['name']) + self._error = True + if 'expiry' not in attributes: + print("ERROR: no expiry for experiment %s" % attributes['name']) + self._error = True + if attributes['name'] == 'monitoring_experiment': + if attributes['expiry'] != 'never-ever': + print("ERROR: monitoring_experiment should never expire") + self._error = True + if self._error: + print("Failed to create experiment definition") + return + self._allow_in_fuzzing_config = True + self._name = attributes['name'] + self._description = attributes['description'] + self._expiry = attributes['expiry'] + self._default = None + self._additional_constraints = {} + self._test_tags = [] + + if 'allow_in_fuzzing_config' in attributes: + self._allow_in_fuzzing_config = attributes[ + 'allow_in_fuzzing_config'] + + if 'test_tags' in attributes: + self._test_tags = attributes['test_tags'] + + def IsValid(self, check_expiry=False): + if self._error: + return False + if not check_expiry: + return True + if self._name == 'monitoring_experiment' and self._expiry == 'never-ever': + return True + today = datetime.date.today() + two_quarters_from_now = today + datetime.timedelta(days=180) + expiry = datetime.datetime.strptime(self._expiry, '%Y/%m/%d').date() + if expiry < today: + print("ERROR: experiment %s expired on %s" % + (self._name, self._expiry)) + self._error = True + if expiry > two_quarters_from_now: + print("ERROR: experiment %s expires far in the future on %s" % + (self._name, self._expiry)) + print("expiry should be no more than two quarters from now") + self._error = True + return not self._error + + def AddRolloutSpecification(self, allowed_defaults, rollout_attributes): + if self._error or self._default is not None: + return False + if rollout_attributes['name'] != self._name: + print( + "ERROR: Rollout specification does not apply to this experiment: %s" + % self._name) + return False + if 'default' not in rollout_attributes: + print("ERROR: no default for experiment %s" % + rollout_attributes['name']) + self._error = True + if rollout_attributes['default'] not in allowed_defaults: + print("ERROR: invalid default for experiment %s: %r" % + (rollout_attributes['name'], rollout_attributes['default'])) + self._error = True + if 'additional_constraints' in rollout_attributes: + self._additional_constraints = rollout_attributes[ + 'additional_constraints'] + self._default = rollout_attributes['default'] + return True + + @property + def name(self): + return self._name + + @property + def description(self): + return self._description + + @property + def default(self): + return self._default + + @property + def test_tags(self): + return self._test_tags + + @property + def allow_in_fuzzing_config(self): + return self._allow_in_fuzzing_config + + @property + def additional_constraints(self): + return self._additional_constraints + + +class ExperimentsCompiler(object): + + def __init__(self, + defaults, + final_return, + final_define, + bzl_list_for_defaults=None): + self._defaults = defaults + self._final_return = final_return + self._final_define = final_define + self._bzl_list_for_defaults = bzl_list_for_defaults + self._experiment_definitions = {} + self._experiment_rollouts = {} + + def AddExperimentDefinition(self, experiment_definition): + if experiment_definition.name in self._experiment_definitions: + print("ERROR: Duplicate experiment definition: %s" % + experiment_definition.name) + return False + self._experiment_definitions[ + experiment_definition.name] = experiment_definition + return True + + def AddRolloutSpecification(self, rollout_attributes): + if 'name' not in rollout_attributes: + print("ERROR: experiment with no name: %r in rollout_attribute" % + rollout_attributes) + return False + if rollout_attributes['name'] not in self._experiment_definitions: + print("WARNING: rollout for an undefined experiment: %s ignored" % + rollout_attributes['name']) + return (self._experiment_definitions[ + rollout_attributes['name']].AddRolloutSpecification( + self._defaults, rollout_attributes)) + + def GenerateExperimentsHdr(self, output_file): + with open(output_file, 'w') as H: + PutCopyright(H, "//") + PutBanner( + [H], + ["Auto generated by tools/codegen/core/gen_experiments.py"] + + _CODEGEN_PLACEHOLDER_TEXT.splitlines(), "//") + + print("#ifndef GRPC_SRC_CORE_LIB_EXPERIMENTS_EXPERIMENTS_H", file=H) + print("#define GRPC_SRC_CORE_LIB_EXPERIMENTS_EXPERIMENTS_H", file=H) + print(file=H) + print("#include ", file=H) + print(file=H) + print("#include ", file=H) + print("#include \"src/core/lib/experiments/config.h\"", file=H) + print(file=H) + print("namespace grpc_core {", file=H) + print(file=H) + print("#ifdef GRPC_EXPERIMENTS_ARE_FINAL", file=H) + for _, exp in self._experiment_definitions.items(): + define_fmt = self._final_define[exp.default] + if define_fmt: + print(define_fmt % + ("GRPC_EXPERIMENT_IS_INCLUDED_%s" % exp.name.upper()), + file=H) + print( + "inline bool Is%sEnabled() { %s }" % + (SnakeToPascal(exp.name), self._final_return[exp.default]), + file=H) + print("#else", file=H) + for i, (_, exp) in enumerate(self._experiment_definitions.items()): + print("#define GRPC_EXPERIMENT_IS_INCLUDED_%s" % + exp.name.upper(), + file=H) + print( + "inline bool Is%sEnabled() { return IsExperimentEnabled(%d); }" + % (SnakeToPascal(exp.name), i), + file=H) + print(file=H) + print("constexpr const size_t kNumExperiments = %d;" % + len(self._experiment_definitions.keys()), + file=H) + print( + "extern const ExperimentMetadata g_experiment_metadata[kNumExperiments];", + file=H) + print(file=H) + print("#endif", file=H) + print("} // namespace grpc_core", file=H) + print(file=H) + print("#endif // GRPC_SRC_CORE_LIB_EXPERIMENTS_EXPERIMENTS_H", + file=H) + + def GenerateExperimentsSrc(self, output_file): + with open(output_file, 'w') as C: + PutCopyright(C, "//") + PutBanner( + [C], + ["Auto generated by tools/codegen/core/gen_experiments.py"], + "//") + + print("#include ", file=C) + print("#include \"src/core/lib/experiments/experiments.h\"", file=C) + print(file=C) + print("#ifndef GRPC_EXPERIMENTS_ARE_FINAL", file=C) + print("namespace {", file=C) + have_defaults = set() + for _, exp in self._experiment_definitions.items(): + print("const char* const description_%s = %s;" % + (exp.name, ToCStr(exp.description)), + file=C) + print( + "const char* const additional_constraints_%s = %s;" % + (exp.name, ToCStr(json.dumps(exp.additional_constraints))), + file=C) + have_defaults.add(exp.default) + if 'kDefaultForDebugOnly' in have_defaults: + print("#ifdef NDEBUG", file=C) + if 'kDefaultForDebugOnly' in have_defaults: + print("const bool kDefaultForDebugOnly = false;", file=C) + print("#else", file=C) + if 'kDefaultForDebugOnly' in have_defaults: + print("const bool kDefaultForDebugOnly = true;", file=C) + print("#endif", file=C) + print("}", file=C) + print(file=C) + print("namespace grpc_core {", file=C) + print(file=C) + print("const ExperimentMetadata g_experiment_metadata[] = {", + file=C) + for _, exp in self._experiment_definitions.items(): + print( + " {%s, description_%s, additional_constraints_%s, %s, %s}," + % (ToCStr(exp.name), exp.name, exp.name, + 'true' if exp.default else 'false', + 'true' if exp.allow_in_fuzzing_config else 'false'), + file=C) + print("};", file=C) + print(file=C) + print("} // namespace grpc_core", file=C) + print("#endif", file=C) + + def GenExperimentsBzl(self, output_file): + if self._bzl_list_for_defaults is None: + return + + bzl_to_tags_to_experiments = dict( + (key, collections.defaultdict(list)) + for key in self._bzl_list_for_defaults.keys() + if key is not None) + + for _, exp in self._experiment_definitions.items(): + for tag in exp.test_tags: + bzl_to_tags_to_experiments[exp.default][tag].append(exp.name) + + with open(output_file, 'w') as B: + PutCopyright(B, "#") + PutBanner( + [B], + ["Auto generated by tools/codegen/core/gen_experiments.py"], + "#") + + print( + "\"\"\"Dictionary of tags to experiments so we know when to test different experiments.\"\"\"", + file=B) + + bzl_to_tags_to_experiments = sorted( + (self._bzl_list_for_defaults[default], tags_to_experiments) + for default, tags_to_experiments in + bzl_to_tags_to_experiments.items() + if self._bzl_list_for_defaults[default] is not None) + + print(file=B) + print("EXPERIMENTS = {", file=B) + for key, tags_to_experiments in bzl_to_tags_to_experiments: + print(" \"%s\": {" % key, file=B) + for tag, experiments in sorted(tags_to_experiments.items()): + print(" \"%s\": [" % tag, file=B) + for experiment in sorted(experiments): + print(" \"%s\"," % experiment, file=B) + print(" ],", file=B) + print(" },", file=B) + print("}", file=B) diff --git a/tools/codegen/core/gen_experiments.py b/tools/codegen/core/gen_experiments.py index 19af812fc99..0dac030e93e 100755 --- a/tools/codegen/core/gen_experiments.py +++ b/tools/codegen/core/gen_experiments.py @@ -22,28 +22,12 @@ Experiment definitions are in src/core/lib/experiments/experiments.yaml from __future__ import print_function -import collections -import ctypes -import datetime -import json -import math -import os -import re +import argparse import sys +import experiments_compiler as exp import yaml -# TODO(ctiller): if we ever add another argument switch this to argparse -check_dates = True -if sys.argv[1:] == ["--check"]: - check_dates = False # for formatting checks we don't verify expiry dates - -with open('src/core/lib/experiments/experiments.yaml') as f: - attrs = yaml.safe_load(f.read()) - -with open('src/core/lib/experiments/rollouts.yaml') as f: - rollouts = yaml.safe_load(f.read()) - DEFAULTS = { 'broken': 'false', False: 'false', @@ -72,279 +56,80 @@ BZL_LIST_FOR_DEFAULTS = { 'debug': 'dbg', } -error = False -today = datetime.date.today() -two_quarters_from_now = today + datetime.timedelta(days=180) -experiment_annotation = 'gRPC experiments:' -for rollout_attr in rollouts: - if 'name' not in rollout_attr: - print("experiment with no name: %r" % attr) - error = True - continue - if 'default' not in rollout_attr: - print("no default for experiment %s" % rollout_attr['name']) - error = True - if rollout_attr['default'] not in DEFAULTS: - print("invalid default for experiment %s: %r" % - (rollout_attr['name'], rollout_attr['default'])) - error = True -for attr in attrs: - if 'name' not in attr: - print("experiment with no name: %r" % attr) - error = True - continue # can't run other diagnostics because we don't know a name - if 'description' not in attr: - print("no description for experiment %s" % attr['name']) - error = True - if 'owner' not in attr: - print("no owner for experiment %s" % attr['name']) - error = True - if 'expiry' not in attr: - print("no expiry for experiment %s" % attr['name']) - error = True - if attr['name'] == 'monitoring_experiment': - if attr['expiry'] != 'never-ever': - print("monitoring_experiment should never expire") - error = True - else: - expiry = datetime.datetime.strptime(attr['expiry'], '%Y/%m/%d').date() - if check_dates: - if expiry < today: - print("experiment %s expired on %s" % - (attr['name'], attr['expiry'])) - error = True - if expiry > two_quarters_from_now: - print("experiment %s expires far in the future on %s" % - (attr['name'], attr['expiry'])) - print("expiry should be no more than two quarters from now") - error = True - experiment_annotation += attr['name'] + ':0,' - -if len(experiment_annotation) > 2000: - print("comma-delimited string of experiments is too long") - error = True - -if error: - sys.exit(1) - - -def c_str(s, encoding='ascii'): - if isinstance(s, str): - s = s.encode(encoding) - result = '' - for c in s: - c = chr(c) if isinstance(c, int) else c - if not (32 <= ord(c) < 127) or c in ('\\', '"'): - result += '\\%03o' % ord(c) - else: - result += c - return '"' + result + '"' - - -def snake_to_pascal(s): - return ''.join(x.capitalize() for x in s.split('_')) - - -# utility: print a big comment block into a set of files -def put_banner(files, banner, prefix): - for f in files: - for line in banner: - if not line: - print(prefix, file=f) - else: - print('%s %s' % (prefix, line), file=f) - print(file=f) - - -def put_copyright(file, prefix): - # copy-paste copyright notice from this file - with open(sys.argv[0]) as my_source: - copyright = [] - for line in my_source: - if line[0] != '#': - break - for line in my_source: - if line[0] == '#': - copyright.append(line) - break - for line in my_source: - if line[0] != '#': - break - copyright.append(line) - put_banner([file], [line[2:].rstrip() for line in copyright], prefix) - - -def get_rollout_attr_for_experiment(name): - for rollout_attr in rollouts: - if rollout_attr['name'] == name: - return rollout_attr - print('WARNING. experiment: %r has no rollout config. Disabling it.' % name) - return {'name': name, 'default': 'false'} +def ParseCommandLineArguments(args): + """Wrapper for argparse command line arguments handling. + + Args: + args: List of command line arguments. + + Returns: + Command line arguments namespace built by argparse.ArgumentParser(). + """ + # formatter_class=argparse.ArgumentDefaultsHelpFormatter is not used here + # intentionally, We want more formatting than this class can provide. + flag_parser = argparse.ArgumentParser() + flag_parser.add_argument( + '--check', + action='store_false', + help='If specified, disables checking experiment expiry dates', + ) + flag_parser.add_argument( + '--disable_gen_hdrs', + action='store_true', + help='If specified, disables generation of experiments hdr files', + ) + flag_parser.add_argument( + '--disable_gen_srcs', + action='store_true', + help='If specified, disables generation of experiments source files', + ) + flag_parser.add_argument( + '--disable_gen_bzl', + action='store_true', + help='If specified, disables generation of experiments.bzl file', + ) + return flag_parser.parse_args(args) + + +args = ParseCommandLineArguments(sys.argv[1:]) -WTF = """ -This file contains the autogenerated parts of the experiments API. - -It generates two symbols for each experiment. - -For the experiment named new_car_project, it generates: - -- a function IsNewCarProjectEnabled() that returns true if the experiment - should be enabled at runtime. - -- a macro GRPC_EXPERIMENT_IS_INCLUDED_NEW_CAR_PROJECT that is defined if the - experiment *could* be enabled at runtime. - -The function is used to determine whether to run the experiment or -non-experiment code path. - -If the experiment brings significant bloat, the macro can be used to avoid -including the experiment code path in the binary for binaries that are size -sensitive. - -By default that includes our iOS and Android builds. - -Finally, a small array is included that contains the metadata for each -experiment. - -A macro, GRPC_EXPERIMENTS_ARE_FINAL, controls whether we fix experiment -configuration at build time (if it's defined) or allow it to be tuned at -runtime (if it's disabled). - -If you are using the Bazel build system, that macro can be configured with ---define=grpc_experiments_are_final=true -""" - -with open('src/core/lib/experiments/experiments.h', 'w') as H: - put_copyright(H, "//") - - put_banner( - [H], - ["Automatically generated by tools/codegen/core/gen_experiments.py"] + - WTF.splitlines(), "//") - - print("#ifndef GRPC_SRC_CORE_LIB_EXPERIMENTS_EXPERIMENTS_H", file=H) - print("#define GRPC_SRC_CORE_LIB_EXPERIMENTS_EXPERIMENTS_H", file=H) - print(file=H) - print("#include ", file=H) - print(file=H) - print("#include ", file=H) - print("#include \"src/core/lib/experiments/config.h\"", file=H) - print(file=H) - print("namespace grpc_core {", file=H) - print(file=H) - print("#ifdef GRPC_EXPERIMENTS_ARE_FINAL", file=H) - for i, attr in enumerate(attrs): - rollout_attr = get_rollout_attr_for_experiment(attr['name']) - define_fmt = FINAL_DEFINE[rollout_attr['default']] - if define_fmt: - print(define_fmt % - ("GRPC_EXPERIMENT_IS_INCLUDED_%s" % attr['name'].upper()), - file=H) - print("inline bool Is%sEnabled() { %s }" % (snake_to_pascal( - attr['name']), FINAL_RETURN[rollout_attr['default']]), - file=H) - print("#else", file=H) - for i, attr in enumerate(attrs): - print("#define GRPC_EXPERIMENT_IS_INCLUDED_%s" % attr['name'].upper(), - file=H) - print("inline bool Is%sEnabled() { return IsExperimentEnabled(%d); }" % - (snake_to_pascal(attr['name']), i), - file=H) - print(file=H) - print("constexpr const size_t kNumExperiments = %d;" % len(attrs), file=H) - print( - "extern const ExperimentMetadata g_experiment_metadata[kNumExperiments];", - file=H) - print(file=H) - print("#endif", file=H) - print("} // namespace grpc_core", file=H) - print(file=H) - print("#endif // GRPC_SRC_CORE_LIB_EXPERIMENTS_EXPERIMENTS_H", file=H) - -with open('src/core/lib/experiments/experiments.cc', 'w') as C: - put_copyright(C, "//") - - put_banner( - [C], - ["Automatically generated by tools/codegen/core/gen_experiments.py"], - "//") +with open('src/core/lib/experiments/experiments.yaml') as f: + attrs = yaml.safe_load(f.read()) - print("#include ", file=C) - print("#include \"src/core/lib/experiments/experiments.h\"", file=C) - print(file=C) - print("#ifndef GRPC_EXPERIMENTS_ARE_FINAL", file=C) - print("namespace {", file=C) - for attr in attrs: - print("const char* const description_%s = %s;" % - (attr['name'], c_str(attr['description'])), - file=C) - print("const char* const additional_constraints_%s = \"\";" % - attr['name'], - file=C) - have_defaults = set( - DEFAULTS[rollout_attr['default']] for rollout_attr in rollouts) - if 'kDefaultForDebugOnly' in have_defaults: - print("#ifdef NDEBUG", file=C) - if 'kDefaultForDebugOnly' in have_defaults: - print("const bool kDefaultForDebugOnly = false;", file=C) - print("#else", file=C) - if 'kDefaultForDebugOnly' in have_defaults: - print("const bool kDefaultForDebugOnly = true;", file=C) - print("#endif", file=C) - print("}", file=C) - print(file=C) - print("namespace grpc_core {", file=C) - print(file=C) - print("const ExperimentMetadata g_experiment_metadata[] = {", file=C) - for attr in attrs: - rollout_attr = get_rollout_attr_for_experiment(attr['name']) - print( - " {%s, description_%s, additional_constraints_%s, %s, %s}," % - (c_str(attr['name']), attr['name'], attr['name'], - DEFAULTS[rollout_attr['default']], - 'true' if attr.get('allow_in_fuzzing_config', True) else 'false'), - file=C) - print("};", file=C) - print(file=C) - print("} // namespace grpc_core", file=C) - print("#endif", file=C) +with open('src/core/lib/experiments/rollouts.yaml') as f: + rollouts = yaml.safe_load(f.read()) -bzl_to_tags_to_experiments = dict((key, collections.defaultdict(list)) - for key in BZL_LIST_FOR_DEFAULTS.keys() - if key is not None) +compiler = exp.ExperimentsCompiler(DEFAULTS, FINAL_RETURN, FINAL_DEFINE, + BZL_LIST_FOR_DEFAULTS) +experiment_annotation = "gRPC Experiments: " for attr in attrs: - rollout_attr = get_rollout_attr_for_experiment(attr['name']) - for tag in attr['test_tags']: - bzl_to_tags_to_experiments[rollout_attr['default']][tag].append( - attr['name']) + exp_definition = exp.ExperimentDefinition(attr) + if not exp_definition.IsValid(args.check): + sys.exit(1) + experiment_annotation += exp_definition.name + ':0,' + if not compiler.AddExperimentDefinition(exp_definition): + print("Experiment = %s ERROR adding" % exp_definition.name) + sys.exit(1) -with open('bazel/experiments.bzl', 'w') as B: - put_copyright(B, "#") +if len(experiment_annotation) > 2000: + print("comma-delimited string of experiments is too long") + sys.exit(1) - put_banner( - [B], - ["Automatically generated by tools/codegen/core/gen_experiments.py"], - "#") +for rollout_attr in rollouts: + if not compiler.AddRolloutSpecification(rollout_attr): + print("ERROR adding rollout spec") + sys.exit(1) - print( - "\"\"\"Dictionary of tags to experiments so we know when to test different experiments.\"\"\"", - file=B) +if not args.disable_gen_hdrs: + print("Generating experiments headers") + compiler.GenerateExperimentsHdr('src/core/lib/experiments/experiments.h') - bzl_to_tags_to_experiments = sorted( - (BZL_LIST_FOR_DEFAULTS[default], tags_to_experiments) - for default, tags_to_experiments in bzl_to_tags_to_experiments.items() - if BZL_LIST_FOR_DEFAULTS[default] is not None) +if not args.disable_gen_srcs: + print("Generating experiments srcs") + compiler.GenerateExperimentsSrc('src/core/lib/experiments/experiments.cc') - print(file=B) - print("EXPERIMENTS = {", file=B) - for key, tags_to_experiments in bzl_to_tags_to_experiments: - print(" \"%s\": {" % key, file=B) - for tag, experiments in sorted(tags_to_experiments.items()): - print(" \"%s\": [" % tag, file=B) - for experiment in sorted(experiments): - print(" \"%s\"," % experiment, file=B) - print(" ],", file=B) - print(" },", file=B) - print("}", file=B) +if not args.disable_gen_bzl: + print("Generating experiments.bzl") + compiler.GenExperimentsBzl('bazel/experiments.bzl')