#!/usr/bin/env python3 # Copyright 2022 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ Generate experiment related code artifacts. Invoke as: tools/codegen/core/gen_experiments.py Experiment definitions are in src/core/lib/experiments/experiments.yaml """ from __future__ import print_function import collections import ctypes import datetime import json import math import os import re import sys import yaml # TODO(ctiller): if we ever add another argument switch this to argparse check_dates = True if sys.argv[1:] == ["--check"]: check_dates = False # for formatting checks we don't verify expiry dates with open('src/core/lib/experiments/experiments.yaml') as f: attrs = yaml.load(f.read(), Loader=yaml.FullLoader) DEFAULTS = { 'broken': 'false', False: 'false', True: 'true', 'debug': 'kDefaultForDebugOnly', 'release': 'kDefaultForReleaseOnly', } BZL_LIST_FOR_DEFAULTS = { 'broken': None, False: 'off', True: 'on', 'debug': 'dbg', 'release': 'opt', } error = False today = datetime.date.today() two_quarters_from_now = today + datetime.timedelta(days=180) for attr in attrs: if 'name' not in attr: print("experiment with no name: %r" % attr) error = True continue # can't run other diagnostics because we don't know a name if 'description' not in attr: print("no description for experiment %s" % attr['name']) error = True if 'default' not in attr: print("no default for experiment %s" % attr['name']) error = True if attr['default'] not in DEFAULTS: print("invalid default for experiment %s: %r" % (attr['name'], attr['default'])) error = True if 'owner' not in attr: print("no owner for experiment %s" % attr['name']) error = True if 'expiry' not in attr: print("no expiry for experiment %s" % attr['name']) error = True expiry = datetime.datetime.strptime(attr['expiry'], '%Y/%m/%d').date() if check_dates: if expiry < today: print("experiment %s expired on %s" % (attr['name'], attr['expiry'])) error = True if expiry > two_quarters_from_now: print("experiment %s expires far in the future on %s" % (attr['name'], attr['expiry'])) print("expiry should be no more than two quarters from now") error = True if error: sys.exit(1) def c_str(s, encoding='ascii'): if isinstance(s, str): s = s.encode(encoding) result = '' for c in s: c = chr(c) if isinstance(c, int) else c if not (32 <= ord(c) < 127) or c in ('\\', '"'): result += '\\%03o' % ord(c) else: result += c return '"' + result + '"' def snake_to_pascal(s): return ''.join(x.capitalize() for x in s.split('_')) # utility: print a big comment block into a set of files def put_banner(files, banner, prefix): for f in files: for line in banner: if not line: print(prefix, file=f) else: print('%s %s' % (prefix, line), file=f) print(file=f) def put_copyright(file, prefix): # copy-paste copyright notice from this file with open(sys.argv[0]) as my_source: copyright = [] for line in my_source: if line[0] != '#': break for line in my_source: if line[0] == '#': copyright.append(line) break for line in my_source: if line[0] != '#': break copyright.append(line) put_banner([file], [line[2:].rstrip() for line in copyright], prefix) EXPERIMENT_METADATA = """struct ExperimentMetadata { const char* name; const char* description; bool default_value; };""" with open('src/core/lib/experiments/experiments.h', 'w') as H: put_copyright(H, "//") put_banner( [H], ["Automatically generated by tools/codegen/core/gen_experiments.py"], "//") print("#ifndef GRPC_CORE_LIB_EXPERIMENTS_EXPERIMENTS_H", file=H) print("#define GRPC_CORE_LIB_EXPERIMENTS_EXPERIMENTS_H", file=H) print(file=H) print("#include ", file=H) print(file=H) print("#include ", file=H) print("#include \"src/core/lib/experiments/config.h\"", file=H) print(file=H) print("namespace grpc_core {", file=H) print(file=H) for i, attr in enumerate(attrs): print("inline bool Is%sEnabled() { return IsExperimentEnabled(%d); }" % (snake_to_pascal(attr['name']), i), file=H) print(file=H) print(EXPERIMENT_METADATA, file=H) print(file=H) print("constexpr const size_t kNumExperiments = %d;" % len(attrs), file=H) print( "extern const ExperimentMetadata g_experiment_metadata[kNumExperiments];", file=H) print(file=H) print("} // namespace grpc_core", file=H) print(file=H) print("#endif // GRPC_CORE_LIB_EXPERIMENTS_EXPERIMENTS_H", file=H) with open('src/core/lib/experiments/experiments.cc', 'w') as C: put_copyright(C, "//") put_banner( [C], ["Automatically generated by tools/codegen/core/gen_experiments.py"], "//") print("#include ", file=C) print("#include \"src/core/lib/experiments/experiments.h\"", file=C) print(file=C) print("namespace {", file=C) for attr in attrs: print("const char* const description_%s = %s;" % (attr['name'], c_str(attr['description'])), file=C) have_defaults = set(DEFAULTS[attr['default']] for attr in attrs) if 'kDefaultForDebugOnly' in have_defaults or 'kDefaultForReleaseOnly' in have_defaults: print("#ifdef NDEBUG", file=C) if 'kDefaultForDebugOnly' in have_defaults: print("const bool kDefaultForDebugOnly = false;", file=C) if 'kDefaultForReleaseOnly' in have_defaults: print("const bool kDefaultForReleaseOnly = true;", file=C) print("#else", file=C) if 'kDefaultForDebugOnly' in have_defaults: print("const bool kDefaultForDebugOnly = true;", file=C) if 'kDefaultForReleaseOnly' in have_defaults: print("const bool kDefaultForReleaseOnly = false;", file=C) print("#endif", file=C) print("}", file=C) print(file=C) print("namespace grpc_core {", file=C) print(file=C) print("const ExperimentMetadata g_experiment_metadata[] = {", file=C) for attr in attrs: print(" {%s, description_%s, %s}," % (c_str(attr['name']), attr['name'], DEFAULTS[attr['default']]), file=C) print("};", file=C) print(file=C) print("} // namespace grpc_core", file=C) bzl_to_tags_to_experiments = dict((key, collections.defaultdict(list)) for key in BZL_LIST_FOR_DEFAULTS.keys() if key is not None) for attr in attrs: for tag in attr['test_tags']: bzl_to_tags_to_experiments[attr['default']][tag].append(attr['name']) with open('bazel/experiments.bzl', 'w') as B: put_copyright(B, "#") put_banner( [B], ["Automatically generated by tools/codegen/core/gen_experiments.py"], "#") print( "\"\"\"Dictionary of tags to experiments so we know when to test different experiments.\"\"\"", file=B) bzl_to_tags_to_experiments = sorted( (BZL_LIST_FOR_DEFAULTS[default], tags_to_experiments) for default, tags_to_experiments in bzl_to_tags_to_experiments.items() if BZL_LIST_FOR_DEFAULTS[default] is not None) print(file=B) print("EXPERIMENTS = {", file=B) for key, tags_to_experiments in bzl_to_tags_to_experiments: print(" \"%s\": {" % key, file=B) for tag, experiments in sorted(tags_to_experiments.items()): print(" \"%s\": [" % tag, file=B) for experiment in sorted(experiments): print(" \"%s\"," % experiment, file=B) print(" ],", file=B) print(" },", file=B) print("}", file=B)