From 57d8462aee47ead0d2cba51ee7d516a8ab6c6d83 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 29 Nov 2023 11:29:46 -0800 Subject: [PATCH] Remove IWYU requirements for changes PiperOrigin-RevId: 586404245 --- .../dockerfile/grpc_iwyu/Dockerfile.template | 30 - tools/buildgen/generate_projects.sh | 2 - tools/distrib/fix_build_deps.py | 684 ------------------ tools/distrib/iwyu.sh | 63 -- tools/distrib/iwyu_mappings.imp | 9 - tools/dockerfile/grpc_iwyu/Dockerfile | 28 - tools/dockerfile/grpc_iwyu/iwyu.sh | 139 ---- tools/internal_ci/linux/grpc_iwyu.cfg | 30 - .../linux/pull_request/grpc_iwyu.cfg | 30 - tools/run_tests/run_tests.py | 1 - tools/run_tests/sanity/iwyu_tests.yaml | 2 - 11 files changed, 1018 deletions(-) delete mode 100644 templates/tools/dockerfile/grpc_iwyu/Dockerfile.template delete mode 100755 tools/distrib/fix_build_deps.py delete mode 100755 tools/distrib/iwyu.sh delete mode 100644 tools/distrib/iwyu_mappings.imp delete mode 100644 tools/dockerfile/grpc_iwyu/Dockerfile delete mode 100755 tools/dockerfile/grpc_iwyu/iwyu.sh delete mode 100644 tools/internal_ci/linux/grpc_iwyu.cfg delete mode 100644 tools/internal_ci/linux/pull_request/grpc_iwyu.cfg delete mode 100644 tools/run_tests/sanity/iwyu_tests.yaml diff --git a/templates/tools/dockerfile/grpc_iwyu/Dockerfile.template b/templates/tools/dockerfile/grpc_iwyu/Dockerfile.template deleted file mode 100644 index ecb4deef283..00000000000 --- a/templates/tools/dockerfile/grpc_iwyu/Dockerfile.template +++ /dev/null @@ -1,30 +0,0 @@ -%YAML 1.2 ---- | - # Copyright 2021 gRPC authors. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - - # Docker file for running IWYU. - # Updated: 2022-11-03 - - FROM silkeh/clang:16-bullseye - - # Install prerequisites for the iwyu script - RUN apt-get update && apt-get install -y python3 jq git cmake python zlib1g-dev libtinfo-dev libclang-16-dev && apt-get clean - ADD iwyu.sh / - - # When running locally, we'll be impersonating the current user, so we need - # to make the script runnable by everyone. - RUN chmod a+rx /iwyu.sh - - CMD ["echo 'Run with tools/distrib/iwyu.sh'"] diff --git a/tools/buildgen/generate_projects.sh b/tools/buildgen/generate_projects.sh index 9ddd588bb37..e7905a40c95 100755 --- a/tools/buildgen/generate_projects.sh +++ b/tools/buildgen/generate_projects.sh @@ -28,8 +28,6 @@ fi cd `dirname $0`/../.. -tools/distrib/fix_build_deps.py - echo "Generating build_autogenerated.yaml from bazel BUILD file" rm -f build_autogenerated.yaml python3 tools/buildgen/extract_metadata_from_bazel_xml.py diff --git a/tools/distrib/fix_build_deps.py b/tools/distrib/fix_build_deps.py deleted file mode 100755 index 0c014ec2b76..00000000000 --- a/tools/distrib/fix_build_deps.py +++ /dev/null @@ -1,684 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2022 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import collections -from doctest import SKIP -import multiprocessing -import os -import re -import sys - -import run_buildozer - -# find our home -ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "../..")) -os.chdir(ROOT) - -vendors = collections.defaultdict(list) -scores = collections.defaultdict(int) -avoidness = collections.defaultdict(int) -consumes = {} -no_update = set() -buildozer_commands = [] -original_deps = {} -original_external_deps = {} -skip_headers = collections.defaultdict(set) - -# TODO(ctiller): ideally we wouldn't hardcode a bunch of paths here. -# We can likely parse out BUILD files from dependencies to generate this index. -EXTERNAL_DEPS = { - "absl/algorithm/container.h": "absl/algorithm:container", - "absl/base/attributes.h": "absl/base:core_headers", - "absl/base/call_once.h": "absl/base", - # TODO(ctiller) remove this - "absl/base/internal/endian.h": "absl/base:endian", - "absl/base/thread_annotations.h": "absl/base:core_headers", - "absl/container/flat_hash_map.h": "absl/container:flat_hash_map", - "absl/container/flat_hash_set.h": "absl/container:flat_hash_set", - "absl/container/inlined_vector.h": "absl/container:inlined_vector", - "absl/cleanup/cleanup.h": "absl/cleanup", - "absl/debugging/failure_signal_handler.h": ( - "absl/debugging:failure_signal_handler" - ), - "absl/debugging/stacktrace.h": "absl/debugging:stacktrace", - "absl/debugging/symbolize.h": "absl/debugging:symbolize", - "absl/flags/flag.h": "absl/flags:flag", - "absl/flags/marshalling.h": "absl/flags:marshalling", - "absl/flags/parse.h": "absl/flags:parse", - "absl/functional/any_invocable.h": "absl/functional:any_invocable", - "absl/functional/bind_front.h": "absl/functional:bind_front", - "absl/functional/function_ref.h": "absl/functional:function_ref", - "absl/hash/hash.h": "absl/hash", - "absl/memory/memory.h": "absl/memory", - "absl/meta/type_traits.h": "absl/meta:type_traits", - "absl/numeric/int128.h": "absl/numeric:int128", - "absl/random/random.h": "absl/random", - "absl/random/bit_gen_ref.h": "absl/random:bit_gen_ref", - "absl/random/mocking_bit_gen.h": "absl/random:mocking_bit_gen", - "absl/random/distributions.h": "absl/random:distributions", - "absl/random/uniform_int_distribution.h": "absl/random:distributions", - "absl/status/status.h": "absl/status", - "absl/status/statusor.h": "absl/status:statusor", - "absl/strings/ascii.h": "absl/strings", - "absl/strings/cord.h": "absl/strings:cord", - "absl/strings/escaping.h": "absl/strings", - "absl/strings/match.h": "absl/strings", - "absl/strings/numbers.h": "absl/strings", - "absl/strings/str_cat.h": "absl/strings", - "absl/strings/str_format.h": "absl/strings:str_format", - "absl/strings/str_join.h": "absl/strings", - "absl/strings/str_replace.h": "absl/strings", - "absl/strings/str_split.h": "absl/strings", - "absl/strings/string_view.h": "absl/strings", - "absl/strings/strip.h": "absl/strings", - "absl/strings/substitute.h": "absl/strings", - "absl/synchronization/mutex.h": "absl/synchronization", - "absl/synchronization/notification.h": "absl/synchronization", - "absl/time/clock.h": "absl/time", - "absl/time/time.h": "absl/time", - "absl/types/optional.h": "absl/types:optional", - "absl/types/span.h": "absl/types:span", - "absl/types/variant.h": "absl/types:variant", - "absl/utility/utility.h": "absl/utility", - "address_sorting/address_sorting.h": "address_sorting", - "google/cloud/opentelemetry/resource_detector.h": "google_cloud_cpp:opentelemetry", - "opentelemetry/common/attribute_value.h": "otel/api", - "opentelemetry/common/key_value_iterable.h": "otel/api", - "opentelemetry/nostd/function_ref.h": "otel/api", - "opentelemetry/nostd/string_view.h": "otel/api", - "opentelemetry/context/context.h": "otel/api", - "opentelemetry/metrics/meter.h": "otel/api", - "opentelemetry/metrics/meter_provider.h": "otel/api", - "opentelemetry/metrics/provider.h": "otel/api", - "opentelemetry/metrics/sync_instruments.h": "otel/api", - "opentelemetry/nostd/shared_ptr.h": "otel/api", - "opentelemetry/nostd/unique_ptr.h": "otel/api", - "opentelemetry/sdk/metrics/meter_provider.h": "otel/sdk/src/metrics", - "opentelemetry/sdk/common/attribute_utils.h": "otel/sdk:headers", - "opentelemetry/sdk/resource/resource.h": "otel/sdk:headers", - "opentelemetry/sdk/resource/resource_detector.h": "otel/sdk:headers", - "opentelemetry/sdk/resource/semantic_conventions.h": "otel/sdk:headers", - "ares.h": "cares", - "fuzztest/fuzztest.h": ["fuzztest", "fuzztest_main"], - "google/api/monitored_resource.pb.h": ( - "google/api:monitored_resource_cc_proto" - ), - "google/devtools/cloudtrace/v2/tracing.grpc.pb.h": ( - "googleapis_trace_grpc_service" - ), - "google/logging/v2/logging.grpc.pb.h": "googleapis_logging_grpc_service", - "google/logging/v2/logging.pb.h": "googleapis_logging_cc_proto", - "google/logging/v2/log_entry.pb.h": "googleapis_logging_cc_proto", - "google/monitoring/v3/metric_service.grpc.pb.h": ( - "googleapis_monitoring_grpc_service" - ), - "gmock/gmock.h": "gtest", - "gtest/gtest.h": "gtest", - "opencensus/exporters/stats/stackdriver/stackdriver_exporter.h": ( - "opencensus-stats-stackdriver_exporter" - ), - "opencensus/exporters/trace/stackdriver/stackdriver_exporter.h": ( - "opencensus-trace-stackdriver_exporter" - ), - "opencensus/trace/context_util.h": "opencensus-trace-context_util", - "opencensus/trace/propagation/grpc_trace_bin.h": ( - "opencensus-trace-propagation" - ), - "opencensus/tags/context_util.h": "opencensus-tags-context_util", - "opencensus/trace/span_context.h": "opencensus-trace-span_context", - "openssl/base.h": "libssl", - "openssl/bio.h": "libssl", - "openssl/bn.h": "libcrypto", - "openssl/buffer.h": "libcrypto", - "openssl/crypto.h": "libcrypto", - "openssl/digest.h": "libssl", - "openssl/engine.h": "libcrypto", - "openssl/err.h": "libcrypto", - "openssl/evp.h": "libcrypto", - "openssl/hmac.h": "libcrypto", - "openssl/mem.h": "libcrypto", - "openssl/param_build.h": "libcrypto", - "openssl/pem.h": "libcrypto", - "openssl/rsa.h": "libcrypto", - "openssl/sha.h": "libcrypto", - "openssl/ssl.h": "libssl", - "openssl/tls1.h": "libssl", - "openssl/x509.h": "libcrypto", - "openssl/x509v3.h": "libcrypto", - "re2/re2.h": "re2", - "upb/base/string_view.h": "upb_base_lib", - "upb/collections/map.h": "upb_collections_lib", - "upb/reflection/def.h": "upb_reflection", - "upb/json/encode.h": "upb_json_lib", - "upb/mem/arena.h": "upb_mem_lib", - "upb/text/encode.h": "upb_textformat_lib", - "upb/reflection/def.hpp": "upb_reflection", - "upb/upb.h": "upb_amalgamation_lib", - "upb/upb.hpp": "upb_lib", - "xxhash.h": "xxhash", - "zlib.h": "madler_zlib", -} - -INTERNAL_DEPS = { - "test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.h": ( - "//test/core/event_engine/fuzzing_event_engine" - ), - "test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.pb.h": "//test/core/event_engine/fuzzing_event_engine:fuzzing_event_engine_proto", - "test/core/experiments/test_experiments.h": "//test/core/experiments:test_experiments_lib", - "google/api/expr/v1alpha1/syntax.upb.h": "google_api_expr_v1alpha1_syntax_upb", - "google/rpc/status.upb.h": "google_rpc_status_upb", - "google/protobuf/any.upb.h": "protobuf_any_upb", - "google/protobuf/duration.upb.h": "protobuf_duration_upb", - "google/protobuf/struct.upb.h": "protobuf_struct_upb", - "google/protobuf/timestamp.upb.h": "protobuf_timestamp_upb", - "google/protobuf/wrappers.upb.h": "protobuf_wrappers_upb", - "grpc/status.h": "grpc_public_hdrs", - "src/proto/grpc/channelz/channelz.grpc.pb.h": ( - "//src/proto/grpc/channelz:channelz_proto" - ), - "src/proto/grpc/core/stats.pb.h": "//src/proto/grpc/core:stats_proto", - "src/proto/grpc/health/v1/health.upb.h": "grpc_health_upb", - "src/proto/grpc/lb/v1/load_reporter.grpc.pb.h": ( - "//src/proto/grpc/lb/v1:load_reporter_proto" - ), - "src/proto/grpc/lb/v1/load_balancer.upb.h": "grpc_lb_upb", - "src/proto/grpc/reflection/v1alpha/reflection.grpc.pb.h": ( - "//src/proto/grpc/reflection/v1alpha:reflection_proto" - ), - "src/proto/grpc/gcp/transport_security_common.upb.h": "alts_upb", - "src/proto/grpc/gcp/handshaker.upb.h": "alts_upb", - "src/proto/grpc/gcp/altscontext.upb.h": "alts_upb", - "src/proto/grpc/lookup/v1/rls.upb.h": "rls_upb", - "src/proto/grpc/lookup/v1/rls_config.upb.h": "rls_config_upb", - "src/proto/grpc/lookup/v1/rls_config.upbdefs.h": "rls_config_upbdefs", - "src/proto/grpc/testing/xds/v3/csds.grpc.pb.h": ( - "//src/proto/grpc/testing/xds/v3:csds_proto" - ), - "xds/data/orca/v3/orca_load_report.upb.h": "xds_orca_upb", - "xds/service/orca/v3/orca.upb.h": "xds_orca_service_upb", - "xds/type/v3/typed_struct.upb.h": "xds_type_upb", -} - - -class FakeSelects: - def config_setting_group(self, **kwargs): - pass - - -num_cc_libraries = 0 -num_opted_out_cc_libraries = 0 -parsing_path = None - - -# Convert the source or header target to a relative path. -def _get_filename(name, parsing_path): - filename = "%s%s" % ( - ( - parsing_path + "/" - if (parsing_path and not name.startswith("//")) - else "" - ), - name, - ) - filename = filename.replace("//:", "") - filename = filename.replace("//src/core:", "src/core/") - filename = filename.replace( - "//src/cpp/ext/filters/census:", "src/cpp/ext/filters/census/" - ) - return filename - - -def grpc_cc_library( - name, - hdrs=[], - public_hdrs=[], - srcs=[], - select_deps=None, - tags=[], - deps=[], - external_deps=[], - proto=None, - **kwargs, -): - global args - global num_cc_libraries - global num_opted_out_cc_libraries - global parsing_path - assert parsing_path is not None - name = "//%s:%s" % (parsing_path, name) - num_cc_libraries += 1 - if select_deps or "nofixdeps" in tags: - if args.whats_left and not select_deps and "nofixdeps" not in tags: - num_opted_out_cc_libraries += 1 - print("Not opted in: {}".format(name)) - no_update.add(name) - scores[name] = len(public_hdrs + hdrs) - # avoid_dep is the internal way of saying prefer something else - # we add grpc_avoid_dep to allow internal grpc-only stuff to avoid each - # other, whilst not biasing dependent projects - if "avoid_dep" in tags or "grpc_avoid_dep" in tags: - avoidness[name] += 10 - if proto: - proto_hdr = "%s%s" % ( - (parsing_path + "/" if parsing_path else ""), - proto.replace(".proto", ".pb.h"), - ) - skip_headers[name].add(proto_hdr) - - for hdr in hdrs + public_hdrs: - vendors[_get_filename(hdr, parsing_path)].append(name) - inc = set() - original_deps[name] = frozenset(deps) - original_external_deps[name] = frozenset(external_deps) - for src in hdrs + public_hdrs + srcs: - for line in open(_get_filename(src, parsing_path)): - m = re.search(r"^#include <(.*)>", line) - if m: - inc.add(m.group(1)) - m = re.search(r'^#include "(.*)"', line) - if m: - inc.add(m.group(1)) - consumes[name] = list(inc) - - -def grpc_proto_library(name, srcs, **kwargs): - global parsing_path - assert parsing_path is not None - name = "//%s:%s" % (parsing_path, name) - for src in srcs: - proto_hdr = src.replace(".proto", ".pb.h") - vendors[_get_filename(proto_hdr, parsing_path)].append(name) - - -def buildozer(cmd, target): - buildozer_commands.append("%s|%s" % (cmd, target)) - - -def buildozer_set_list(name, values, target, via=""): - if not values: - buildozer("remove %s" % name, target) - return - adjust = via if via else name - buildozer( - "set %s %s" % (adjust, " ".join('"%s"' % s for s in values)), target - ) - if via: - buildozer("remove %s" % name, target) - buildozer("rename %s %s" % (via, name), target) - - -def score_edit_distance(proposed, existing): - """Score a proposed change primarily by edit distance""" - sum = 0 - for p in proposed: - if p not in existing: - sum += 1 - for e in existing: - if e not in proposed: - sum += 1 - return sum - - -def total_score(proposal): - return sum(scores[dep] for dep in proposal) - - -def total_avoidness(proposal): - return sum(avoidness[dep] for dep in proposal) - - -def score_list_size(proposed, existing): - """Score a proposed change primarily by number of dependencies""" - return len(proposed) - - -def score_best(proposed, existing): - """Score a proposed change primarily by dependency score""" - return 0 - - -SCORERS = { - "edit_distance": score_edit_distance, - "list_size": score_list_size, - "best": score_best, -} - -parser = argparse.ArgumentParser(description="Fix build dependencies") -parser.add_argument( - "targets", nargs="*", default=[], help="targets to fix (empty => all)" -) -parser.add_argument( - "--score", - type=str, - default="edit_distance", - help="scoring function to use: one of " + ", ".join(SCORERS.keys()), -) -parser.add_argument( - "--whats_left", - action="store_true", - default=False, - help="show what is left to opt in", -) -parser.add_argument( - "--explain", - action="store_true", - default=False, - help="try to explain some decisions", -) -parser.add_argument( - "--why", - type=str, - default=None, - help="with --explain, target why a given dependency is needed", -) -args = parser.parse_args() - -for dirname in [ - "", - "src/core", - "src/cpp/ext/gcp", - "src/cpp/ext/csm", - "src/cpp/ext/otel", - "test/core/backoff", - "test/core/experiments", - "test/core/uri", - "test/core/util", - "test/core/end2end", - "test/core/event_engine", - "test/core/filters", - "test/core/promise", - "test/core/resource_quota", - "test/core/transport/chaotic_good", - "fuzztest", - "fuzztest/core/channel", - "fuzztest/core/transport/chttp2", -]: - parsing_path = dirname - exec( - open("%sBUILD" % (dirname + "/" if dirname else ""), "r").read(), - { - "load": lambda filename, *args: None, - "licenses": lambda licenses: None, - "package": lambda **kwargs: None, - "exports_files": lambda files, visibility=None: None, - "bool_flag": lambda **kwargs: None, - "config_setting": lambda **kwargs: None, - "selects": FakeSelects(), - "python_config_settings": lambda **kwargs: None, - "grpc_cc_binary": grpc_cc_library, - "grpc_cc_library": grpc_cc_library, - "grpc_cc_test": grpc_cc_library, - "grpc_core_end2end_test": lambda **kwargs: None, - "grpc_fuzzer": grpc_cc_library, - "grpc_fuzz_test": grpc_cc_library, - "grpc_proto_fuzzer": grpc_cc_library, - "grpc_proto_library": grpc_proto_library, - "select": lambda d: d["//conditions:default"], - "glob": lambda files: None, - "grpc_end2end_tests": lambda: None, - "grpc_upb_proto_library": lambda name, **kwargs: None, - "grpc_upb_proto_reflection_library": lambda name, **kwargs: None, - "grpc_generate_one_off_targets": lambda: None, - "grpc_generate_one_off_internal_targets": lambda: None, - "grpc_package": lambda **kwargs: None, - "filegroup": lambda name, **kwargs: None, - "sh_library": lambda name, **kwargs: None, - }, - {}, - ) - parsing_path = None - -if args.whats_left: - print( - "{}/{} libraries are opted in".format( - num_cc_libraries - num_opted_out_cc_libraries, num_cc_libraries - ) - ) - - -def make_relative_path(dep, lib): - if lib is None: - return dep - lib_path = lib[: lib.rfind(":") + 1] - if dep.startswith(lib_path): - return dep[len(lib_path) :] - return dep - - -if args.whats_left: - print( - "{}/{} libraries are opted in".format( - num_cc_libraries - num_opted_out_cc_libraries, num_cc_libraries - ) - ) - - -# Keeps track of all possible sets of dependencies that could satify the -# problem. (models the list monad in Haskell!) -class Choices: - def __init__(self, library, substitutions): - self.library = library - self.to_add = [] - self.to_remove = [] - self.substitutions = substitutions - - def add_one_of(self, choices, trigger): - if not choices: - return - choices = sum( - [self.apply_substitutions(choice) for choice in choices], [] - ) - if args.explain and (args.why is None or args.why in choices): - print( - "{}: Adding one of {} for {}".format( - self.library, choices, trigger - ) - ) - self.to_add.append( - tuple( - make_relative_path(choice, self.library) for choice in choices - ) - ) - - def add(self, choice, trigger): - self.add_one_of([choice], trigger) - - def remove(self, remove): - for remove in self.apply_substitutions(remove): - self.to_remove.append(make_relative_path(remove, self.library)) - - def apply_substitutions(self, dep): - if dep in self.substitutions: - return self.substitutions[dep] - return [dep] - - def best(self, scorer): - choices = set() - choices.add(frozenset()) - - for add in sorted(set(self.to_add), key=lambda x: (len(x), x)): - new_choices = set() - for append_choice in add: - for choice in choices: - new_choices.add(choice.union([append_choice])) - choices = new_choices - for remove in sorted(set(self.to_remove)): - new_choices = set() - for choice in choices: - new_choices.add(choice.difference([remove])) - choices = new_choices - - best = None - - def final_scorer(x): - return (total_avoidness(x), scorer(x), total_score(x)) - - for choice in choices: - if best is None or final_scorer(choice) < final_scorer(best): - best = choice - return best - - -def make_library(library): - error = False - hdrs = sorted(consumes[library]) - # we need a little trickery here since grpc_base has channel.cc, which calls grpc_init - # which is in grpc, which is illegal but hard to change - # once EventEngine lands we can clean this up - deps = Choices( - library, - {"//:grpc_base": ["//:grpc", "//:grpc_unsecure"]} - if library.startswith("//test/") - else {}, - ) - external_deps = Choices(None, {}) - for hdr in hdrs: - if hdr in skip_headers[library]: - continue - - if hdr == "systemd/sd-daemon.h": - continue - - if hdr == "src/core/lib/profiling/stap_probes.h": - continue - - if hdr.startswith("src/libfuzzer/"): - continue - - if hdr == "grpc/grpc.h" and library.startswith("//test:"): - # not the root build including grpc.h ==> //:grpc - deps.add_one_of(["//:grpc", "//:grpc_unsecure"], hdr) - continue - - if hdr in INTERNAL_DEPS: - dep = INTERNAL_DEPS[hdr] - if isinstance(dep, list): - for d in dep: - deps.add(d, hdr) - else: - if not ("//" in dep): - dep = "//:" + dep - deps.add(dep, hdr) - continue - - if hdr in vendors: - deps.add_one_of(vendors[hdr], hdr) - continue - - if "include/" + hdr in vendors: - deps.add_one_of(vendors["include/" + hdr], hdr) - continue - - if "." not in hdr: - # assume a c++ system include - continue - - if hdr in EXTERNAL_DEPS: - if isinstance(EXTERNAL_DEPS[hdr], list): - for dep in EXTERNAL_DEPS[hdr]: - external_deps.add(dep, hdr) - else: - external_deps.add(EXTERNAL_DEPS[hdr], hdr) - continue - - if hdr.startswith("opencensus/"): - trail = hdr[len("opencensus/") :] - trail = trail[: trail.find("/")] - external_deps.add("opencensus-" + trail, hdr) - continue - - if hdr.startswith("envoy/"): - path, file = os.path.split(hdr) - file = file.split(".") - path = path.split("/") - dep = "_".join(path[:-1] + [file[1]]) - deps.add(dep, hdr) - continue - - if hdr.startswith("google/protobuf/") and not hdr.endswith(".upb.h"): - external_deps.add("protobuf_headers", hdr) - continue - - if "/" not in hdr: - # assume a system include - continue - - is_sys_include = False - for sys_path in [ - "sys", - "arpa", - "gperftools", - "netinet", - "linux", - "android", - "mach", - "net", - "CoreFoundation", - ]: - if hdr.startswith(sys_path + "/"): - is_sys_include = True - break - if is_sys_include: - # assume a system include - continue - - print( - "# ERROR: can't categorize header: %s used by %s" % (hdr, library) - ) - error = True - - deps.remove(library) - - deps = sorted( - deps.best(lambda x: SCORERS[args.score](x, original_deps[library])) - ) - external_deps = sorted( - external_deps.best( - lambda x: SCORERS[args.score](x, original_external_deps[library]) - ) - ) - - return (library, error, deps, external_deps) - - -def main() -> None: - update_libraries = [] - for library in sorted(consumes.keys()): - if library in no_update: - continue - if args.targets and library not in args.targets: - continue - update_libraries.append(library) - with multiprocessing.Pool(processes=multiprocessing.cpu_count()) as p: - updated_libraries = p.map(make_library, update_libraries, 1) - - error = False - for library, lib_error, deps, external_deps in updated_libraries: - if lib_error: - error = True - continue - buildozer_set_list("external_deps", external_deps, library, via="deps") - buildozer_set_list("deps", deps, library) - - run_buildozer.run_buildozer(buildozer_commands) - - if error: - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/tools/distrib/iwyu.sh b/tools/distrib/iwyu.sh deleted file mode 100755 index 9a00fde2b4d..00000000000 --- a/tools/distrib/iwyu.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -# Copyright 2021 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -echo "NOTE: to automagically apply fixes, invoke with --fix" - -set -ex - -# change to root directory -cd $(dirname $0)/../.. -REPO_ROOT=$(pwd) - -# grep targets with manual tag, which is not included in a result of bazel build using ... -# let's get a list of them using query command and pass it to gen_compilation_database.py -export MANUAL_TARGETS=$(bazel query 'attr("tags", "manual", tests(//test/cpp/...))' | grep -v _on_ios) - -# generate a clang compilation database for all C/C++ sources in the repo. -tools/distrib/gen_compilation_database.py \ - --include_headers \ - --ignore_system_headers \ - --dedup_targets \ - "//:*" \ - "//src/core/..." \ - "//src/cpp/ext/csm/..." \ - "//src/cpp/ext/gcp/..." \ - "//src/cpp/ext/otel/..." \ - "//src/compiler/..." \ - "//test/core/..." \ - "//test/cpp/..." \ - "//fuzztest/..." \ - $MANUAL_TARGETS - -# run iwyu against the checked out codebase -# when modifying the checked-out files, the current user will be impersonated -# so that the updated files don't end up being owned by "root". -if [ "$IWYU_SKIP_DOCKER" == "" ] -then - # build iwyu docker image - docker build -t grpc_iwyu tools/dockerfile/grpc_iwyu - - docker run \ - -e TEST="$TEST" \ - -e CHANGED_FILES="$CHANGED_FILES" \ - -e IWYU_ROOT="/local-code" \ - --rm=true \ - -v "${REPO_ROOT}":/local-code \ - -v "${HOME/.cache/bazel}":"${HOME/.cache/bazel}" \ - --user "$(id -u):$(id -g)" \ - -t grpc_iwyu /iwyu.sh "$@" -else - IWYU_ROOT="${REPO_ROOT}" tools/dockerfile/grpc_iwyu/iwyu.sh -fi diff --git a/tools/distrib/iwyu_mappings.imp b/tools/distrib/iwyu_mappings.imp deleted file mode 100644 index 3f993a9aee5..00000000000 --- a/tools/distrib/iwyu_mappings.imp +++ /dev/null @@ -1,9 +0,0 @@ -[ - { include: ["", "private", "", "public"] }, - { include: ["", "public", "\"src/core/lib/iomgr/sockaddr.h\"", "public"]}, - { include: ["", "public", "\"src/core/lib/iomgr/sockaddr.h\"", "public"]}, - { include: ["", "private", "", "public"] }, - { include: ["", "private", "", "public"] }, - # workaround: https://github.com/include-what-you-use/include-what-you-use/issues/908 - { symbol: ["std::max", "private", "", "public" ] }, -] diff --git a/tools/dockerfile/grpc_iwyu/Dockerfile b/tools/dockerfile/grpc_iwyu/Dockerfile deleted file mode 100644 index 01b9a2d3a4a..00000000000 --- a/tools/dockerfile/grpc_iwyu/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2021 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Docker file for running IWYU. -# Updated: 2022-11-03 - -FROM silkeh/clang:16-bullseye - -# Install prerequisites for the iwyu script -RUN apt-get update && apt-get install -y python3 jq git cmake python zlib1g-dev libtinfo-dev libclang-16-dev && apt-get clean -ADD iwyu.sh / - -# When running locally, we'll be impersonating the current user, so we need -# to make the script runnable by everyone. -RUN chmod a+rx /iwyu.sh - -CMD ["echo 'Run with tools/distrib/iwyu.sh'"] diff --git a/tools/dockerfile/grpc_iwyu/iwyu.sh b/tools/dockerfile/grpc_iwyu/iwyu.sh deleted file mode 100755 index 96f5ba74592..00000000000 --- a/tools/dockerfile/grpc_iwyu/iwyu.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/bin/sh -# Copyright 2017 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -x - -cd ${IWYU_ROOT} - -export PATH=${PATH}:${IWYU_ROOT}/iwyu_build/bin - -# number of CPUs available -CPU_COUNT=`nproc` - -rm -rf iwyu || true -git clone https://github.com/include-what-you-use/include-what-you-use.git iwyu - -############################################################################### -# -# BEWARE! BEWARE! BEWARE! BEWARE! BEWARE! BEWARE! BEWARE! BEWARE! -# -# Changing the version of iwyu can bring along subtle changes. -# You *must* test the new version of iwyu: -# 1. run it on the entire codebase before submitting -# 2. UPLOAD A CHANGE THAT SHOULD BE BROKEN AFTER SUBMISSION OF THIS CHANGE -# ensure that the broken change is caught by the new version of iwyu -# -# BEWARE! BEWARE! BEWARE! BEWARE! BEWARE! BEWARE! BEWARE! BEWARE! -# -############################################################################### - -# latest commit on the clang_16 branch -cd ${IWYU_ROOT}/iwyu -git checkout 7301b1fc88e5e16d8df73aecea55037d9c0a371b -if [ $? -ne 0 ]; then - echo "Failed to checkout iwyu commit" - exit 1 -fi -mkdir -p ${IWYU_ROOT}/iwyu_build -cd ${IWYU_ROOT}/iwyu_build -cmake -G "Unix Makefiles" -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_ROOT_DIR=/usr/lib/llvm-16 ${IWYU_ROOT}/iwyu -if [ $? -ne 0 ]; then - echo "Failed to cmake iwyu" - exit 1 -fi -make -j $CPU_COUNT -if [ $? -ne 0 ]; then - echo "Failed to make iwyu" - exit 1 -fi -cd ${IWYU_ROOT} - -cat compile_commands.json \ - | sed "s/ -DNDEBUG//g" \ - | sed "s/ -std=c\\+\\+14/ -std=c++17/g" \ - | sed "s,\"file\": \",\"file\": \"${IWYU_ROOT}/,g" \ - > compile_commands_for_iwyu.json - -export ENABLED_MODULES=' - src/core/ext - src/core/lib - src/cpp - src/python/grpcio_observability - test/core - fuzztest -' - -export DISABLED_MODULES=' - src/core/lib/gpr - src/core/lib/iomgr - src/core/ext/transport/binder - test/core/alts - test/core/iomgr - test/core/security - test/core/tsi - test/core/transport/binder -' - -export INCLUSION_REGEX=`echo $ENABLED_MODULES | sed 's/ /|/g' | sed 's,\\(.*\\),^(\\1)/,g'` -export EXCLUSION_REGEX=`echo $DISABLED_MODULES | sed 's/ /|/g' | sed 's,\\(.*\\),^(\\1)/,g'` - -# figure out which files to include -cat compile_commands.json | jq -r '.[].file' \ - | grep -E $INCLUSION_REGEX \ - | grep -v -E "/upb-gen/|/upbdefs-gen/" \ - | grep -v -E $EXCLUSION_REGEX \ - | grep -v src/core/lib/security/credentials/tls/grpc_tls_credentials_options.h \ - | grep -v test/core/end2end/end2end_tests.cc \ - | sort \ - > iwyu_files0.txt - -cat iwyu_files0.txt \ - | xargs -d '\n' ls -1df 2> /dev/null \ - > iwyu_files.txt \ - || true - -echo '#!/bin/sh -${IWYU_ROOT}/iwyu/iwyu_tool.py -p compile_commands_for_iwyu.json $1 \ - -- -Xiwyu --no_fwd_decls \ - -Xiwyu --update_comments \ - -Xiwyu --mapping_file=${IWYU_ROOT}/tools/distrib/iwyu_mappings.imp \ - | grep -v -E "port_platform.h" \ - | grep -v -E "repeated_ptr_field.h" \ - | grep -v -E "repeated_field.h" \ - | grep -v -E "^(- )?namespace " \ - > iwyu/iwyu.`echo $1 | sha1sum`.out -' > iwyu/run_iwyu_on.sh -chmod +x iwyu/run_iwyu_on.sh - -# run iwyu, filtering out changes to port_platform.h -xargs -n 1 -P $CPU_COUNT -a iwyu_files.txt ${IWYU_ROOT}/iwyu/run_iwyu_on.sh - -cat iwyu/iwyu.*.out > iwyu.out - -# apply the suggested changes -${IWYU_ROOT}/iwyu/fix_includes.py \ - --nocomments \ - --nosafe_headers \ - --ignore_re='^(include/.*|src/core/lib/security/credentials/tls/grpc_tls_credentials_options\.h|external/.*)' \ - < iwyu.out \ - | grep 'IWYU edited 0 files on your behalf' - -if [ $? -ne 0 ] -then - echo "Iwyu edited some files. Here is the diff of files edited by iwyu:" - git --no-pager diff - # Exit with a non zero error code to ensure sanity checks fail accordingly. - exit 1 -fi diff --git a/tools/internal_ci/linux/grpc_iwyu.cfg b/tools/internal_ci/linux/grpc_iwyu.cfg deleted file mode 100644 index 33fbbd68bad..00000000000 --- a/tools/internal_ci/linux/grpc_iwyu.cfg +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2017 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Config file for the internal CI (in protobuf text format) - -# Location of the continuous shell script in repository. -build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" -timeout_mins: 60 -action { - define_artifacts { - regex: "**/*sponge_log.*" - regex: "github/grpc/reports/**" - } -} - -env_vars { - key: "RUN_TESTS_FLAGS" - value: "-f basictests linux iwyu --inner_jobs 16 -j 1 --internal_ci --bq_result_table aggregate_results" -} diff --git a/tools/internal_ci/linux/pull_request/grpc_iwyu.cfg b/tools/internal_ci/linux/pull_request/grpc_iwyu.cfg deleted file mode 100644 index 4d7fcecc9a6..00000000000 --- a/tools/internal_ci/linux/pull_request/grpc_iwyu.cfg +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2017 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Config file for the internal CI (in protobuf text format) - -# Location of the continuous shell script in repository. -build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" -timeout_mins: 60 -action { - define_artifacts { - regex: "**/*sponge_log.*" - regex: "github/grpc/reports/**" - } -} - -env_vars { - key: "RUN_TESTS_FLAGS" - value: "-f basictests linux iwyu --inner_jobs 16 -j 1 --internal_ci" -} diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 64d2cfcbeb0..88a9a648bbd 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -1304,7 +1304,6 @@ _LANGUAGES = { "objc": ObjCLanguage(), "sanity": Sanity("sanity_tests.yaml"), "clang-tidy": Sanity("clang_tidy_tests.yaml"), - "iwyu": Sanity("iwyu_tests.yaml"), } _MSBUILD_CONFIG = { diff --git a/tools/run_tests/sanity/iwyu_tests.yaml b/tools/run_tests/sanity/iwyu_tests.yaml deleted file mode 100644 index 2c9c7836d7f..00000000000 --- a/tools/run_tests/sanity/iwyu_tests.yaml +++ /dev/null @@ -1,2 +0,0 @@ -- script: tools/distrib/iwyu.sh - cpu_cost: 1000