mirror of https://github.com/grpc/grpc.git
Craig Tiller
1 year ago
committed by
Copybara-Service
parent
492dfba681
commit
57d8462aee
11 changed files with 0 additions and 1018 deletions
@ -1,30 +0,0 @@ |
||||
%YAML 1.2 |
||||
--- | |
||||
# Copyright 2021 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# Docker file for running IWYU. |
||||
# Updated: 2022-11-03 |
||||
|
||||
FROM silkeh/clang:16-bullseye |
||||
|
||||
# Install prerequisites for the iwyu script |
||||
RUN apt-get update && apt-get install -y python3 jq git cmake python zlib1g-dev libtinfo-dev libclang-16-dev && apt-get clean |
||||
ADD iwyu.sh / |
||||
|
||||
# When running locally, we'll be impersonating the current user, so we need |
||||
# to make the script runnable by everyone. |
||||
RUN chmod a+rx /iwyu.sh |
||||
|
||||
CMD ["echo 'Run with tools/distrib/iwyu.sh'"] |
||||
@ -1,684 +0,0 @@ |
||||
#!/usr/bin/env python3 |
||||
|
||||
# Copyright 2022 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
import argparse |
||||
import collections |
||||
from doctest import SKIP |
||||
import multiprocessing |
||||
import os |
||||
import re |
||||
import sys |
||||
|
||||
import run_buildozer |
||||
|
||||
# find our home |
||||
ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "../..")) |
||||
os.chdir(ROOT) |
||||
|
||||
vendors = collections.defaultdict(list) |
||||
scores = collections.defaultdict(int) |
||||
avoidness = collections.defaultdict(int) |
||||
consumes = {} |
||||
no_update = set() |
||||
buildozer_commands = [] |
||||
original_deps = {} |
||||
original_external_deps = {} |
||||
skip_headers = collections.defaultdict(set) |
||||
|
||||
# TODO(ctiller): ideally we wouldn't hardcode a bunch of paths here. |
||||
# We can likely parse out BUILD files from dependencies to generate this index. |
||||
EXTERNAL_DEPS = { |
||||
"absl/algorithm/container.h": "absl/algorithm:container", |
||||
"absl/base/attributes.h": "absl/base:core_headers", |
||||
"absl/base/call_once.h": "absl/base", |
||||
# TODO(ctiller) remove this |
||||
"absl/base/internal/endian.h": "absl/base:endian", |
||||
"absl/base/thread_annotations.h": "absl/base:core_headers", |
||||
"absl/container/flat_hash_map.h": "absl/container:flat_hash_map", |
||||
"absl/container/flat_hash_set.h": "absl/container:flat_hash_set", |
||||
"absl/container/inlined_vector.h": "absl/container:inlined_vector", |
||||
"absl/cleanup/cleanup.h": "absl/cleanup", |
||||
"absl/debugging/failure_signal_handler.h": ( |
||||
"absl/debugging:failure_signal_handler" |
||||
), |
||||
"absl/debugging/stacktrace.h": "absl/debugging:stacktrace", |
||||
"absl/debugging/symbolize.h": "absl/debugging:symbolize", |
||||
"absl/flags/flag.h": "absl/flags:flag", |
||||
"absl/flags/marshalling.h": "absl/flags:marshalling", |
||||
"absl/flags/parse.h": "absl/flags:parse", |
||||
"absl/functional/any_invocable.h": "absl/functional:any_invocable", |
||||
"absl/functional/bind_front.h": "absl/functional:bind_front", |
||||
"absl/functional/function_ref.h": "absl/functional:function_ref", |
||||
"absl/hash/hash.h": "absl/hash", |
||||
"absl/memory/memory.h": "absl/memory", |
||||
"absl/meta/type_traits.h": "absl/meta:type_traits", |
||||
"absl/numeric/int128.h": "absl/numeric:int128", |
||||
"absl/random/random.h": "absl/random", |
||||
"absl/random/bit_gen_ref.h": "absl/random:bit_gen_ref", |
||||
"absl/random/mocking_bit_gen.h": "absl/random:mocking_bit_gen", |
||||
"absl/random/distributions.h": "absl/random:distributions", |
||||
"absl/random/uniform_int_distribution.h": "absl/random:distributions", |
||||
"absl/status/status.h": "absl/status", |
||||
"absl/status/statusor.h": "absl/status:statusor", |
||||
"absl/strings/ascii.h": "absl/strings", |
||||
"absl/strings/cord.h": "absl/strings:cord", |
||||
"absl/strings/escaping.h": "absl/strings", |
||||
"absl/strings/match.h": "absl/strings", |
||||
"absl/strings/numbers.h": "absl/strings", |
||||
"absl/strings/str_cat.h": "absl/strings", |
||||
"absl/strings/str_format.h": "absl/strings:str_format", |
||||
"absl/strings/str_join.h": "absl/strings", |
||||
"absl/strings/str_replace.h": "absl/strings", |
||||
"absl/strings/str_split.h": "absl/strings", |
||||
"absl/strings/string_view.h": "absl/strings", |
||||
"absl/strings/strip.h": "absl/strings", |
||||
"absl/strings/substitute.h": "absl/strings", |
||||
"absl/synchronization/mutex.h": "absl/synchronization", |
||||
"absl/synchronization/notification.h": "absl/synchronization", |
||||
"absl/time/clock.h": "absl/time", |
||||
"absl/time/time.h": "absl/time", |
||||
"absl/types/optional.h": "absl/types:optional", |
||||
"absl/types/span.h": "absl/types:span", |
||||
"absl/types/variant.h": "absl/types:variant", |
||||
"absl/utility/utility.h": "absl/utility", |
||||
"address_sorting/address_sorting.h": "address_sorting", |
||||
"google/cloud/opentelemetry/resource_detector.h": "google_cloud_cpp:opentelemetry", |
||||
"opentelemetry/common/attribute_value.h": "otel/api", |
||||
"opentelemetry/common/key_value_iterable.h": "otel/api", |
||||
"opentelemetry/nostd/function_ref.h": "otel/api", |
||||
"opentelemetry/nostd/string_view.h": "otel/api", |
||||
"opentelemetry/context/context.h": "otel/api", |
||||
"opentelemetry/metrics/meter.h": "otel/api", |
||||
"opentelemetry/metrics/meter_provider.h": "otel/api", |
||||
"opentelemetry/metrics/provider.h": "otel/api", |
||||
"opentelemetry/metrics/sync_instruments.h": "otel/api", |
||||
"opentelemetry/nostd/shared_ptr.h": "otel/api", |
||||
"opentelemetry/nostd/unique_ptr.h": "otel/api", |
||||
"opentelemetry/sdk/metrics/meter_provider.h": "otel/sdk/src/metrics", |
||||
"opentelemetry/sdk/common/attribute_utils.h": "otel/sdk:headers", |
||||
"opentelemetry/sdk/resource/resource.h": "otel/sdk:headers", |
||||
"opentelemetry/sdk/resource/resource_detector.h": "otel/sdk:headers", |
||||
"opentelemetry/sdk/resource/semantic_conventions.h": "otel/sdk:headers", |
||||
"ares.h": "cares", |
||||
"fuzztest/fuzztest.h": ["fuzztest", "fuzztest_main"], |
||||
"google/api/monitored_resource.pb.h": ( |
||||
"google/api:monitored_resource_cc_proto" |
||||
), |
||||
"google/devtools/cloudtrace/v2/tracing.grpc.pb.h": ( |
||||
"googleapis_trace_grpc_service" |
||||
), |
||||
"google/logging/v2/logging.grpc.pb.h": "googleapis_logging_grpc_service", |
||||
"google/logging/v2/logging.pb.h": "googleapis_logging_cc_proto", |
||||
"google/logging/v2/log_entry.pb.h": "googleapis_logging_cc_proto", |
||||
"google/monitoring/v3/metric_service.grpc.pb.h": ( |
||||
"googleapis_monitoring_grpc_service" |
||||
), |
||||
"gmock/gmock.h": "gtest", |
||||
"gtest/gtest.h": "gtest", |
||||
"opencensus/exporters/stats/stackdriver/stackdriver_exporter.h": ( |
||||
"opencensus-stats-stackdriver_exporter" |
||||
), |
||||
"opencensus/exporters/trace/stackdriver/stackdriver_exporter.h": ( |
||||
"opencensus-trace-stackdriver_exporter" |
||||
), |
||||
"opencensus/trace/context_util.h": "opencensus-trace-context_util", |
||||
"opencensus/trace/propagation/grpc_trace_bin.h": ( |
||||
"opencensus-trace-propagation" |
||||
), |
||||
"opencensus/tags/context_util.h": "opencensus-tags-context_util", |
||||
"opencensus/trace/span_context.h": "opencensus-trace-span_context", |
||||
"openssl/base.h": "libssl", |
||||
"openssl/bio.h": "libssl", |
||||
"openssl/bn.h": "libcrypto", |
||||
"openssl/buffer.h": "libcrypto", |
||||
"openssl/crypto.h": "libcrypto", |
||||
"openssl/digest.h": "libssl", |
||||
"openssl/engine.h": "libcrypto", |
||||
"openssl/err.h": "libcrypto", |
||||
"openssl/evp.h": "libcrypto", |
||||
"openssl/hmac.h": "libcrypto", |
||||
"openssl/mem.h": "libcrypto", |
||||
"openssl/param_build.h": "libcrypto", |
||||
"openssl/pem.h": "libcrypto", |
||||
"openssl/rsa.h": "libcrypto", |
||||
"openssl/sha.h": "libcrypto", |
||||
"openssl/ssl.h": "libssl", |
||||
"openssl/tls1.h": "libssl", |
||||
"openssl/x509.h": "libcrypto", |
||||
"openssl/x509v3.h": "libcrypto", |
||||
"re2/re2.h": "re2", |
||||
"upb/base/string_view.h": "upb_base_lib", |
||||
"upb/collections/map.h": "upb_collections_lib", |
||||
"upb/reflection/def.h": "upb_reflection", |
||||
"upb/json/encode.h": "upb_json_lib", |
||||
"upb/mem/arena.h": "upb_mem_lib", |
||||
"upb/text/encode.h": "upb_textformat_lib", |
||||
"upb/reflection/def.hpp": "upb_reflection", |
||||
"upb/upb.h": "upb_amalgamation_lib", |
||||
"upb/upb.hpp": "upb_lib", |
||||
"xxhash.h": "xxhash", |
||||
"zlib.h": "madler_zlib", |
||||
} |
||||
|
||||
INTERNAL_DEPS = { |
||||
"test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.h": ( |
||||
"//test/core/event_engine/fuzzing_event_engine" |
||||
), |
||||
"test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.pb.h": "//test/core/event_engine/fuzzing_event_engine:fuzzing_event_engine_proto", |
||||
"test/core/experiments/test_experiments.h": "//test/core/experiments:test_experiments_lib", |
||||
"google/api/expr/v1alpha1/syntax.upb.h": "google_api_expr_v1alpha1_syntax_upb", |
||||
"google/rpc/status.upb.h": "google_rpc_status_upb", |
||||
"google/protobuf/any.upb.h": "protobuf_any_upb", |
||||
"google/protobuf/duration.upb.h": "protobuf_duration_upb", |
||||
"google/protobuf/struct.upb.h": "protobuf_struct_upb", |
||||
"google/protobuf/timestamp.upb.h": "protobuf_timestamp_upb", |
||||
"google/protobuf/wrappers.upb.h": "protobuf_wrappers_upb", |
||||
"grpc/status.h": "grpc_public_hdrs", |
||||
"src/proto/grpc/channelz/channelz.grpc.pb.h": ( |
||||
"//src/proto/grpc/channelz:channelz_proto" |
||||
), |
||||
"src/proto/grpc/core/stats.pb.h": "//src/proto/grpc/core:stats_proto", |
||||
"src/proto/grpc/health/v1/health.upb.h": "grpc_health_upb", |
||||
"src/proto/grpc/lb/v1/load_reporter.grpc.pb.h": ( |
||||
"//src/proto/grpc/lb/v1:load_reporter_proto" |
||||
), |
||||
"src/proto/grpc/lb/v1/load_balancer.upb.h": "grpc_lb_upb", |
||||
"src/proto/grpc/reflection/v1alpha/reflection.grpc.pb.h": ( |
||||
"//src/proto/grpc/reflection/v1alpha:reflection_proto" |
||||
), |
||||
"src/proto/grpc/gcp/transport_security_common.upb.h": "alts_upb", |
||||
"src/proto/grpc/gcp/handshaker.upb.h": "alts_upb", |
||||
"src/proto/grpc/gcp/altscontext.upb.h": "alts_upb", |
||||
"src/proto/grpc/lookup/v1/rls.upb.h": "rls_upb", |
||||
"src/proto/grpc/lookup/v1/rls_config.upb.h": "rls_config_upb", |
||||
"src/proto/grpc/lookup/v1/rls_config.upbdefs.h": "rls_config_upbdefs", |
||||
"src/proto/grpc/testing/xds/v3/csds.grpc.pb.h": ( |
||||
"//src/proto/grpc/testing/xds/v3:csds_proto" |
||||
), |
||||
"xds/data/orca/v3/orca_load_report.upb.h": "xds_orca_upb", |
||||
"xds/service/orca/v3/orca.upb.h": "xds_orca_service_upb", |
||||
"xds/type/v3/typed_struct.upb.h": "xds_type_upb", |
||||
} |
||||
|
||||
|
||||
class FakeSelects: |
||||
def config_setting_group(self, **kwargs): |
||||
pass |
||||
|
||||
|
||||
num_cc_libraries = 0 |
||||
num_opted_out_cc_libraries = 0 |
||||
parsing_path = None |
||||
|
||||
|
||||
# Convert the source or header target to a relative path. |
||||
def _get_filename(name, parsing_path): |
||||
filename = "%s%s" % ( |
||||
( |
||||
parsing_path + "/" |
||||
if (parsing_path and not name.startswith("//")) |
||||
else "" |
||||
), |
||||
name, |
||||
) |
||||
filename = filename.replace("//:", "") |
||||
filename = filename.replace("//src/core:", "src/core/") |
||||
filename = filename.replace( |
||||
"//src/cpp/ext/filters/census:", "src/cpp/ext/filters/census/" |
||||
) |
||||
return filename |
||||
|
||||
|
||||
def grpc_cc_library( |
||||
name, |
||||
hdrs=[], |
||||
public_hdrs=[], |
||||
srcs=[], |
||||
select_deps=None, |
||||
tags=[], |
||||
deps=[], |
||||
external_deps=[], |
||||
proto=None, |
||||
**kwargs, |
||||
): |
||||
global args |
||||
global num_cc_libraries |
||||
global num_opted_out_cc_libraries |
||||
global parsing_path |
||||
assert parsing_path is not None |
||||
name = "//%s:%s" % (parsing_path, name) |
||||
num_cc_libraries += 1 |
||||
if select_deps or "nofixdeps" in tags: |
||||
if args.whats_left and not select_deps and "nofixdeps" not in tags: |
||||
num_opted_out_cc_libraries += 1 |
||||
print("Not opted in: {}".format(name)) |
||||
no_update.add(name) |
||||
scores[name] = len(public_hdrs + hdrs) |
||||
# avoid_dep is the internal way of saying prefer something else |
||||
# we add grpc_avoid_dep to allow internal grpc-only stuff to avoid each |
||||
# other, whilst not biasing dependent projects |
||||
if "avoid_dep" in tags or "grpc_avoid_dep" in tags: |
||||
avoidness[name] += 10 |
||||
if proto: |
||||
proto_hdr = "%s%s" % ( |
||||
(parsing_path + "/" if parsing_path else ""), |
||||
proto.replace(".proto", ".pb.h"), |
||||
) |
||||
skip_headers[name].add(proto_hdr) |
||||
|
||||
for hdr in hdrs + public_hdrs: |
||||
vendors[_get_filename(hdr, parsing_path)].append(name) |
||||
inc = set() |
||||
original_deps[name] = frozenset(deps) |
||||
original_external_deps[name] = frozenset(external_deps) |
||||
for src in hdrs + public_hdrs + srcs: |
||||
for line in open(_get_filename(src, parsing_path)): |
||||
m = re.search(r"^#include <(.*)>", line) |
||||
if m: |
||||
inc.add(m.group(1)) |
||||
m = re.search(r'^#include "(.*)"', line) |
||||
if m: |
||||
inc.add(m.group(1)) |
||||
consumes[name] = list(inc) |
||||
|
||||
|
||||
def grpc_proto_library(name, srcs, **kwargs): |
||||
global parsing_path |
||||
assert parsing_path is not None |
||||
name = "//%s:%s" % (parsing_path, name) |
||||
for src in srcs: |
||||
proto_hdr = src.replace(".proto", ".pb.h") |
||||
vendors[_get_filename(proto_hdr, parsing_path)].append(name) |
||||
|
||||
|
||||
def buildozer(cmd, target): |
||||
buildozer_commands.append("%s|%s" % (cmd, target)) |
||||
|
||||
|
||||
def buildozer_set_list(name, values, target, via=""): |
||||
if not values: |
||||
buildozer("remove %s" % name, target) |
||||
return |
||||
adjust = via if via else name |
||||
buildozer( |
||||
"set %s %s" % (adjust, " ".join('"%s"' % s for s in values)), target |
||||
) |
||||
if via: |
||||
buildozer("remove %s" % name, target) |
||||
buildozer("rename %s %s" % (via, name), target) |
||||
|
||||
|
||||
def score_edit_distance(proposed, existing): |
||||
"""Score a proposed change primarily by edit distance""" |
||||
sum = 0 |
||||
for p in proposed: |
||||
if p not in existing: |
||||
sum += 1 |
||||
for e in existing: |
||||
if e not in proposed: |
||||
sum += 1 |
||||
return sum |
||||
|
||||
|
||||
def total_score(proposal): |
||||
return sum(scores[dep] for dep in proposal) |
||||
|
||||
|
||||
def total_avoidness(proposal): |
||||
return sum(avoidness[dep] for dep in proposal) |
||||
|
||||
|
||||
def score_list_size(proposed, existing): |
||||
"""Score a proposed change primarily by number of dependencies""" |
||||
return len(proposed) |
||||
|
||||
|
||||
def score_best(proposed, existing): |
||||
"""Score a proposed change primarily by dependency score""" |
||||
return 0 |
||||
|
||||
|
||||
SCORERS = { |
||||
"edit_distance": score_edit_distance, |
||||
"list_size": score_list_size, |
||||
"best": score_best, |
||||
} |
||||
|
||||
parser = argparse.ArgumentParser(description="Fix build dependencies") |
||||
parser.add_argument( |
||||
"targets", nargs="*", default=[], help="targets to fix (empty => all)" |
||||
) |
||||
parser.add_argument( |
||||
"--score", |
||||
type=str, |
||||
default="edit_distance", |
||||
help="scoring function to use: one of " + ", ".join(SCORERS.keys()), |
||||
) |
||||
parser.add_argument( |
||||
"--whats_left", |
||||
action="store_true", |
||||
default=False, |
||||
help="show what is left to opt in", |
||||
) |
||||
parser.add_argument( |
||||
"--explain", |
||||
action="store_true", |
||||
default=False, |
||||
help="try to explain some decisions", |
||||
) |
||||
parser.add_argument( |
||||
"--why", |
||||
type=str, |
||||
default=None, |
||||
help="with --explain, target why a given dependency is needed", |
||||
) |
||||
args = parser.parse_args() |
||||
|
||||
for dirname in [ |
||||
"", |
||||
"src/core", |
||||
"src/cpp/ext/gcp", |
||||
"src/cpp/ext/csm", |
||||
"src/cpp/ext/otel", |
||||
"test/core/backoff", |
||||
"test/core/experiments", |
||||
"test/core/uri", |
||||
"test/core/util", |
||||
"test/core/end2end", |
||||
"test/core/event_engine", |
||||
"test/core/filters", |
||||
"test/core/promise", |
||||
"test/core/resource_quota", |
||||
"test/core/transport/chaotic_good", |
||||
"fuzztest", |
||||
"fuzztest/core/channel", |
||||
"fuzztest/core/transport/chttp2", |
||||
]: |
||||
parsing_path = dirname |
||||
exec( |
||||
open("%sBUILD" % (dirname + "/" if dirname else ""), "r").read(), |
||||
{ |
||||
"load": lambda filename, *args: None, |
||||
"licenses": lambda licenses: None, |
||||
"package": lambda **kwargs: None, |
||||
"exports_files": lambda files, visibility=None: None, |
||||
"bool_flag": lambda **kwargs: None, |
||||
"config_setting": lambda **kwargs: None, |
||||
"selects": FakeSelects(), |
||||
"python_config_settings": lambda **kwargs: None, |
||||
"grpc_cc_binary": grpc_cc_library, |
||||
"grpc_cc_library": grpc_cc_library, |
||||
"grpc_cc_test": grpc_cc_library, |
||||
"grpc_core_end2end_test": lambda **kwargs: None, |
||||
"grpc_fuzzer": grpc_cc_library, |
||||
"grpc_fuzz_test": grpc_cc_library, |
||||
"grpc_proto_fuzzer": grpc_cc_library, |
||||
"grpc_proto_library": grpc_proto_library, |
||||
"select": lambda d: d["//conditions:default"], |
||||
"glob": lambda files: None, |
||||
"grpc_end2end_tests": lambda: None, |
||||
"grpc_upb_proto_library": lambda name, **kwargs: None, |
||||
"grpc_upb_proto_reflection_library": lambda name, **kwargs: None, |
||||
"grpc_generate_one_off_targets": lambda: None, |
||||
"grpc_generate_one_off_internal_targets": lambda: None, |
||||
"grpc_package": lambda **kwargs: None, |
||||
"filegroup": lambda name, **kwargs: None, |
||||
"sh_library": lambda name, **kwargs: None, |
||||
}, |
||||
{}, |
||||
) |
||||
parsing_path = None |
||||
|
||||
if args.whats_left: |
||||
print( |
||||
"{}/{} libraries are opted in".format( |
||||
num_cc_libraries - num_opted_out_cc_libraries, num_cc_libraries |
||||
) |
||||
) |
||||
|
||||
|
||||
def make_relative_path(dep, lib): |
||||
if lib is None: |
||||
return dep |
||||
lib_path = lib[: lib.rfind(":") + 1] |
||||
if dep.startswith(lib_path): |
||||
return dep[len(lib_path) :] |
||||
return dep |
||||
|
||||
|
||||
if args.whats_left: |
||||
print( |
||||
"{}/{} libraries are opted in".format( |
||||
num_cc_libraries - num_opted_out_cc_libraries, num_cc_libraries |
||||
) |
||||
) |
||||
|
||||
|
||||
# Keeps track of all possible sets of dependencies that could satify the |
||||
# problem. (models the list monad in Haskell!) |
||||
class Choices: |
||||
def __init__(self, library, substitutions): |
||||
self.library = library |
||||
self.to_add = [] |
||||
self.to_remove = [] |
||||
self.substitutions = substitutions |
||||
|
||||
def add_one_of(self, choices, trigger): |
||||
if not choices: |
||||
return |
||||
choices = sum( |
||||
[self.apply_substitutions(choice) for choice in choices], [] |
||||
) |
||||
if args.explain and (args.why is None or args.why in choices): |
||||
print( |
||||
"{}: Adding one of {} for {}".format( |
||||
self.library, choices, trigger |
||||
) |
||||
) |
||||
self.to_add.append( |
||||
tuple( |
||||
make_relative_path(choice, self.library) for choice in choices |
||||
) |
||||
) |
||||
|
||||
def add(self, choice, trigger): |
||||
self.add_one_of([choice], trigger) |
||||
|
||||
def remove(self, remove): |
||||
for remove in self.apply_substitutions(remove): |
||||
self.to_remove.append(make_relative_path(remove, self.library)) |
||||
|
||||
def apply_substitutions(self, dep): |
||||
if dep in self.substitutions: |
||||
return self.substitutions[dep] |
||||
return [dep] |
||||
|
||||
def best(self, scorer): |
||||
choices = set() |
||||
choices.add(frozenset()) |
||||
|
||||
for add in sorted(set(self.to_add), key=lambda x: (len(x), x)): |
||||
new_choices = set() |
||||
for append_choice in add: |
||||
for choice in choices: |
||||
new_choices.add(choice.union([append_choice])) |
||||
choices = new_choices |
||||
for remove in sorted(set(self.to_remove)): |
||||
new_choices = set() |
||||
for choice in choices: |
||||
new_choices.add(choice.difference([remove])) |
||||
choices = new_choices |
||||
|
||||
best = None |
||||
|
||||
def final_scorer(x): |
||||
return (total_avoidness(x), scorer(x), total_score(x)) |
||||
|
||||
for choice in choices: |
||||
if best is None or final_scorer(choice) < final_scorer(best): |
||||
best = choice |
||||
return best |
||||
|
||||
|
||||
def make_library(library): |
||||
error = False |
||||
hdrs = sorted(consumes[library]) |
||||
# we need a little trickery here since grpc_base has channel.cc, which calls grpc_init |
||||
# which is in grpc, which is illegal but hard to change |
||||
# once EventEngine lands we can clean this up |
||||
deps = Choices( |
||||
library, |
||||
{"//:grpc_base": ["//:grpc", "//:grpc_unsecure"]} |
||||
if library.startswith("//test/") |
||||
else {}, |
||||
) |
||||
external_deps = Choices(None, {}) |
||||
for hdr in hdrs: |
||||
if hdr in skip_headers[library]: |
||||
continue |
||||
|
||||
if hdr == "systemd/sd-daemon.h": |
||||
continue |
||||
|
||||
if hdr == "src/core/lib/profiling/stap_probes.h": |
||||
continue |
||||
|
||||
if hdr.startswith("src/libfuzzer/"): |
||||
continue |
||||
|
||||
if hdr == "grpc/grpc.h" and library.startswith("//test:"): |
||||
# not the root build including grpc.h ==> //:grpc |
||||
deps.add_one_of(["//:grpc", "//:grpc_unsecure"], hdr) |
||||
continue |
||||
|
||||
if hdr in INTERNAL_DEPS: |
||||
dep = INTERNAL_DEPS[hdr] |
||||
if isinstance(dep, list): |
||||
for d in dep: |
||||
deps.add(d, hdr) |
||||
else: |
||||
if not ("//" in dep): |
||||
dep = "//:" + dep |
||||
deps.add(dep, hdr) |
||||
continue |
||||
|
||||
if hdr in vendors: |
||||
deps.add_one_of(vendors[hdr], hdr) |
||||
continue |
||||
|
||||
if "include/" + hdr in vendors: |
||||
deps.add_one_of(vendors["include/" + hdr], hdr) |
||||
continue |
||||
|
||||
if "." not in hdr: |
||||
# assume a c++ system include |
||||
continue |
||||
|
||||
if hdr in EXTERNAL_DEPS: |
||||
if isinstance(EXTERNAL_DEPS[hdr], list): |
||||
for dep in EXTERNAL_DEPS[hdr]: |
||||
external_deps.add(dep, hdr) |
||||
else: |
||||
external_deps.add(EXTERNAL_DEPS[hdr], hdr) |
||||
continue |
||||
|
||||
if hdr.startswith("opencensus/"): |
||||
trail = hdr[len("opencensus/") :] |
||||
trail = trail[: trail.find("/")] |
||||
external_deps.add("opencensus-" + trail, hdr) |
||||
continue |
||||
|
||||
if hdr.startswith("envoy/"): |
||||
path, file = os.path.split(hdr) |
||||
file = file.split(".") |
||||
path = path.split("/") |
||||
dep = "_".join(path[:-1] + [file[1]]) |
||||
deps.add(dep, hdr) |
||||
continue |
||||
|
||||
if hdr.startswith("google/protobuf/") and not hdr.endswith(".upb.h"): |
||||
external_deps.add("protobuf_headers", hdr) |
||||
continue |
||||
|
||||
if "/" not in hdr: |
||||
# assume a system include |
||||
continue |
||||
|
||||
is_sys_include = False |
||||
for sys_path in [ |
||||
"sys", |
||||
"arpa", |
||||
"gperftools", |
||||
"netinet", |
||||
"linux", |
||||
"android", |
||||
"mach", |
||||
"net", |
||||
"CoreFoundation", |
||||
]: |
||||
if hdr.startswith(sys_path + "/"): |
||||
is_sys_include = True |
||||
break |
||||
if is_sys_include: |
||||
# assume a system include |
||||
continue |
||||
|
||||
print( |
||||
"# ERROR: can't categorize header: %s used by %s" % (hdr, library) |
||||
) |
||||
error = True |
||||
|
||||
deps.remove(library) |
||||
|
||||
deps = sorted( |
||||
deps.best(lambda x: SCORERS[args.score](x, original_deps[library])) |
||||
) |
||||
external_deps = sorted( |
||||
external_deps.best( |
||||
lambda x: SCORERS[args.score](x, original_external_deps[library]) |
||||
) |
||||
) |
||||
|
||||
return (library, error, deps, external_deps) |
||||
|
||||
|
||||
def main() -> None: |
||||
update_libraries = [] |
||||
for library in sorted(consumes.keys()): |
||||
if library in no_update: |
||||
continue |
||||
if args.targets and library not in args.targets: |
||||
continue |
||||
update_libraries.append(library) |
||||
with multiprocessing.Pool(processes=multiprocessing.cpu_count()) as p: |
||||
updated_libraries = p.map(make_library, update_libraries, 1) |
||||
|
||||
error = False |
||||
for library, lib_error, deps, external_deps in updated_libraries: |
||||
if lib_error: |
||||
error = True |
||||
continue |
||||
buildozer_set_list("external_deps", external_deps, library, via="deps") |
||||
buildozer_set_list("deps", deps, library) |
||||
|
||||
run_buildozer.run_buildozer(buildozer_commands) |
||||
|
||||
if error: |
||||
sys.exit(1) |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
main() |
||||
@ -1,63 +0,0 @@ |
||||
#!/bin/bash |
||||
# Copyright 2021 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
echo "NOTE: to automagically apply fixes, invoke with --fix" |
||||
|
||||
set -ex |
||||
|
||||
# change to root directory |
||||
cd $(dirname $0)/../.. |
||||
REPO_ROOT=$(pwd) |
||||
|
||||
# grep targets with manual tag, which is not included in a result of bazel build using ... |
||||
# let's get a list of them using query command and pass it to gen_compilation_database.py |
||||
export MANUAL_TARGETS=$(bazel query 'attr("tags", "manual", tests(//test/cpp/...))' | grep -v _on_ios) |
||||
|
||||
# generate a clang compilation database for all C/C++ sources in the repo. |
||||
tools/distrib/gen_compilation_database.py \ |
||||
--include_headers \ |
||||
--ignore_system_headers \ |
||||
--dedup_targets \ |
||||
"//:*" \ |
||||
"//src/core/..." \ |
||||
"//src/cpp/ext/csm/..." \ |
||||
"//src/cpp/ext/gcp/..." \ |
||||
"//src/cpp/ext/otel/..." \ |
||||
"//src/compiler/..." \ |
||||
"//test/core/..." \ |
||||
"//test/cpp/..." \ |
||||
"//fuzztest/..." \ |
||||
$MANUAL_TARGETS |
||||
|
||||
# run iwyu against the checked out codebase |
||||
# when modifying the checked-out files, the current user will be impersonated |
||||
# so that the updated files don't end up being owned by "root". |
||||
if [ "$IWYU_SKIP_DOCKER" == "" ] |
||||
then |
||||
# build iwyu docker image |
||||
docker build -t grpc_iwyu tools/dockerfile/grpc_iwyu |
||||
|
||||
docker run \ |
||||
-e TEST="$TEST" \ |
||||
-e CHANGED_FILES="$CHANGED_FILES" \ |
||||
-e IWYU_ROOT="/local-code" \ |
||||
--rm=true \ |
||||
-v "${REPO_ROOT}":/local-code \ |
||||
-v "${HOME/.cache/bazel}":"${HOME/.cache/bazel}" \ |
||||
--user "$(id -u):$(id -g)" \ |
||||
-t grpc_iwyu /iwyu.sh "$@" |
||||
else |
||||
IWYU_ROOT="${REPO_ROOT}" tools/dockerfile/grpc_iwyu/iwyu.sh |
||||
fi |
||||
@ -1,9 +0,0 @@ |
||||
[ |
||||
{ include: ["<ext/alloc_traits.h>", "private", "<vector>", "public"] }, |
||||
{ include: ["<netinet/in.h>", "public", "\"src/core/lib/iomgr/sockaddr.h\"", "public"]}, |
||||
{ include: ["<sys/socket.h>", "public", "\"src/core/lib/iomgr/sockaddr.h\"", "public"]}, |
||||
{ include: ["<openssl/base.h>", "private", "<openssl/crypto.h>", "public"] }, |
||||
{ include: ["<openssl/digest.h>", "private", "<openssl/evp.h>", "public"] }, |
||||
# workaround: https://github.com/include-what-you-use/include-what-you-use/issues/908 |
||||
{ symbol: ["std::max", "private", "<algorithm>", "public" ] }, |
||||
] |
||||
@ -1,28 +0,0 @@ |
||||
# Copyright 2021 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# Docker file for running IWYU. |
||||
# Updated: 2022-11-03 |
||||
|
||||
FROM silkeh/clang:16-bullseye |
||||
|
||||
# Install prerequisites for the iwyu script |
||||
RUN apt-get update && apt-get install -y python3 jq git cmake python zlib1g-dev libtinfo-dev libclang-16-dev && apt-get clean |
||||
ADD iwyu.sh / |
||||
|
||||
# When running locally, we'll be impersonating the current user, so we need |
||||
# to make the script runnable by everyone. |
||||
RUN chmod a+rx /iwyu.sh |
||||
|
||||
CMD ["echo 'Run with tools/distrib/iwyu.sh'"] |
||||
@ -1,139 +0,0 @@ |
||||
#!/bin/sh |
||||
# Copyright 2017 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set -x |
||||
|
||||
cd ${IWYU_ROOT} |
||||
|
||||
export PATH=${PATH}:${IWYU_ROOT}/iwyu_build/bin |
||||
|
||||
# number of CPUs available |
||||
CPU_COUNT=`nproc` |
||||
|
||||
rm -rf iwyu || true |
||||
git clone https://github.com/include-what-you-use/include-what-you-use.git iwyu |
||||
|
||||
############################################################################### |
||||
# |
||||
# BEWARE! BEWARE! BEWARE! BEWARE! BEWARE! BEWARE! BEWARE! BEWARE! |
||||
# |
||||
# Changing the version of iwyu can bring along subtle changes. |
||||
# You *must* test the new version of iwyu: |
||||
# 1. run it on the entire codebase before submitting |
||||
# 2. UPLOAD A CHANGE THAT SHOULD BE BROKEN AFTER SUBMISSION OF THIS CHANGE |
||||
# ensure that the broken change is caught by the new version of iwyu |
||||
# |
||||
# BEWARE! BEWARE! BEWARE! BEWARE! BEWARE! BEWARE! BEWARE! BEWARE! |
||||
# |
||||
############################################################################### |
||||
|
||||
# latest commit on the clang_16 branch |
||||
cd ${IWYU_ROOT}/iwyu |
||||
git checkout 7301b1fc88e5e16d8df73aecea55037d9c0a371b |
||||
if [ $? -ne 0 ]; then |
||||
echo "Failed to checkout iwyu commit" |
||||
exit 1 |
||||
fi |
||||
mkdir -p ${IWYU_ROOT}/iwyu_build |
||||
cd ${IWYU_ROOT}/iwyu_build |
||||
cmake -G "Unix Makefiles" -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_ROOT_DIR=/usr/lib/llvm-16 ${IWYU_ROOT}/iwyu |
||||
if [ $? -ne 0 ]; then |
||||
echo "Failed to cmake iwyu" |
||||
exit 1 |
||||
fi |
||||
make -j $CPU_COUNT |
||||
if [ $? -ne 0 ]; then |
||||
echo "Failed to make iwyu" |
||||
exit 1 |
||||
fi |
||||
cd ${IWYU_ROOT} |
||||
|
||||
cat compile_commands.json \ |
||||
| sed "s/ -DNDEBUG//g" \ |
||||
| sed "s/ -std=c\\+\\+14/ -std=c++17/g" \ |
||||
| sed "s,\"file\": \",\"file\": \"${IWYU_ROOT}/,g" \ |
||||
> compile_commands_for_iwyu.json |
||||
|
||||
export ENABLED_MODULES=' |
||||
src/core/ext |
||||
src/core/lib |
||||
src/cpp |
||||
src/python/grpcio_observability |
||||
test/core |
||||
fuzztest |
||||
' |
||||
|
||||
export DISABLED_MODULES=' |
||||
src/core/lib/gpr |
||||
src/core/lib/iomgr |
||||
src/core/ext/transport/binder |
||||
test/core/alts |
||||
test/core/iomgr |
||||
test/core/security |
||||
test/core/tsi |
||||
test/core/transport/binder |
||||
' |
||||
|
||||
export INCLUSION_REGEX=`echo $ENABLED_MODULES | sed 's/ /|/g' | sed 's,\\(.*\\),^(\\1)/,g'` |
||||
export EXCLUSION_REGEX=`echo $DISABLED_MODULES | sed 's/ /|/g' | sed 's,\\(.*\\),^(\\1)/,g'` |
||||
|
||||
# figure out which files to include |
||||
cat compile_commands.json | jq -r '.[].file' \ |
||||
| grep -E $INCLUSION_REGEX \ |
||||
| grep -v -E "/upb-gen/|/upbdefs-gen/" \ |
||||
| grep -v -E $EXCLUSION_REGEX \ |
||||
| grep -v src/core/lib/security/credentials/tls/grpc_tls_credentials_options.h \ |
||||
| grep -v test/core/end2end/end2end_tests.cc \ |
||||
| sort \ |
||||
> iwyu_files0.txt |
||||
|
||||
cat iwyu_files0.txt \ |
||||
| xargs -d '\n' ls -1df 2> /dev/null \ |
||||
> iwyu_files.txt \ |
||||
|| true |
||||
|
||||
echo '#!/bin/sh |
||||
${IWYU_ROOT}/iwyu/iwyu_tool.py -p compile_commands_for_iwyu.json $1 \ |
||||
-- -Xiwyu --no_fwd_decls \ |
||||
-Xiwyu --update_comments \ |
||||
-Xiwyu --mapping_file=${IWYU_ROOT}/tools/distrib/iwyu_mappings.imp \ |
||||
| grep -v -E "port_platform.h" \ |
||||
| grep -v -E "repeated_ptr_field.h" \ |
||||
| grep -v -E "repeated_field.h" \ |
||||
| grep -v -E "^(- )?namespace " \ |
||||
> iwyu/iwyu.`echo $1 | sha1sum`.out |
||||
' > iwyu/run_iwyu_on.sh |
||||
chmod +x iwyu/run_iwyu_on.sh |
||||
|
||||
# run iwyu, filtering out changes to port_platform.h |
||||
xargs -n 1 -P $CPU_COUNT -a iwyu_files.txt ${IWYU_ROOT}/iwyu/run_iwyu_on.sh |
||||
|
||||
cat iwyu/iwyu.*.out > iwyu.out |
||||
|
||||
# apply the suggested changes |
||||
${IWYU_ROOT}/iwyu/fix_includes.py \ |
||||
--nocomments \ |
||||
--nosafe_headers \ |
||||
--ignore_re='^(include/.*|src/core/lib/security/credentials/tls/grpc_tls_credentials_options\.h|external/.*)' \ |
||||
< iwyu.out \ |
||||
| grep 'IWYU edited 0 files on your behalf' |
||||
|
||||
if [ $? -ne 0 ] |
||||
then |
||||
echo "Iwyu edited some files. Here is the diff of files edited by iwyu:" |
||||
git --no-pager diff |
||||
# Exit with a non zero error code to ensure sanity checks fail accordingly. |
||||
exit 1 |
||||
fi |
||||
@ -1,30 +0,0 @@ |
||||
# Copyright 2017 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# Config file for the internal CI (in protobuf text format) |
||||
|
||||
# Location of the continuous shell script in repository. |
||||
build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" |
||||
timeout_mins: 60 |
||||
action { |
||||
define_artifacts { |
||||
regex: "**/*sponge_log.*" |
||||
regex: "github/grpc/reports/**" |
||||
} |
||||
} |
||||
|
||||
env_vars { |
||||
key: "RUN_TESTS_FLAGS" |
||||
value: "-f basictests linux iwyu --inner_jobs 16 -j 1 --internal_ci --bq_result_table aggregate_results" |
||||
} |
||||
@ -1,30 +0,0 @@ |
||||
# Copyright 2017 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# Config file for the internal CI (in protobuf text format) |
||||
|
||||
# Location of the continuous shell script in repository. |
||||
build_file: "grpc/tools/internal_ci/linux/grpc_run_tests_matrix.sh" |
||||
timeout_mins: 60 |
||||
action { |
||||
define_artifacts { |
||||
regex: "**/*sponge_log.*" |
||||
regex: "github/grpc/reports/**" |
||||
} |
||||
} |
||||
|
||||
env_vars { |
||||
key: "RUN_TESTS_FLAGS" |
||||
value: "-f basictests linux iwyu --inner_jobs 16 -j 1 --internal_ci" |
||||
} |
||||
@ -1,2 +0,0 @@ |
||||
- script: tools/distrib/iwyu.sh |
||||
cpu_cost: 1000 |
||||
Loading…
Reference in new issue