|
|
|
#!/usr/bin/env python3
|
|
|
|
|
|
|
|
# Copyright 2022 gRPC authors.
|
|
|
|
#
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
|
# You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
# limitations under the License.
|
|
|
|
|
|
|
|
import argparse
|
|
|
|
import collections
|
|
|
|
from doctest import SKIP
|
|
|
|
import multiprocessing
|
|
|
|
import os
|
|
|
|
import re
|
|
|
|
import sys
|
|
|
|
|
|
|
|
import run_buildozer
|
|
|
|
|
|
|
|
# find our home
|
|
|
|
ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '../..'))
|
|
|
|
os.chdir(ROOT)
|
|
|
|
|
|
|
|
vendors = collections.defaultdict(list)
|
|
|
|
scores = collections.defaultdict(int)
|
|
|
|
avoidness = collections.defaultdict(int)
|
|
|
|
consumes = {}
|
|
|
|
no_update = set()
|
|
|
|
buildozer_commands = []
|
|
|
|
original_deps = {}
|
|
|
|
original_external_deps = {}
|
|
|
|
skip_headers = collections.defaultdict(set)
|
|
|
|
|
|
|
|
# TODO(ctiller): ideally we wouldn't hardcode a bunch of paths here.
|
|
|
|
# We can likely parse out BUILD files from dependencies to generate this index.
|
|
|
|
EXTERNAL_DEPS = {
|
|
|
|
'absl/algorithm/container.h':
|
|
|
|
'absl/algorithm:container',
|
|
|
|
'absl/base/attributes.h':
|
|
|
|
'absl/base:core_headers',
|
|
|
|
'absl/base/call_once.h':
|
|
|
|
'absl/base',
|
|
|
|
# TODO(ctiller) remove this
|
|
|
|
'absl/base/internal/endian.h':
|
|
|
|
'absl/base',
|
|
|
|
'absl/base/thread_annotations.h':
|
|
|
|
'absl/base:core_headers',
|
|
|
|
'absl/container/flat_hash_map.h':
|
|
|
|
'absl/container:flat_hash_map',
|
|
|
|
'absl/container/flat_hash_set.h':
|
|
|
|
'absl/container:flat_hash_set',
|
|
|
|
'absl/container/inlined_vector.h':
|
|
|
|
'absl/container:inlined_vector',
|
|
|
|
'absl/cleanup/cleanup.h':
|
|
|
|
'absl/cleanup',
|
|
|
|
'absl/debugging/failure_signal_handler.h':
|
|
|
|
'absl/debugging:failure_signal_handler',
|
|
|
|
'absl/debugging/stacktrace.h':
|
|
|
|
'absl/debugging:stacktrace',
|
|
|
|
'absl/debugging/symbolize.h':
|
|
|
|
'absl/debugging:symbolize',
|
|
|
|
'absl/flags/flag.h':
|
|
|
|
'absl/flags:flag',
|
|
|
|
'absl/flags/marshalling.h':
|
|
|
|
'absl/flags:marshalling',
|
|
|
|
'absl/flags/parse.h':
|
|
|
|
'absl/flags:parse',
|
|
|
|
'absl/functional/any_invocable.h':
|
|
|
|
'absl/functional:any_invocable',
|
|
|
|
'absl/functional/bind_front.h':
|
|
|
|
'absl/functional:bind_front',
|
|
|
|
'absl/functional/function_ref.h':
|
|
|
|
'absl/functional:function_ref',
|
|
|
|
'absl/hash/hash.h':
|
|
|
|
'absl/hash',
|
|
|
|
'absl/memory/memory.h':
|
|
|
|
'absl/memory',
|
|
|
|
'absl/meta/type_traits.h':
|
|
|
|
'absl/meta:type_traits',
|
|
|
|
'absl/numeric/int128.h':
|
|
|
|
'absl/numeric:int128',
|
|
|
|
'absl/random/random.h':
|
|
|
|
'absl/random',
|
|
|
|
'absl/random/distributions.h':
|
|
|
|
'absl/random:distributions',
|
|
|
|
'absl/random/uniform_int_distribution.h':
|
|
|
|
'absl/random:distributions',
|
|
|
|
'absl/status/status.h':
|
|
|
|
'absl/status',
|
|
|
|
'absl/status/statusor.h':
|
|
|
|
'absl/status:statusor',
|
|
|
|
'absl/strings/ascii.h':
|
|
|
|
'absl/strings',
|
|
|
|
'absl/strings/cord.h':
|
|
|
|
'absl/strings:cord',
|
|
|
|
'absl/strings/escaping.h':
|
|
|
|
'absl/strings',
|
|
|
|
'absl/strings/match.h':
|
|
|
|
'absl/strings',
|
|
|
|
'absl/strings/numbers.h':
|
|
|
|
'absl/strings',
|
|
|
|
'absl/strings/str_cat.h':
|
|
|
|
'absl/strings',
|
|
|
|
'absl/strings/str_format.h':
|
|
|
|
'absl/strings:str_format',
|
|
|
|
'absl/strings/str_join.h':
|
|
|
|
'absl/strings',
|
|
|
|
'absl/strings/str_replace.h':
|
|
|
|
'absl/strings',
|
|
|
|
'absl/strings/str_split.h':
|
|
|
|
'absl/strings',
|
|
|
|
'absl/strings/string_view.h':
|
|
|
|
'absl/strings',
|
|
|
|
'absl/strings/strip.h':
|
|
|
|
'absl/strings',
|
|
|
|
'absl/strings/substitute.h':
|
|
|
|
'absl/strings',
|
|
|
|
'absl/synchronization/mutex.h':
|
|
|
|
'absl/synchronization',
|
|
|
|
'absl/synchronization/notification.h':
|
|
|
|
'absl/synchronization',
|
|
|
|
'absl/time/clock.h':
|
|
|
|
'absl/time',
|
|
|
|
'absl/time/time.h':
|
|
|
|
'absl/time',
|
|
|
|
'absl/types/optional.h':
|
|
|
|
'absl/types:optional',
|
|
|
|
'absl/types/span.h':
|
|
|
|
'absl/types:span',
|
|
|
|
'absl/types/variant.h':
|
|
|
|
'absl/types:variant',
|
|
|
|
'absl/utility/utility.h':
|
|
|
|
'absl/utility',
|
|
|
|
'address_sorting/address_sorting.h':
|
|
|
|
'address_sorting',
|
|
|
|
'ares.h':
|
|
|
|
'cares',
|
|
|
|
'fuzztest/fuzztest.h': ['fuzztest', 'fuzztest_main'],
|
|
|
|
'google/api/monitored_resource.pb.h':
|
|
|
|
'google/api:monitored_resource_cc_proto',
|
|
|
|
'google/devtools/cloudtrace/v2/tracing.grpc.pb.h':
|
|
|
|
'googleapis_trace_grpc_service',
|
|
|
|
'google/logging/v2/logging.grpc.pb.h':
|
|
|
|
'googleapis_logging_grpc_service',
|
|
|
|
'google/logging/v2/logging.pb.h':
|
|
|
|
'googleapis_logging_cc_proto',
|
|
|
|
'google/logging/v2/log_entry.pb.h':
|
|
|
|
'googleapis_logging_cc_proto',
|
|
|
|
'google/monitoring/v3/metric_service.grpc.pb.h':
|
|
|
|
'googleapis_monitoring_grpc_service',
|
|
|
|
'gmock/gmock.h':
|
|
|
|
'gtest',
|
|
|
|
'gtest/gtest.h':
|
|
|
|
'gtest',
|
|
|
|
'opencensus/exporters/stats/stackdriver/stackdriver_exporter.h':
|
|
|
|
'opencensus-stats-stackdriver_exporter',
|
|
|
|
'opencensus/exporters/trace/stackdriver/stackdriver_exporter.h':
|
|
|
|
'opencensus-trace-stackdriver_exporter',
|
|
|
|
'opencensus/trace/context_util.h':
|
|
|
|
'opencensus-trace-context_util',
|
|
|
|
'opencensus/trace/propagation/grpc_trace_bin.h':
|
|
|
|
'opencensus-trace-propagation',
|
|
|
|
'opencensus/tags/context_util.h':
|
|
|
|
'opencensus-tags-context_util',
|
|
|
|
'opencensus/trace/span_context.h':
|
|
|
|
'opencensus-trace-span_context',
|
|
|
|
'openssl/base.h':
|
|
|
|
'libssl',
|
|
|
|
'openssl/bio.h':
|
|
|
|
'libssl',
|
|
|
|
'openssl/bn.h':
|
|
|
|
'libcrypto',
|
|
|
|
'openssl/buffer.h':
|
|
|
|
'libcrypto',
|
|
|
|
'openssl/crypto.h':
|
|
|
|
'libcrypto',
|
|
|
|
'openssl/digest.h':
|
|
|
|
'libssl',
|
|
|
|
'openssl/engine.h':
|
|
|
|
'libcrypto',
|
|
|
|
'openssl/err.h':
|
|
|
|
'libcrypto',
|
|
|
|
'openssl/evp.h':
|
|
|
|
'libcrypto',
|
|
|
|
'openssl/hmac.h':
|
|
|
|
'libcrypto',
|
|
|
|
'openssl/pem.h':
|
|
|
|
'libcrypto',
|
|
|
|
'openssl/rsa.h':
|
|
|
|
'libcrypto',
|
|
|
|
'openssl/sha.h':
|
|
|
|
'libcrypto',
|
|
|
|
'openssl/ssl.h':
|
|
|
|
'libssl',
|
|
|
|
'openssl/tls1.h':
|
|
|
|
'libssl',
|
|
|
|
'openssl/x509.h':
|
|
|
|
'libcrypto',
|
|
|
|
'openssl/x509v3.h':
|
|
|
|
'libcrypto',
|
|
|
|
're2/re2.h':
|
|
|
|
're2',
|
|
|
|
'upb/arena.h':
|
|
|
|
'upb_lib',
|
[protobuf] Upgrade third_party/protobuf to 22.x (#32606)
The very non-trivial upgrade of third_party/protobuf to 22.x
This PR strives to be as small as possible and many changes that were
compatible with protobuf 21.x and didn't have to be merged atomically
with the upgrade were already merged.
Due to the complexity of the upgrade, this PR wasn't created
automatically by a tool, but manually. Subsequent upgraded of
third_party/protobuf with our OSS release script should work again once
this change is merged.
This is best reviewed commit-by-commit, I tried to group changes in
logical areas.
Notable changes:
- the upgrade of third_party/protobuf submodule, the bazel protobuf
dependency itself
- upgrade of UPB dependency to 22.x (in the past, we used to always
upgrade upb to "main", but upb now has release branch as well). UPB
needs to be upgraded atomically with protobuf since there's a de-facto
circular dependency (new protobuf depends on new upb, which depends on
new protobuf for codegen).
- some protobuf and upb bazel rules are now aliases, so `
extract_metadata_from_bazel_xml.py` and `gen_upb_api_from_bazel_xml.py`
had to be modified to be able to follow aliases and reach the actual
aliased targets.
- some protobuf public headers were renamed, so especially
`src/compiler` needed to be updated to use the new headers.
- protobuf and upb now both depend on utf8_range project, so since we
bundle upb with grpc in some languages, we now have to bundle utf8_range
as well (hence changes in build for python, PHP, objC, cmake etc).
- protoc now depends on absl and utf8_range (previously protobuf had
absl dependency, but not for the codegen part), so python's
make_grpcio_tools.py required partial rewrite to be able to handle those
dependencies in the grpcio_tools build.
- many updates and fixes required for C++ distribtests (currently they
all pass, but we'll probably need to follow up, make protobuf's and
grpc's handling of dependencies more aligned and revisit the
distribtests)
- bunch of other changes mostly due to overhaul of protobuf's and upb's
internal build layout.
TODOs:
- [DONE] make sure IWYU and clang_tidy_code pass
- create a list of followups (e.g. work to reenable the few tests I had
to disable and to remove workaround I had to use)
- [DONE in cl/523706129] figure out problem(s) with internal import
---------
Co-authored-by: Craig Tiller <ctiller@google.com>
2 years ago
|
|
|
'upb/base/string_view.h':
|
|
|
|
'upb_lib',
|
|
|
|
'upb/collections/map.h':
|
|
|
|
'upb_collections_lib',
|
|
|
|
'upb/def.h':
|
|
|
|
'upb_lib',
|
|
|
|
'upb/json_encode.h':
|
|
|
|
'upb_json_lib',
|
[protobuf] Upgrade third_party/protobuf to 22.x (#32606)
The very non-trivial upgrade of third_party/protobuf to 22.x
This PR strives to be as small as possible and many changes that were
compatible with protobuf 21.x and didn't have to be merged atomically
with the upgrade were already merged.
Due to the complexity of the upgrade, this PR wasn't created
automatically by a tool, but manually. Subsequent upgraded of
third_party/protobuf with our OSS release script should work again once
this change is merged.
This is best reviewed commit-by-commit, I tried to group changes in
logical areas.
Notable changes:
- the upgrade of third_party/protobuf submodule, the bazel protobuf
dependency itself
- upgrade of UPB dependency to 22.x (in the past, we used to always
upgrade upb to "main", but upb now has release branch as well). UPB
needs to be upgraded atomically with protobuf since there's a de-facto
circular dependency (new protobuf depends on new upb, which depends on
new protobuf for codegen).
- some protobuf and upb bazel rules are now aliases, so `
extract_metadata_from_bazel_xml.py` and `gen_upb_api_from_bazel_xml.py`
had to be modified to be able to follow aliases and reach the actual
aliased targets.
- some protobuf public headers were renamed, so especially
`src/compiler` needed to be updated to use the new headers.
- protobuf and upb now both depend on utf8_range project, so since we
bundle upb with grpc in some languages, we now have to bundle utf8_range
as well (hence changes in build for python, PHP, objC, cmake etc).
- protoc now depends on absl and utf8_range (previously protobuf had
absl dependency, but not for the codegen part), so python's
make_grpcio_tools.py required partial rewrite to be able to handle those
dependencies in the grpcio_tools build.
- many updates and fixes required for C++ distribtests (currently they
all pass, but we'll probably need to follow up, make protobuf's and
grpc's handling of dependencies more aligned and revisit the
distribtests)
- bunch of other changes mostly due to overhaul of protobuf's and upb's
internal build layout.
TODOs:
- [DONE] make sure IWYU and clang_tidy_code pass
- create a list of followups (e.g. work to reenable the few tests I had
to disable and to remove workaround I had to use)
- [DONE in cl/523706129] figure out problem(s) with internal import
---------
Co-authored-by: Craig Tiller <ctiller@google.com>
2 years ago
|
|
|
'upb/mem/arena.h':
|
|
|
|
'upb_lib',
|
|
|
|
'upb/text_encode.h':
|
|
|
|
'upb_textformat_lib',
|
|
|
|
'upb/def.hpp':
|
|
|
|
'upb_reflection',
|
|
|
|
'upb/upb.h':
|
|
|
|
'upb_lib',
|
|
|
|
'upb/upb.hpp':
|
|
|
|
'upb_lib',
|
|
|
|
'xxhash.h':
|
|
|
|
'xxhash',
|
|
|
|
'zlib.h':
|
|
|
|
'madler_zlib',
|
|
|
|
}
|
|
|
|
|
|
|
|
INTERNAL_DEPS = {
|
[fuzz] Turn core end2end tests into fuzzers (#33013)
Add a new binary that runs all core end2end tests in fuzzing mode.
In this mode FuzzingEventEngine is substituted for the default event
engine. This means that time is simulated, as is IO. The FEE gets
control of callback delays also.
In our tests the `Step()` function becomes, instead of a single call to
`completion_queue_next`, a series of calls to that function and
`FuzzingEventEngine::Tick`, driving forward the event loop until
progress can be made.
PR guide:
---
**New binaries**
`core_end2end_test_fuzzer` - the new fuzzer itself
`seed_end2end_corpus` - a tool that produces an interesting seed corpus
**Config changes for safe fuzzing**
The implementation tries to use the config fuzzing work we've previously
deployed in api_fuzzer to fuzz across experiments. Since some
experiments are far too experimental to be safe in such fuzzing (and
this will always be the case):
- a new flag is added to experiments to opt-out of this fuzzing
- a new hook is added to the config system to allow variables to
re-write their inputs before setting them during the fuzz
**Event manager/IO changes**
Changes are made to the event engine shims so that tcp_server_posix can
run with a non-FD carrying EventEngine. These are in my mind a bit
clunky, but they work and they're in code that we expect to delete in
the medium term, so I think overall the approach is good.
**Changes to time**
A small tweak is made to fix a bug initializing time for fuzzers in
time.cc - we were previously failing to initialize
`g_process_epoch_cycles`
**Changes to `Crash`**
A version that prints to stdio is added so that we can reliably print a
crash from the fuzzer.
**Changes to CqVerifier**
Hooks are added to allow the top level loop to hook the verification
functions with a function that steps time between CQ polls.
**Changes to end2end fixtures**
State machinery moves from the fixture to the test infra, to keep the
customizations for fuzzing or not in one place. This means that fixtures
are now just client/server factories, which is overall nice.
It did necessitate moving some bespoke machinery into
h2_ssl_cert_test.cc - this file is beginning to be problematic in
borrowing parts but not all of the e2e test machinery. Some future PR
needs to solve this.
A cq arg is added to the Make functions since the cq is now owned by the
test and not the fixture.
**Changes to test registration**
`TEST_P` is replaced by `CORE_END2END_TEST` and our own test registry is
used as a first depot for test information.
The gtest version of these tests: queries that registry to manually
register tests with gtest. This ultimately changes the name of our tests
again (I think for the last time) - the new names are shorter and more
readable, so I don't count this as a regression.
The fuzzer version of these tests: constructs a database of fuzzable
tests that it can consult to look up a particular suite/test/config
combination specified by the fuzzer to fuzz against. This gives us a
single fuzzer that can test all 3k-ish fuzzing ready tests and cross
polinate configuration between them.
**Changes to test config**
The zero size registry stuff was causing some problems with the event
engine feature macros, so instead I've removed those and used GTEST_SKIP
in the problematic tests. I think that's the approach we move towards in
the future.
**Which tests are included**
Configs that are compatible - those that do not do fd manipulation
directly (these are incompatible with FuzzingEventEngine), and those
that do not join threads on their shutdown path (as these are
incompatible with our cq wait methodology). Each we can talk about in
the future - fd manipulation would be a significant expansion of
FuzzingEventEngine, and is probably not worth it, however many uses of
background threads now should probably evolve to be EventEngine::Run
calls in the future, and then would be trivially enabled in the fuzzers.
Some tests currently fail in the fuzzing environment, a
`SKIP_IF_FUZZING` macro is used for these few to disable them if in the
fuzzing environment. We'll burn these down in the future.
**Changes to fuzzing_event_engine**
Changes are made to time: an exponential sweep forward is used now -
this catches small time precision things early, but makes decade long
timers (we have them) able to be used right now. In the future we'll
just skip time forward to the next scheduled timer, but that approach
doesn't yet work due to legacy timer system interactions.
Changes to port assignment: we ensure that ports are legal numbers
before assigning them via `grpc_pick_port_or_die`.
A race condition between time checking and io is fixed.
---------
Co-authored-by: ctiller <ctiller@users.noreply.github.com>
2 years ago
|
|
|
"test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.h":
|
|
|
|
"//test/core/event_engine/fuzzing_event_engine",
|
|
|
|
"test/core/event_engine/fuzzing_event_engine/fuzzing_event_engine.pb.h":
|
|
|
|
"//test/core/event_engine/fuzzing_event_engine:fuzzing_event_engine_proto",
|
|
|
|
'google/api/expr/v1alpha1/syntax.upb.h':
|
|
|
|
'google_type_expr_upb',
|
|
|
|
'google/rpc/status.upb.h':
|
|
|
|
'google_rpc_status_upb',
|
|
|
|
'google/protobuf/any.upb.h':
|
|
|
|
'protobuf_any_upb',
|
|
|
|
'google/protobuf/duration.upb.h':
|
|
|
|
'protobuf_duration_upb',
|
|
|
|
'google/protobuf/struct.upb.h':
|
|
|
|
'protobuf_struct_upb',
|
|
|
|
'google/protobuf/timestamp.upb.h':
|
|
|
|
'protobuf_timestamp_upb',
|
|
|
|
'google/protobuf/wrappers.upb.h':
|
|
|
|
'protobuf_wrappers_upb',
|
|
|
|
'grpc/status.h':
|
|
|
|
'grpc_public_hdrs',
|
|
|
|
'src/proto/grpc/channelz/channelz.grpc.pb.h':
|
|
|
|
'//src/proto/grpc/channelz:channelz_proto',
|
|
|
|
'src/proto/grpc/core/stats.pb.h':
|
|
|
|
'//src/proto/grpc/core:stats_proto',
|
|
|
|
'src/proto/grpc/health/v1/health.upb.h':
|
|
|
|
'grpc_health_upb',
|
|
|
|
'src/proto/grpc/lb/v1/load_reporter.grpc.pb.h':
|
|
|
|
'//src/proto/grpc/lb/v1:load_reporter_proto',
|
|
|
|
'src/proto/grpc/lb/v1/load_balancer.upb.h':
|
|
|
|
'grpc_lb_upb',
|
|
|
|
'src/proto/grpc/reflection/v1alpha/reflection.grpc.pb.h':
|
|
|
|
'//src/proto/grpc/reflection/v1alpha:reflection_proto',
|
|
|
|
'src/proto/grpc/gcp/transport_security_common.upb.h':
|
|
|
|
'alts_upb',
|
|
|
|
'src/proto/grpc/gcp/handshaker.upb.h':
|
|
|
|
'alts_upb',
|
|
|
|
'src/proto/grpc/gcp/altscontext.upb.h':
|
|
|
|
'alts_upb',
|
|
|
|
'src/proto/grpc/lookup/v1/rls.upb.h':
|
|
|
|
'rls_upb',
|
|
|
|
'src/proto/grpc/lookup/v1/rls_config.upb.h':
|
|
|
|
'rls_config_upb',
|
|
|
|
'src/proto/grpc/lookup/v1/rls_config.upbdefs.h':
|
|
|
|
'rls_config_upbdefs',
|
|
|
|
'src/proto/grpc/testing/xds/v3/csds.grpc.pb.h':
|
|
|
|
'//src/proto/grpc/testing/xds/v3:csds_proto',
|
|
|
|
'xds/data/orca/v3/orca_load_report.upb.h':
|
|
|
|
'xds_orca_upb',
|
|
|
|
'xds/service/orca/v3/orca.upb.h':
|
|
|
|
'xds_orca_service_upb',
|
|
|
|
'xds/type/v3/typed_struct.upb.h':
|
|
|
|
'xds_type_upb',
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
class FakeSelects:
|
|
|
|
|
|
|
|
def config_setting_group(self, **kwargs):
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
num_cc_libraries = 0
|
|
|
|
num_opted_out_cc_libraries = 0
|
|
|
|
parsing_path = None
|
|
|
|
|
|
|
|
|
|
|
|
# Convert the source or header target to a relative path.
|
|
|
|
def _get_filename(name, parsing_path):
|
|
|
|
filename = '%s%s' % (
|
|
|
|
(parsing_path + '/' if
|
|
|
|
(parsing_path and not name.startswith('//')) else ''), name)
|
|
|
|
filename = filename.replace('//:', '')
|
|
|
|
filename = filename.replace('//src/core:', 'src/core/')
|
|
|
|
filename = filename.replace('//src/cpp/ext/filters/census:',
|
|
|
|
'src/cpp/ext/filters/census/')
|
|
|
|
return filename
|
|
|
|
|
|
|
|
|
|
|
|
def grpc_cc_library(name,
|
|
|
|
hdrs=[],
|
|
|
|
public_hdrs=[],
|
|
|
|
srcs=[],
|
|
|
|
select_deps=None,
|
|
|
|
tags=[],
|
|
|
|
deps=[],
|
|
|
|
external_deps=[],
|
|
|
|
proto=None,
|
|
|
|
**kwargs):
|
|
|
|
global args
|
|
|
|
global num_cc_libraries
|
|
|
|
global num_opted_out_cc_libraries
|
|
|
|
global parsing_path
|
|
|
|
assert (parsing_path is not None)
|
|
|
|
name = '//%s:%s' % (parsing_path, name)
|
|
|
|
num_cc_libraries += 1
|
|
|
|
if select_deps or 'nofixdeps' in tags:
|
|
|
|
if args.whats_left and not select_deps and 'nofixdeps' not in tags:
|
|
|
|
num_opted_out_cc_libraries += 1
|
|
|
|
print("Not opted in: {}".format(name))
|
|
|
|
no_update.add(name)
|
|
|
|
scores[name] = len(public_hdrs + hdrs)
|
|
|
|
# avoid_dep is the internal way of saying prefer something else
|
|
|
|
# we add grpc_avoid_dep to allow internal grpc-only stuff to avoid each
|
|
|
|
# other, whilst not biasing dependent projects
|
|
|
|
if 'avoid_dep' in tags or 'grpc_avoid_dep' in tags:
|
|
|
|
avoidness[name] += 10
|
|
|
|
if proto:
|
|
|
|
proto_hdr = '%s%s' % ((parsing_path + '/' if parsing_path else ''),
|
|
|
|
proto.replace('.proto', '.pb.h'))
|
|
|
|
skip_headers[name].add(proto_hdr)
|
|
|
|
|
|
|
|
for hdr in hdrs + public_hdrs:
|
|
|
|
vendors[_get_filename(hdr, parsing_path)].append(name)
|
|
|
|
inc = set()
|
|
|
|
original_deps[name] = frozenset(deps)
|
|
|
|
original_external_deps[name] = frozenset(external_deps)
|
|
|
|
for src in hdrs + public_hdrs + srcs:
|
|
|
|
for line in open(_get_filename(src, parsing_path)):
|
|
|
|
m = re.search(r'^#include <(.*)>', line)
|
|
|
|
if m:
|
|
|
|
inc.add(m.group(1))
|
|
|
|
m = re.search(r'^#include "(.*)"', line)
|
|
|
|
if m:
|
|
|
|
inc.add(m.group(1))
|
|
|
|
consumes[name] = list(inc)
|
|
|
|
|
|
|
|
|
|
|
|
def grpc_proto_library(name, srcs, **kwargs):
|
|
|
|
global parsing_path
|
|
|
|
assert (parsing_path is not None)
|
|
|
|
name = '//%s:%s' % (parsing_path, name)
|
|
|
|
for src in srcs:
|
|
|
|
proto_hdr = src.replace('.proto', '.pb.h')
|
|
|
|
vendors[_get_filename(proto_hdr, parsing_path)].append(name)
|
|
|
|
|
|
|
|
|
|
|
|
def buildozer(cmd, target):
|
|
|
|
buildozer_commands.append('%s|%s' % (cmd, target))
|
|
|
|
|
|
|
|
|
|
|
|
def buildozer_set_list(name, values, target, via=""):
|
|
|
|
if not values:
|
|
|
|
buildozer('remove %s' % name, target)
|
|
|
|
return
|
|
|
|
adjust = via if via else name
|
|
|
|
buildozer('set %s %s' % (adjust, ' '.join('"%s"' % s for s in values)),
|
|
|
|
target)
|
|
|
|
if via:
|
|
|
|
buildozer('remove %s' % name, target)
|
|
|
|
buildozer('rename %s %s' % (via, name), target)
|
|
|
|
|
|
|
|
|
|
|
|
def score_edit_distance(proposed, existing):
|
|
|
|
"""Score a proposed change primarily by edit distance"""
|
|
|
|
sum = 0
|
|
|
|
for p in proposed:
|
|
|
|
if p not in existing:
|
|
|
|
sum += 1
|
|
|
|
for e in existing:
|
|
|
|
if e not in proposed:
|
|
|
|
sum += 1
|
|
|
|
return sum
|
|
|
|
|
|
|
|
|
|
|
|
def total_score(proposal):
|
|
|
|
return sum(scores[dep] for dep in proposal)
|
|
|
|
|
|
|
|
|
|
|
|
def total_avoidness(proposal):
|
|
|
|
return sum(avoidness[dep] for dep in proposal)
|
|
|
|
|
|
|
|
|
|
|
|
def score_list_size(proposed, existing):
|
|
|
|
"""Score a proposed change primarily by number of dependencies"""
|
|
|
|
return len(proposed)
|
|
|
|
|
|
|
|
|
|
|
|
def score_best(proposed, existing):
|
|
|
|
"""Score a proposed change primarily by dependency score"""
|
|
|
|
return 0
|
|
|
|
|
|
|
|
|
|
|
|
SCORERS = {
|
|
|
|
'edit_distance': score_edit_distance,
|
|
|
|
'list_size': score_list_size,
|
|
|
|
'best': score_best,
|
|
|
|
}
|
|
|
|
|
|
|
|
parser = argparse.ArgumentParser(description='Fix build dependencies')
|
|
|
|
parser.add_argument('targets',
|
|
|
|
nargs='*',
|
|
|
|
default=[],
|
|
|
|
help='targets to fix (empty => all)')
|
|
|
|
parser.add_argument('--score',
|
|
|
|
type=str,
|
|
|
|
default='edit_distance',
|
|
|
|
help='scoring function to use: one of ' +
|
|
|
|
', '.join(SCORERS.keys()))
|
|
|
|
parser.add_argument('--whats_left',
|
|
|
|
action='store_true',
|
|
|
|
default=False,
|
|
|
|
help='show what is left to opt in')
|
|
|
|
parser.add_argument('--explain',
|
|
|
|
action='store_true',
|
|
|
|
default=False,
|
|
|
|
help='try to explain some decisions')
|
|
|
|
parser.add_argument(
|
|
|
|
'--why',
|
|
|
|
type=str,
|
|
|
|
default=None,
|
|
|
|
help='with --explain, target why a given dependency is needed')
|
|
|
|
args = parser.parse_args()
|
|
|
|
|
|
|
|
for dirname in [
|
|
|
|
"",
|
|
|
|
"src/core",
|
|
|
|
"src/cpp/ext/gcp",
|
|
|
|
"test/core/backoff",
|
|
|
|
"test/core/uri",
|
|
|
|
"test/core/util",
|
|
|
|
"test/core/end2end",
|
|
|
|
"test/core/event_engine",
|
|
|
|
"test/core/filters",
|
|
|
|
"test/core/promise",
|
|
|
|
"test/core/resource_quota",
|
|
|
|
"test/core/transport/chaotic_good",
|
|
|
|
"fuzztest",
|
|
|
|
"fuzztest/core/channel",
|
|
|
|
]:
|
|
|
|
parsing_path = dirname
|
|
|
|
exec(
|
|
|
|
open('%sBUILD' % (dirname + '/' if dirname else ''), 'r').read(), {
|
|
|
|
'load': lambda filename, *args: None,
|
|
|
|
'licenses': lambda licenses: None,
|
|
|
|
'package': lambda **kwargs: None,
|
|
|
|
'exports_files': lambda files, visibility=None: None,
|
|
|
|
'bool_flag': lambda **kwargs: None,
|
|
|
|
'config_setting': lambda **kwargs: None,
|
|
|
|
'selects': FakeSelects(),
|
|
|
|
'python_config_settings': lambda **kwargs: None,
|
|
|
|
'grpc_cc_binary': grpc_cc_library,
|
|
|
|
'grpc_cc_library': grpc_cc_library,
|
|
|
|
'grpc_cc_test': grpc_cc_library,
|
|
|
|
'grpc_core_end2end_test': lambda **kwargs: None,
|
|
|
|
'grpc_fuzzer': grpc_cc_library,
|
|
|
|
'grpc_fuzz_test': grpc_cc_library,
|
|
|
|
'grpc_proto_fuzzer': grpc_cc_library,
|
|
|
|
'grpc_proto_library': grpc_proto_library,
|
|
|
|
'select': lambda d: d["//conditions:default"],
|
|
|
|
'glob': lambda files: None,
|
|
|
|
'grpc_end2end_tests': lambda: None,
|
|
|
|
'grpc_upb_proto_library': lambda name, **kwargs: None,
|
|
|
|
'grpc_upb_proto_reflection_library': lambda name, **kwargs: None,
|
|
|
|
'grpc_generate_one_off_targets': lambda: None,
|
|
|
|
'grpc_generate_one_off_internal_targets': lambda: None,
|
|
|
|
'grpc_package': lambda **kwargs: None,
|
|
|
|
'filegroup': lambda name, **kwargs: None,
|
|
|
|
'sh_library': lambda name, **kwargs: None,
|
|
|
|
}, {})
|
|
|
|
parsing_path = None
|
|
|
|
|
|
|
|
if args.whats_left:
|
|
|
|
print("{}/{} libraries are opted in".format(
|
|
|
|
num_cc_libraries - num_opted_out_cc_libraries, num_cc_libraries))
|
|
|
|
|
|
|
|
|
|
|
|
def make_relative_path(dep, lib):
|
|
|
|
if lib is None:
|
|
|
|
return dep
|
|
|
|
lib_path = lib[:lib.rfind(':') + 1]
|
|
|
|
if dep.startswith(lib_path):
|
|
|
|
return dep[len(lib_path):]
|
|
|
|
return dep
|
|
|
|
|
|
|
|
|
|
|
|
if args.whats_left:
|
|
|
|
print("{}/{} libraries are opted in".format(
|
|
|
|
num_cc_libraries - num_opted_out_cc_libraries, num_cc_libraries))
|
|
|
|
|
|
|
|
|
|
|
|
# Keeps track of all possible sets of dependencies that could satify the
|
|
|
|
# problem. (models the list monad in Haskell!)
|
|
|
|
class Choices:
|
|
|
|
|
|
|
|
def __init__(self, library, substitutions):
|
|
|
|
self.library = library
|
|
|
|
self.to_add = []
|
|
|
|
self.to_remove = []
|
|
|
|
self.substitutions = substitutions
|
|
|
|
|
|
|
|
def add_one_of(self, choices, trigger):
|
|
|
|
if not choices:
|
|
|
|
return
|
|
|
|
choices = sum([self.apply_substitutions(choice) for choice in choices],
|
|
|
|
[])
|
|
|
|
if args.explain and (args.why is None or args.why in choices):
|
|
|
|
print("{}: Adding one of {} for {}".format(self.library, choices,
|
|
|
|
trigger))
|
|
|
|
self.to_add.append(
|
|
|
|
tuple(
|
|
|
|
make_relative_path(choice, self.library) for choice in choices))
|
|
|
|
|
|
|
|
def add(self, choice, trigger):
|
|
|
|
self.add_one_of([choice], trigger)
|
|
|
|
|
|
|
|
def remove(self, remove):
|
|
|
|
for remove in self.apply_substitutions(remove):
|
|
|
|
self.to_remove.append(make_relative_path(remove, self.library))
|
|
|
|
|
|
|
|
def apply_substitutions(self, dep):
|
|
|
|
if dep in self.substitutions:
|
|
|
|
return self.substitutions[dep]
|
|
|
|
return [dep]
|
|
|
|
|
|
|
|
def best(self, scorer):
|
|
|
|
choices = set()
|
|
|
|
choices.add(frozenset())
|
|
|
|
|
|
|
|
for add in sorted(set(self.to_add), key=lambda x: (len(x), x)):
|
|
|
|
new_choices = set()
|
|
|
|
for append_choice in add:
|
|
|
|
for choice in choices:
|
|
|
|
new_choices.add(choice.union([append_choice]))
|
|
|
|
choices = new_choices
|
|
|
|
for remove in sorted(set(self.to_remove)):
|
|
|
|
new_choices = set()
|
|
|
|
for choice in choices:
|
|
|
|
new_choices.add(choice.difference([remove]))
|
|
|
|
choices = new_choices
|
|
|
|
|
|
|
|
best = None
|
|
|
|
|
|
|
|
def final_scorer(x):
|
|
|
|
return (total_avoidness(x), scorer(x), total_score(x))
|
|
|
|
|
|
|
|
for choice in choices:
|
|
|
|
if best is None or final_scorer(choice) < final_scorer(best):
|
|
|
|
best = choice
|
|
|
|
return best
|
|
|
|
|
|
|
|
|
|
|
|
def make_library(library):
|
|
|
|
error = False
|
|
|
|
hdrs = sorted(consumes[library])
|
|
|
|
# we need a little trickery here since grpc_base has channel.cc, which calls grpc_init
|
|
|
|
# which is in grpc, which is illegal but hard to change
|
|
|
|
# once EventEngine lands we can clean this up
|
|
|
|
deps = Choices(library, {'//:grpc_base': ['//:grpc', '//:grpc_unsecure']}
|
|
|
|
if library.startswith('//test/') else {})
|
|
|
|
external_deps = Choices(None, {})
|
|
|
|
for hdr in hdrs:
|
|
|
|
if hdr in skip_headers[library]:
|
|
|
|
continue
|
|
|
|
|
|
|
|
if hdr == 'systemd/sd-daemon.h':
|
|
|
|
continue
|
|
|
|
|
|
|
|
if hdr == 'src/core/lib/profiling/stap_probes.h':
|
|
|
|
continue
|
|
|
|
|
|
|
|
if hdr.startswith('src/libfuzzer/'):
|
|
|
|
continue
|
|
|
|
|
|
|
|
if hdr == 'grpc/grpc.h' and library.startswith('//test:'):
|
|
|
|
# not the root build including grpc.h ==> //:grpc
|
|
|
|
deps.add_one_of(['//:grpc', '//:grpc_unsecure'], hdr)
|
|
|
|
continue
|
|
|
|
|
|
|
|
if hdr in INTERNAL_DEPS:
|
|
|
|
dep = INTERNAL_DEPS[hdr]
|
|
|
|
if isinstance(dep, list):
|
|
|
|
for d in dep:
|
|
|
|
deps.add(d, hdr)
|
|
|
|
else:
|
|
|
|
if not ('//' in dep):
|
|
|
|
dep = '//:' + dep
|
|
|
|
deps.add(dep, hdr)
|
|
|
|
continue
|
|
|
|
|
|
|
|
if hdr in vendors:
|
|
|
|
deps.add_one_of(vendors[hdr], hdr)
|
|
|
|
continue
|
|
|
|
|
|
|
|
if 'include/' + hdr in vendors:
|
|
|
|
deps.add_one_of(vendors['include/' + hdr], hdr)
|
|
|
|
continue
|
|
|
|
|
|
|
|
if '.' not in hdr:
|
|
|
|
# assume a c++ system include
|
|
|
|
continue
|
|
|
|
|
|
|
|
if hdr in EXTERNAL_DEPS:
|
|
|
|
if isinstance(EXTERNAL_DEPS[hdr], list):
|
|
|
|
for dep in EXTERNAL_DEPS[hdr]:
|
|
|
|
external_deps.add(dep, hdr)
|
|
|
|
else:
|
|
|
|
external_deps.add(EXTERNAL_DEPS[hdr], hdr)
|
|
|
|
continue
|
|
|
|
|
|
|
|
if hdr.startswith('opencensus/'):
|
|
|
|
trail = hdr[len('opencensus/'):]
|
|
|
|
trail = trail[:trail.find('/')]
|
|
|
|
external_deps.add('opencensus-' + trail, hdr)
|
|
|
|
continue
|
|
|
|
|
|
|
|
if hdr.startswith('envoy/'):
|
|
|
|
path, file = os.path.split(hdr)
|
|
|
|
file = file.split('.')
|
|
|
|
path = path.split('/')
|
|
|
|
dep = '_'.join(path[:-1] + [file[1]])
|
|
|
|
deps.add(dep, hdr)
|
|
|
|
continue
|
|
|
|
|
|
|
|
if hdr.startswith('google/protobuf/') and not hdr.endswith('.upb.h'):
|
|
|
|
external_deps.add('protobuf_headers', hdr)
|
|
|
|
continue
|
|
|
|
|
|
|
|
if '/' not in hdr:
|
|
|
|
# assume a system include
|
|
|
|
continue
|
|
|
|
|
|
|
|
is_sys_include = False
|
|
|
|
for sys_path in [
|
|
|
|
'sys',
|
|
|
|
'arpa',
|
|
|
|
'gperftools',
|
|
|
|
'netinet',
|
|
|
|
'linux',
|
|
|
|
'android',
|
|
|
|
'mach',
|
|
|
|
'net',
|
|
|
|
'CoreFoundation',
|
|
|
|
]:
|
|
|
|
if hdr.startswith(sys_path + '/'):
|
|
|
|
is_sys_include = True
|
|
|
|
break
|
|
|
|
if is_sys_include:
|
|
|
|
# assume a system include
|
|
|
|
continue
|
|
|
|
|
|
|
|
print("# ERROR: can't categorize header: %s used by %s" %
|
|
|
|
(hdr, library))
|
|
|
|
error = True
|
|
|
|
|
|
|
|
deps.remove(library)
|
|
|
|
|
|
|
|
deps = sorted(
|
|
|
|
deps.best(lambda x: SCORERS[args.score](x, original_deps[library])))
|
|
|
|
external_deps = sorted(
|
|
|
|
external_deps.best(lambda x: SCORERS[args.score]
|
|
|
|
(x, original_external_deps[library])))
|
|
|
|
|
|
|
|
return (library, error, deps, external_deps)
|
|
|
|
|
|
|
|
|
|
|
|
def main() -> None:
|
|
|
|
update_libraries = []
|
|
|
|
for library in sorted(consumes.keys()):
|
|
|
|
if library in no_update:
|
|
|
|
continue
|
|
|
|
if args.targets and library not in args.targets:
|
|
|
|
continue
|
|
|
|
update_libraries.append(library)
|
|
|
|
with multiprocessing.Pool(processes=multiprocessing.cpu_count()) as p:
|
|
|
|
updated_libraries = p.map(make_library, update_libraries, 1)
|
|
|
|
|
|
|
|
error = False
|
|
|
|
for library, lib_error, deps, external_deps in updated_libraries:
|
|
|
|
if lib_error:
|
|
|
|
error = True
|
|
|
|
continue
|
|
|
|
buildozer_set_list('external_deps', external_deps, library, via='deps')
|
|
|
|
buildozer_set_list('deps', deps, library)
|
|
|
|
|
|
|
|
run_buildozer.run_buildozer(buildozer_commands)
|
|
|
|
|
|
|
|
if error:
|
|
|
|
sys.exit(1)
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
main()
|