@ -1,4 +1,4 @@
#!/usr/bin/env python #!/usr/bin/env python3
# Copyright 2020 The gRPC Authors # Copyright 2020 The gRPC Authors
# #
# Licensed under the Apache License, Version 2.0 (the "License"); # Licensed under the Apache License, Version 2.0 (the "License");
@ -34,14 +34,18 @@ import subprocess
import yaml import yaml
import xml . etree . ElementTree as ET import xml . etree . ElementTree as ET
import os import os
import collections
import sys import sys
import re
from typing import List , Any , Dict , Optional , Iterable
import build_cleaner import build_cleaner
_ROOT = os . path . abspath ( os . path . join ( os . path . dirname ( sys . argv [ 0 ] ) , ' ../.. ' ) ) BuildMetadata = Dict [ str , Any ]
os . chdir ( _ROOT ) BuildDict = Dict [ str , BuildMetadata ]
BuildYaml = Dict [ str , Any ]
def _bazel_query_xml_tree ( query ) : def _bazel_query_xml_tree ( query : str ) - > ET . Element :
""" Get xml output of bazel query invocation, parsed as XML tree """ """ Get xml output of bazel query invocation, parsed as XML tree """
output = subprocess . check_output ( output = subprocess . check_output (
[ ' tools/bazel ' , ' query ' , ' --noimplicit_deps ' , ' --output ' , ' xml ' , query ] ) [ ' tools/bazel ' , ' query ' , ' --noimplicit_deps ' , ' --output ' , ' xml ' , query ] )
@ -98,14 +102,14 @@ def _extract_rules_from_bazel_xml(xml_tree):
return result return result
def _get_bazel_label ( target_name ) : def _get_bazel_label ( target_name : str ) - > str :
if ' : ' in target_name : if ' : ' in target_name :
return ' // %s ' % target_name return ' // %s ' % target_name
else : else :
return ' //: %s ' % target_name return ' //: %s ' % target_name
def _extract_source_file_path ( label ) : def _extract_source_file_path ( label : str ) - > str :
""" Gets relative path to source file from bazel deps listing """ """ Gets relative path to source file from bazel deps listing """
if label . startswith ( ' // ' ) : if label . startswith ( ' // ' ) :
label = label [ len ( ' // ' ) : ] label = label [ len ( ' // ' ) : ]
@ -117,7 +121,7 @@ def _extract_source_file_path(label):
return label return label
def _extract_public_headers ( bazel_rule ) : def _extract_public_headers ( bazel_rule : BuildMetadata ) - > List [ str ] :
""" Gets list of public headers from a bazel rule """ """ Gets list of public headers from a bazel rule """
result = [ ] result = [ ]
for dep in bazel_rule [ ' hdrs ' ] : for dep in bazel_rule [ ' hdrs ' ] :
@ -126,7 +130,7 @@ def _extract_public_headers(bazel_rule):
return list ( sorted ( result ) ) return list ( sorted ( result ) )
def _extract_nonpublic_headers ( bazel_rule ) : def _extract_nonpublic_headers ( bazel_rule : BuildMetadata ) - > List [ str ] :
""" Gets list of non-public headers from a bazel rule """ """ Gets list of non-public headers from a bazel rule """
result = [ ] result = [ ]
for dep in bazel_rule [ ' hdrs ' ] : for dep in bazel_rule [ ' hdrs ' ] :
@ -136,7 +140,7 @@ def _extract_nonpublic_headers(bazel_rule):
return list ( sorted ( result ) ) return list ( sorted ( result ) )
def _extract_sources ( bazel_rule ) : def _extract_sources ( bazel_rule : BuildMetadata ) - > List [ str ] :
""" Gets list of source files from a bazel rule """ """ Gets list of source files from a bazel rule """
result = [ ] result = [ ]
for dep in bazel_rule [ ' srcs ' ] : for dep in bazel_rule [ ' srcs ' ] :
@ -146,12 +150,14 @@ def _extract_sources(bazel_rule):
return list ( sorted ( result ) ) return list ( sorted ( result ) )
def _extract_deps ( bazel_rule ) : def _extract_deps ( bazel_rule : BuildMetadata ,
bazel_rules : BuildDict ) - > List [ str ] :
""" Gets list of deps from from a bazel rule """ """ Gets list of deps from from a bazel rule """
return list ( sorted ( bazel_rule [ ' deps ' ] ) ) return list ( sorted ( bazel_rule [ ' deps ' ] ) )
def _create_target_from_bazel_rule ( target_name , bazel_rules ) : def _create_target_from_bazel_rule ( target_name : str ,
bazel_rules : BuildDict ) - > BuildMetadata :
""" Create build.yaml-like target definition from bazel metadata """ """ Create build.yaml-like target definition from bazel metadata """
bazel_rule = bazel_rules [ _get_bazel_label ( target_name ) ] bazel_rule = bazel_rules [ _get_bazel_label ( target_name ) ]
@ -164,71 +170,16 @@ def _create_target_from_bazel_rule(target_name, bazel_rules):
' _PUBLIC_HEADERS_BAZEL ' : _extract_public_headers ( bazel_rule ) , ' _PUBLIC_HEADERS_BAZEL ' : _extract_public_headers ( bazel_rule ) ,
' _HEADERS_BAZEL ' : _extract_nonpublic_headers ( bazel_rule ) , ' _HEADERS_BAZEL ' : _extract_nonpublic_headers ( bazel_rule ) ,
' _SRC_BAZEL ' : _extract_sources ( bazel_rule ) , ' _SRC_BAZEL ' : _extract_sources ( bazel_rule ) ,
' _DEPS_BAZEL ' : _extract_deps ( bazel_rule ) , ' _DEPS_BAZEL ' : _extract_deps ( bazel_rule , bazel_rules ) ,
' public_headers ' : bazel_rule [ ' _COLLAPSED_PUBLIC_HEADERS ' ] ,
' headers ' : bazel_rule [ ' _COLLAPSED_HEADERS ' ] ,
' src ' : bazel_rule [ ' _COLLAPSED_SRCS ' ] ,
' deps ' : bazel_rule [ ' _COLLAPSED_DEPS ' ] ,
} }
return result return result
def _sort_by_build_order ( lib_names , lib_dict , deps_key_name , verbose = False ) : def _external_dep_name_from_bazel_dependency ( bazel_dep : str ) - > Optional [ str ] :
""" Sort library names to form correct build order. Use metadata from lib_dict """
# we find correct build order by performing a topological sort
# expected output: if library B depends on A, A should be listed first
# all libs that are not in the dictionary are considered external.
external_deps = list (
sorted ( [ lib_name for lib_name in lib_names if lib_name not in lib_dict
] ) )
if verbose :
print ( ' topo_ordering ' + str ( lib_names ) )
print ( ' external_deps ' + str ( external_deps ) )
result = list ( external_deps ) # external deps will be listed first
while len ( result ) < len ( lib_names ) :
more_results = [ ]
for lib in lib_names :
if lib not in result :
dep_set = set ( lib_dict [ lib ] . get ( deps_key_name , [ ] ) )
dep_set = dep_set . intersection ( lib_names )
# if lib only depends on what's already built, add it to the results
if not dep_set . difference ( set ( result ) ) :
more_results . append ( lib )
if not more_results :
raise Exception (
' Cannot sort topologically, there seems to be a cyclic dependency '
)
if verbose :
print ( ' adding ' + str ( more_results ) )
result = result + list (
sorted ( more_results
) ) # when build order doesn't matter, sort lexicographically
return result
# TODO(jtattermusch): deduplicate with transitive_dependencies.py (which has a slightly different logic)
def _populate_transitive_deps ( bazel_rules ) :
""" Add ' transitive_deps ' field for each of the rules """
transitive_deps = { }
for rule_name in bazel_rules . keys ( ) :
transitive_deps [ rule_name ] = set ( bazel_rules [ rule_name ] [ ' deps ' ] )
while True :
deps_added = 0
for rule_name in bazel_rules . keys ( ) :
old_deps = transitive_deps [ rule_name ]
new_deps = set ( old_deps )
for dep_name in old_deps :
new_deps . update ( transitive_deps . get ( dep_name , set ( ) ) )
deps_added + = len ( new_deps ) - len ( old_deps )
transitive_deps [ rule_name ] = new_deps
# if none of the transitive dep sets has changed, we're done
if deps_added == 0 :
break
for rule_name , bazel_rule in bazel_rules . items ( ) :
bazel_rule [ ' transitive_deps ' ] = list ( sorted ( transitive_deps [ rule_name ] ) )
def _external_dep_name_from_bazel_dependency ( bazel_dep ) :
""" Returns name of dependency if external bazel dependency is provided or None """ """ Returns name of dependency if external bazel dependency is provided or None """
if bazel_dep . startswith ( ' @com_google_absl// ' ) : if bazel_dep . startswith ( ' @com_google_absl// ' ) :
# special case for add dependency on one of the absl libraries (there is not just one absl library) # special case for add dependency on one of the absl libraries (there is not just one absl library)
@ -247,98 +198,186 @@ def _external_dep_name_from_bazel_dependency(bazel_dep):
return None return None
def _expand_intermediate_deps ( target_dict , public_dep_names , bazel_rules ) : def _compute_transitive_metadata (
# Some of the libraries defined by bazel won't be exposed in build.yaml rule_name : str , bazel_rules : Any ,
# We call these "intermediate" dependencies. This method expands bazel_label_to_dep_name : Dict [ str , str ] ) - > None :
# the intermediate deps for given target (populates library's """ Computes the final build metadata for Bazel target with rule_name.
# headers, sources and dicts as if the intermediate dependency never existed)
The dependencies that will appear on the deps list are :
# use this dictionary to translate from bazel labels to dep names
bazel_label_to_dep_name = { } * Public build targets including binaries and tests ;
for dep_name in public_dep_names : * External targets , like absl , re2 .
bazel_label_to_dep_name [ _get_bazel_label ( dep_name ) ] = dep_name
All other intermediate dependencies will be merged , which means their
source file , headers , etc . will be collected into one build target . This
step of processing will greatly reduce the complexity of the generated
build specifications for other build systems , like CMake , Make , setuptools .
The final build metadata are :
* _TRANSITIVE_DEPS : all the transitive dependencies including intermediate
targets ;
* _COLLAPSED_DEPS : dependencies that fits our requirement above , and it
will remove duplicated items and produce the shortest
possible dependency list in alphabetical order ;
* _COLLAPSED_SRCS : the merged source files ;
* _COLLAPSED_PUBLIC_HEADERS : the merged public headers ;
* _COLLAPSED_HEADERS : the merged non - public headers ;
* _EXCLUDE_DEPS : intermediate targets to exclude when performing collapsing
of sources and dependencies .
For the collapsed_deps , the algorithm improved cases like :
The result in the past :
end2end_tests - > [ grpc_test_util , grpc , gpr , address_sorting , upb ]
grpc_test_util - > [ grpc , gpr , address_sorting , upb , . . . ]
grpc - > [ gpr , address_sorting , upb , . . . ]
The result of the algorithm :
end2end_tests - > [ grpc_test_util ]
grpc_test_util - > [ grpc ]
grpc - > [ gpr , address_sorting , upb , . . . ]
"""
bazel_rule = bazel_rules [ rule_name ]
direct_deps = _extract_deps ( bazel_rule , bazel_rules )
transitive_deps = set ( )
collapsed_deps = set ( )
exclude_deps = set ( )
collapsed_srcs = set ( _extract_sources ( bazel_rule ) )
collapsed_public_headers = set ( _extract_public_headers ( bazel_rule ) )
collapsed_headers = set ( _extract_nonpublic_headers ( bazel_rule ) )
for dep in direct_deps :
external_dep_name_maybe = _external_dep_name_from_bazel_dependency ( dep )
if dep in bazel_rules :
# Descend recursively, but no need to do that for external deps
if external_dep_name_maybe is None :
if " _PROCESSING_DONE " not in bazel_rules [ dep ] :
# This item is not processed before, compute now
_compute_transitive_metadata ( dep , bazel_rules ,
bazel_label_to_dep_name )
transitive_deps . update ( bazel_rules [ dep ] . get (
' _TRANSITIVE_DEPS ' , [ ] ) )
collapsed_deps . update (
collapsed_deps , bazel_rules [ dep ] . get ( ' _COLLAPSED_DEPS ' , [ ] ) )
exclude_deps . update ( bazel_rules [ dep ] . get ( ' _EXCLUDE_DEPS ' , [ ] ) )
# This dep is a public target, add it as a dependency
if dep in bazel_label_to_dep_name :
transitive_deps . update ( [ bazel_label_to_dep_name [ dep ] ] )
collapsed_deps . update ( collapsed_deps ,
[ bazel_label_to_dep_name [ dep ] ] )
# Add all the transitive deps of our every public dep to exclude
# list since we want to avoid building sources that are already
# built by our dependencies
exclude_deps . update ( bazel_rules [ dep ] [ ' _TRANSITIVE_DEPS ' ] )
continue
target_name = target_dict [ ' name ' ] # This dep is an external target, add it as a dependency
bazel_deps = target_dict [ ' _DEPS_BAZEL ' ] if external_dep_name_maybe is not None :
transitive_deps . update ( [ external_dep_name_maybe ] )
collapsed_deps . update ( collapsed_deps , [ external_dep_name_maybe ] )
continue
# initial values # Direct dependencies are part of transitive dependencies
public_headers = set ( target_dict [ ' _PUBLIC_HEADERS_BAZEL ' ] ) transitive_deps . update ( direct_deps )
headers = set ( target_dict [ ' _HEADERS_BAZEL ' ] )
src = set ( target_dict [ ' _SRC_BAZEL ' ] ) # Calculate transitive public deps (needed for collapsing sources)
deps = set ( ) transitive_public_deps = set (
filter ( lambda x : x in bazel_label_to_dep_name , transitive_deps ) )
# Remove intermediate targets that our public dependencies already depend
# on. This is the step that further shorten the deps list.
collapsed_deps = set ( filter ( lambda x : x not in exclude_deps ,
collapsed_deps ) )
# Compute the final source files and headers for this build target whose
# name is `rule_name` (input argument of this function).
#
# Imaging a public target PX has transitive deps [IA, IB, PY, IC, PZ]. PX,
# PY and PZ are public build targets. And IA, IB, IC are intermediate
# targets. In addition, PY depends on IC.
#
# Translate the condition into dependency graph:
# PX -> [IA, IB, PY, IC, PZ]
# PY -> [IC]
# Public targets: [PX, PY, PZ]
#
# The collapsed dependencies of PX: [PY, PZ].
# The excluded dependencies of X: [PY, IC, PZ].
# (IC is excluded as a dependency of PX. It is already included in PY, hence
# it would be redundant to include it again.)
#
# Target PX should include source files and headers of [PX, IA, IB] as final
# build metadata.
for dep in transitive_deps :
if dep not in exclude_deps and dep not in transitive_public_deps :
if dep in bazel_rules :
collapsed_srcs . update ( _extract_sources ( bazel_rules [ dep ] ) )
collapsed_public_headers . update (
_extract_public_headers ( bazel_rules [ dep ] ) )
collapsed_headers . update (
_extract_nonpublic_headers ( bazel_rules [ dep ] ) )
# This item is a "visited" flag
bazel_rule [ ' _PROCESSING_DONE ' ] = True
# Following items are described in the docstinrg.
bazel_rule [ ' _TRANSITIVE_DEPS ' ] = list ( sorted ( transitive_deps ) )
bazel_rule [ ' _COLLAPSED_DEPS ' ] = list ( sorted ( collapsed_deps ) )
bazel_rule [ ' _COLLAPSED_SRCS ' ] = list ( sorted ( collapsed_srcs ) )
bazel_rule [ ' _COLLAPSED_PUBLIC_HEADERS ' ] = list (
sorted ( collapsed_public_headers ) )
bazel_rule [ ' _COLLAPSED_HEADERS ' ] = list ( sorted ( collapsed_headers ) )
bazel_rule [ ' _EXCLUDE_DEPS ' ] = list ( sorted ( exclude_deps ) )
expansion_blocklist = set ( )
to_expand = set ( bazel_deps )
while to_expand :
# start with the last dependency to be built # TODO(jtattermusch): deduplicate with transitive_dependencies.py (which has a slightly different logic)
build_order = _sort_by_build_order ( list ( to_expand ) , bazel_rules , # TODO(jtattermusch): This is done to avoid introducing too many intermediate
' transitive_deps ' ) # libraries into the build.yaml-based builds (which might in cause issues
# building language-specific artifacts) and also because the libraries
# in build.yaml-based build are generally considered units of distributions
# (= public libraries that are visible to the user and are installable),
# while in bazel builds it is customary to define larger number of smaller
# "sublibraries". The need for elision (and expansion)
# of intermediate libraries can be re-evaluated in the future.
def _populate_transitive_metadata ( bazel_rules : Any ,
public_dep_names : Iterable [ str ] ) - > None :
""" Add ' transitive_deps ' field for each of the rules """
# Create the map between Bazel label and public dependency name
bazel_label_to_dep_name = { }
for dep_name in public_dep_names :
bazel_label_to_dep_name [ _get_bazel_label ( dep_name ) ] = dep_name
bazel_dep = build_order [ - 1 ] # Make sure we reached all the Bazel rules
to_expand . remove ( bazel_dep ) # TODO(lidiz) potentially we could only update a subset of rules
for rule_name in bazel_rules :
if ' _PROCESSING_DONE ' not in bazel_rules [ rule_name ] :
_compute_transitive_metadata ( rule_name , bazel_rules ,
bazel_label_to_dep_name )
is_public = bazel_dep in bazel_label_to_dep_name
external_dep_name_maybe = _external_dep_name_from_bazel_dependency (
bazel_dep )
if is_public : def update_test_metadata_with_transitive_metadata (
# this is not an intermediate dependency we so we add it all_extra_metadata : BuildDict , bazel_rules : BuildDict ) - > None :
# to the list of public dependencies to the list, in the right format """ Patches test build metadata with transitive metadata. """
deps . add ( bazel_label_to_dep_name [ bazel_dep ] ) for lib_name , lib_dict in all_extra_metadata . items ( ) :
# Skip if it isn't not an test
if lib_dict . get ( ' build ' ) != ' test ' or lib_dict . get ( ' _TYPE ' ) != ' target ' :
continue
# we do not want to expand any intermediate libraries that are already included bazel_rule = bazel_rules [ _get_bazel_label ( lib_name ) ]
# by the dependency we just added
expansion_blocklist . update (
bazel_rules [ bazel_dep ] [ ' transitive_deps ' ] )
elif external_dep_name_maybe : if ' //external:benchmark ' in bazel_rule [ ' _TRANSITIVE_DEPS ' ] :
deps . add ( external_dep_name_maybe ) lib_dict [ ' benchmark ' ] = True
lib_dict [ ' defaults ' ] = ' benchmark '
elif bazel_dep . startswith ( if ' //external:gtest ' in bazel_rule [ ' _TRANSITIVE_DEPS ' ] :
' //external: ' ) or not bazel_dep . startswith ( ' // ' ) : lib_dict [ ' gtest ' ] = True
# all the other external deps can be skipped lib_dict [ ' language ' ] = ' c++ '
pass
elif bazel_dep in expansion_blocklist :
# do not expand if a public dependency that depends on this has already been expanded
pass
else : def _generate_build_metadata ( build_extra_metadata : BuildDict ,
if bazel_dep in bazel_rules : bazel_rules : BuildDict ) - > BuildDict :
# this is an intermediate library, expand it
public_headers . update (
_extract_public_headers ( bazel_rules [ bazel_dep ] ) )
headers . update (
_extract_nonpublic_headers ( bazel_rules [ bazel_dep ] ) )
src . update ( _extract_sources ( bazel_rules [ bazel_dep ] ) )
new_deps = _extract_deps ( bazel_rules [ bazel_dep ] )
to_expand . update ( new_deps )
else :
raise Exception ( bazel_dep + ' not in bazel_rules ' )
# make the 'deps' field transitive, but only list non-intermediate deps and selected external deps
bazel_transitive_deps = bazel_rules [ _get_bazel_label (
target_name ) ] [ ' transitive_deps ' ]
for transitive_bazel_dep in bazel_transitive_deps :
public_name = bazel_label_to_dep_name . get ( transitive_bazel_dep , None )
if public_name :
deps . add ( public_name )
external_dep_name_maybe = _external_dep_name_from_bazel_dependency (
transitive_bazel_dep )
if external_dep_name_maybe :
# expanding all absl libraries is technically correct but creates too much noise
if not external_dep_name_maybe . startswith ( ' absl ' ) :
deps . add ( external_dep_name_maybe )
target_dict [ ' public_headers ' ] = list ( sorted ( public_headers ) )
target_dict [ ' headers ' ] = list ( sorted ( headers ) )
target_dict [ ' src ' ] = list ( sorted ( src ) )
target_dict [ ' deps ' ] = list ( sorted ( deps ) )
def _generate_build_metadata ( build_extra_metadata , bazel_rules ) :
""" Generate build metadata in build.yaml-like format bazel build metadata and build.yaml-specific " extra metadata " . """ """ Generate build metadata in build.yaml-like format bazel build metadata and build.yaml-specific " extra metadata " . """
lib_names = list ( build_extra_metadata . keys ( ) ) lib_names = list ( build_extra_metadata . keys ( ) )
result = { } result = { }
@ -346,20 +385,6 @@ def _generate_build_metadata(build_extra_metadata, bazel_rules):
for lib_name in lib_names : for lib_name in lib_names :
lib_dict = _create_target_from_bazel_rule ( lib_name , bazel_rules ) lib_dict = _create_target_from_bazel_rule ( lib_name , bazel_rules )
# Figure out the final list of headers and sources for given target.
# While this is mostly based on bazel build metadata, build.yaml does
# not necessarily expose all the targets that are present in bazel build.
# These "intermediate dependencies" might get flattened.
# TODO(jtattermusch): This is done to avoid introducing too many intermediate
# libraries into the build.yaml-based builds (which might in cause issues
# building language-specific artifacts) and also because the libraries
# in build.yaml-based build are generally considered units of distributions
# (= public libraries that are visible to the user and are installable),
# while in bazel builds it is customary to define larger number of smaller
# "sublibraries". The need for elision (and expansion)
# of intermediate libraries can be re-evaluated in the future.
_expand_intermediate_deps ( lib_dict , lib_names , bazel_rules )
# populate extra properties from the build.yaml-specific "extra metadata" # populate extra properties from the build.yaml-specific "extra metadata"
lib_dict . update ( build_extra_metadata . get ( lib_name , { } ) ) lib_dict . update ( build_extra_metadata . get ( lib_name , { } ) )
@ -376,8 +401,8 @@ def _generate_build_metadata(build_extra_metadata, bazel_rules):
if to_name : if to_name :
# store lib under the new name and also change its 'name' property # store lib under the new name and also change its 'name' property
if to_name in result : if to_name in result :
raise Exception ( ' Cannot rename target ' + lib_name + ' , ' + raise Exception ( ' Cannot rename target ' + str ( lib_name ) + ' , ' +
to_name + ' already exists. ' ) str ( to_name ) + ' already exists. ' )
lib_dict = result . pop ( lib_name ) lib_dict = result . pop ( lib_name )
lib_dict [ ' name ' ] = to_name lib_dict [ ' name ' ] = to_name
result [ to_name ] = lib_dict result [ to_name ] = lib_dict
@ -389,15 +414,10 @@ def _generate_build_metadata(build_extra_metadata, bazel_rules):
for dep in lib_dict_to_update [ ' deps ' ] for dep in lib_dict_to_update [ ' deps ' ]
] ) ] )
# make sure deps are listed in reverse topological order (e.g. "grpc gpr" and not "gpr grpc")
for lib_dict in result . values ( ) :
lib_dict [ ' deps ' ] = list (
reversed ( _sort_by_build_order ( lib_dict [ ' deps ' ] , result , ' deps ' ) ) )
return result return result
def _convert_to_build_yaml_like ( lib_dict ) : def _convert_to_build_yaml_like ( lib_dict : BuildMetadata ) - > BuildYaml :
lib_names = [ lib_names = [
lib_name for lib_name in list ( lib_dict . keys ( ) ) lib_name for lib_name in list ( lib_dict . keys ( ) )
if lib_dict [ lib_name ] . get ( ' _TYPE ' , ' library ' ) == ' library ' if lib_dict [ lib_name ] . get ( ' _TYPE ' , ' library ' ) == ' library '
@ -440,7 +460,7 @@ def _convert_to_build_yaml_like(lib_dict):
return build_yaml_like return build_yaml_like
def _extract_cc_tests ( bazel_rules ) : def _extract_cc_tests ( bazel_rules : BuildDict ) - > List [ str ] :
""" Gets list of cc_test tests from bazel rules """ """ Gets list of cc_test tests from bazel rules """
result = [ ] result = [ ]
for bazel_rule in bazel_rules . values ( ) : for bazel_rule in bazel_rules . values ( ) :
@ -452,7 +472,7 @@ def _extract_cc_tests(bazel_rules):
return list ( sorted ( result ) ) return list ( sorted ( result ) )
def _exclude_unwanted_cc_tests ( tests ) : def _exclude_unwanted_cc_tests ( tests : List [ str ] ) - > List [ str ] :
""" Filters out bazel tests that we don ' t want to run with other build systems or we cannot build them reasonably """ """ Filters out bazel tests that we don ' t want to run with other build systems or we cannot build them reasonably """
# most qps tests are autogenerated, we are fine without them # most qps tests are autogenerated, we are fine without them
@ -518,7 +538,8 @@ def _exclude_unwanted_cc_tests(tests):
return tests return tests
def _generate_build_extra_metadata_for_tests ( tests , bazel_rules ) : def _generate_build_extra_metadata_for_tests (
tests : List [ str ] , bazel_rules : BuildDict ) - > BuildDict :
""" For given tests, generate the " extra metadata " that we need for our " build.yaml " -like output. The extra metadata is generated from the bazel rule metadata by using a bunch of heuristics. """ """ For given tests, generate the " extra metadata " that we need for our " build.yaml " -like output. The extra metadata is generated from the bazel rule metadata by using a bunch of heuristics. """
test_metadata = { } test_metadata = { }
for test in tests : for test in tests :
@ -567,19 +588,11 @@ def _generate_build_extra_metadata_for_tests(tests, bazel_rules):
platforms . append ( ' windows ' ) platforms . append ( ' windows ' )
test_dict [ ' platforms ' ] = platforms test_dict [ ' platforms ' ] = platforms
if ' //external:benchmark ' in bazel_rule [ ' transitive_deps ' ] :
test_dict [ ' benchmark ' ] = True
test_dict [ ' defaults ' ] = ' benchmark '
cmdline_args = bazel_rule [ ' args ' ] cmdline_args = bazel_rule [ ' args ' ]
if cmdline_args : if cmdline_args :
test_dict [ ' args ' ] = list ( cmdline_args ) test_dict [ ' args ' ] = list ( cmdline_args )
uses_gtest = ' //external:gtest ' in bazel_rule [ ' transitive_deps ' ] if test . startswith ( ' test/cpp ' ) :
if uses_gtest :
test_dict [ ' gtest ' ] = True
if test . startswith ( ' test/cpp ' ) or uses_gtest :
test_dict [ ' language ' ] = ' c++ ' test_dict [ ' language ' ] = ' c++ '
elif test . startswith ( ' test/core ' ) : elif test . startswith ( ' test/core ' ) :
@ -615,7 +628,7 @@ def _generate_build_extra_metadata_for_tests(tests, bazel_rules):
return test_metadata return test_metadata
def _detect_and_print_issues ( build_yaml_like ) : def _detect_and_print_issues ( build_yaml_like : BuildYaml ) - > None :
""" Try detecting some unusual situations and warn about them. """ """ Try detecting some unusual situations and warn about them. """
for tgt in build_yaml_like [ ' targets ' ] : for tgt in build_yaml_like [ ' targets ' ] :
if tgt [ ' build ' ] == ' test ' : if tgt [ ' build ' ] == ' test ' :
@ -968,6 +981,8 @@ _BAZEL_DEPS_QUERIES = [
' deps( " //:all " ) ' , ' deps( " //:all " ) ' ,
' deps( " //src/compiler/... " ) ' , ' deps( " //src/compiler/... " ) ' ,
' deps( " //src/proto/... " ) ' , ' deps( " //src/proto/... " ) ' ,
# The ^ is needed to differentiate proto_library from go_proto_library
' deps(kind( " ^proto_library " , @envoy_api//envoy/...)) ' ,
] ]
# Step 1: run a bunch of "bazel query --output xml" queries to collect # Step 1: run a bunch of "bazel query --output xml" queries to collect
@ -987,14 +1002,6 @@ for query in _BAZEL_DEPS_QUERIES:
bazel_rules . update ( bazel_rules . update (
_extract_rules_from_bazel_xml ( _bazel_query_xml_tree ( query ) ) ) _extract_rules_from_bazel_xml ( _bazel_query_xml_tree ( query ) ) )
# Step 1a: Knowing the transitive closure of dependencies will make
# the postprocessing simpler, so compute the info for all our rules.
#
# Example:
# '//:grpc' : { ...,
# 'transitive_deps': ['//:gpr_base', ...] }
_populate_transitive_deps ( bazel_rules )
# Step 2: Extract the known bazel cc_test tests. While most tests # Step 2: Extract the known bazel cc_test tests. While most tests
# will be buildable with other build systems just fine, some of these tests # will be buildable with other build systems just fine, some of these tests
# would be too difficult to build and run with other build systems, # would be too difficult to build and run with other build systems,
@ -1049,7 +1056,25 @@ all_extra_metadata.update(_BUILD_EXTRA_METADATA)
all_extra_metadata . update ( all_extra_metadata . update (
_generate_build_extra_metadata_for_tests ( tests , bazel_rules ) ) _generate_build_extra_metadata_for_tests ( tests , bazel_rules ) )
# Step 4: Generate the final metadata for all the targets. # Step 4: Compute the build metadata that will be used in the final build.yaml.
# The final build metadata includes transitive dependencies, and sources/headers
# expanded without intermediate dependencies.
# Example:
# '//:grpc' : { ...,
# '_TRANSITIVE_DEPS': ['//:gpr_base', ...],
# '_COLLAPSED_DEPS': ['gpr', ...],
# '_COLLAPSED_SRCS': [...],
# '_COLLAPSED_PUBLIC_HEADERS': [...],
# '_COLLAPSED_HEADERS': [...]
# }
_populate_transitive_metadata ( bazel_rules , all_extra_metadata . keys ( ) )
# Step 4a: Update the existing test metadata with the updated build metadata.
# Certain build metadata of certain test targets depend on the transitive
# metadata that wasn't available earlier.
update_test_metadata_with_transitive_metadata ( all_extra_metadata , bazel_rules )
# Step 5: Generate the final metadata for all the targets.
# This is done by combining the bazel build metadata and the "extra metadata" # This is done by combining the bazel build metadata and the "extra metadata"
# we obtained in the previous step. # we obtained in the previous step.
# In this step, we also perform some interesting massaging of the target metadata # In this step, we also perform some interesting massaging of the target metadata
@ -1079,7 +1104,7 @@ all_extra_metadata.update(
# ... } # ... }
all_targets_dict = _generate_build_metadata ( all_extra_metadata , bazel_rules ) all_targets_dict = _generate_build_metadata ( all_extra_metadata , bazel_rules )
# Step 5 : convert the dictionary with all the targets to a dict that has # Step 6 : convert the dictionary with all the targets to a dict that has
# the desired "build.yaml"-like layout. # the desired "build.yaml"-like layout.
# TODO(jtattermusch): We use the custom "build.yaml"-like layout because # TODO(jtattermusch): We use the custom "build.yaml"-like layout because
# currently all other build systems use that format as their source of truth. # currently all other build systems use that format as their source of truth.
@ -1096,7 +1121,7 @@ build_yaml_like = _convert_to_build_yaml_like(all_targets_dict)
# detect and report some suspicious situations we've seen before # detect and report some suspicious situations we've seen before
_detect_and_print_issues ( build_yaml_like ) _detect_and_print_issues ( build_yaml_like )
# Step 6 : Store the build_autogenerated.yaml in a deterministic (=sorted) # Step 7 : Store the build_autogenerated.yaml in a deterministic (=sorted)
# and cleaned-up form. # and cleaned-up form.
# A basic overview of the resulting "build.yaml"-like format is here: # A basic overview of the resulting "build.yaml"-like format is here:
# https://github.com/grpc/grpc/blob/master/templates/README.md # https://github.com/grpc/grpc/blob/master/templates/README.md
Jan Tattermusch
4 years ago
committed by
GitHub