mirror of https://github.com/grpc/grpc.git
[build metadata] Bazel to "other build systems" improvements (#33803)
- Extract build metadata for some external dependencies from bazel build. This is achieved by letting extract_metadata_from_bazel_xml.py analyze some external libraries and sources. The logic is basically the same as for internal libraries, I only needed to teach extract_metadata_from_bazel_xml.py which external libraries it is allowed to analyze. * currently, the list of source files is automatically determined for `z`, `upb`, `re2` and `gtest` dependencies (at least for the case where we're building in "embedded" mode - e.g. mostly native extensions for python, php, ruby etc. - cmake has the ability to replace some of these dependencies by actual cmake dependency.) - Eliminate the need for manually written gen_build_yaml.py for some dependencies. - Make the info on target dependencies in build_autogenerated.yaml more accurate and complete. Until now, there were some depdendencies that were allowed to show up in build_autogenerated.yaml and some that were being skipped. This made generating the CMakeLists.txt and Makefile quite confusing (since some dependencies are being explicitly mentioned and some had to be assumed by the build system). - Overhaul the Makefile * the Makefile is currently only used internally (e.g. for ruby and PHP builds) * until now, the makefile wasn't really using the info about which targets depend on what libraries, but it was effectively hardcoding the depedendency data (by magically "knowing" what is the list of all the stuff that e.g. "grpc" depends on). * After the overhaul, the Makefile.template now actually looks at the library dependencies and uses them when generating the makefile. This gives a more correct and easier to maintain makefile. * since csharp is no longer on the master branch, remove all mentions of "csharp" targets in the Makefile. Other notable changes: - make extract_metadata_from_bazel_xml.py capable of resolving workspace bind() rules (so that it knows the real name of the target that is referred to as e.g. `//external:xyz`) TODO: - [DONE] ~~pkgconfig C++ distribtest~~ - [DONE} ~~update third_party/README to reflect changes in how some deps get updated now.~~ Planned followups: - cleanup naming of some targets in build metadata and buildsystem templates: libssl vs boringssl, ares vs cares etc. - further cleanup of Makefile - further cleanup of CMakeLists.txt - remote the need from manually hardcoding extra metadata for targets in build_autogenerated.yaml. Either add logic that determines the properties of targets automatically, or use metadata from bazel BUILD.pull/34079/head^2
Jan Tattermusch
1 year ago
committed by
GitHub
parent
4c3ff07dac
commit
a8508ec465
27 changed files with 3314 additions and 2887 deletions
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -1,48 +0,0 @@ |
||||
#!/usr/bin/env python2.7 |
||||
|
||||
# Copyright 2015 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
import os |
||||
import sys |
||||
import glob |
||||
import yaml |
||||
|
||||
os.chdir(os.path.dirname(sys.argv[0]) + "/../..") |
||||
|
||||
out = {} |
||||
|
||||
out["libs"] = [ |
||||
{ |
||||
# TODO @donnadionne: extracting the list of source files from bazel build to reduce duplication |
||||
"name": "re2", |
||||
"build": "private", |
||||
"language": "c", |
||||
"secure": False, |
||||
"src": sorted( |
||||
glob.glob("third_party/re2/re2/*.cc") |
||||
+ [ |
||||
"third_party/re2/util/pcre.cc", |
||||
"third_party/re2/util/rune.cc", |
||||
"third_party/re2/util/strutil.cc", |
||||
] |
||||
), |
||||
"headers": sorted( |
||||
glob.glob("third_party/re2/re2/*.h") |
||||
+ glob.glob("third_party/re2/util/*.h") |
||||
), |
||||
} |
||||
] |
||||
|
||||
print(yaml.dump(out)) |
||||
@ -1,199 +0,0 @@ |
||||
#!/usr/bin/env python2.7 |
||||
|
||||
# Copyright 2015 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# TODO: This should ideally be in upb submodule to avoid hardcoding this here. |
||||
|
||||
import re |
||||
import os |
||||
import sys |
||||
import yaml |
||||
|
||||
out = {} |
||||
|
||||
try: |
||||
out["libs"] = [ |
||||
{ |
||||
"name": "upb", |
||||
"build": "all", |
||||
"language": "c", |
||||
"src": [ |
||||
"third_party/utf8_range/naive.c", |
||||
"third_party/utf8_range/range2-neon.c", |
||||
"third_party/utf8_range/range2-sse.c", |
||||
"third_party/upb/upb/base/status.c", |
||||
"third_party/upb/upb/collections/array.c", |
||||
"third_party/upb/upb/collections/map_sorter.c", |
||||
"third_party/upb/upb/collections/map.c", |
||||
"third_party/upb/upb/hash/common.c", |
||||
"third_party/upb/upb/json/decode.c", |
||||
"third_party/upb/upb/json/encode.c", |
||||
"third_party/upb/upb/lex/atoi.c", |
||||
"third_party/upb/upb/lex/round_trip.c", |
||||
"third_party/upb/upb/lex/strtod.c", |
||||
"third_party/upb/upb/lex/unicode.c", |
||||
"third_party/upb/upb/mem/alloc.c", |
||||
"third_party/upb/upb/mem/arena.c", |
||||
"third_party/upb/upb/message/accessors.c", |
||||
"third_party/upb/upb/message/message.c", |
||||
"third_party/upb/upb/mini_table/common.c", |
||||
"third_party/upb/upb/mini_table/decode.c", |
||||
"third_party/upb/upb/mini_table/encode.c", |
||||
"third_party/upb/upb/mini_table/extension_registry.c", |
||||
"third_party/upb/upb/reflection/def_builder.c", |
||||
"third_party/upb/upb/reflection/def_pool.c", |
||||
"third_party/upb/upb/reflection/def_type.c", |
||||
"third_party/upb/upb/reflection/desc_state.c", |
||||
"third_party/upb/upb/reflection/enum_def.c", |
||||
"third_party/upb/upb/reflection/enum_reserved_range.c", |
||||
"third_party/upb/upb/reflection/enum_value_def.c", |
||||
"third_party/upb/upb/reflection/extension_range.c", |
||||
"third_party/upb/upb/reflection/field_def.c", |
||||
"third_party/upb/upb/reflection/file_def.c", |
||||
"third_party/upb/upb/reflection/message_def.c", |
||||
"third_party/upb/upb/reflection/message_reserved_range.c", |
||||
"third_party/upb/upb/reflection/message.c", |
||||
"third_party/upb/upb/reflection/method_def.c", |
||||
"third_party/upb/upb/reflection/oneof_def.c", |
||||
"third_party/upb/upb/reflection/service_def.c", |
||||
"third_party/upb/upb/text/encode.c", |
||||
"third_party/upb/upb/wire/decode_fast.c", |
||||
"third_party/upb/upb/wire/decode.c", |
||||
"third_party/upb/upb/wire/encode.c", |
||||
"third_party/upb/upb/wire/eps_copy_input_stream.c", |
||||
"third_party/upb/upb/wire/reader.c", |
||||
"src/core/ext/upb-generated/google/protobuf/descriptor.upb.c", |
||||
"src/core/ext/upbdefs-generated/google/protobuf/descriptor.upbdefs.c", |
||||
], |
||||
"headers": [ |
||||
"third_party/utf8_range/utf8_range.h", |
||||
"third_party/upb/upb/alloc.h", |
||||
"third_party/upb/upb/arena.h", |
||||
"third_party/upb/upb/array.h", |
||||
"third_party/upb/upb/base/descriptor_constants.h", |
||||
"third_party/upb/upb/base/log2.h", |
||||
"third_party/upb/upb/base/status.h", |
||||
"third_party/upb/upb/base/string_view.h", |
||||
"third_party/upb/upb/collections/array_internal.h", |
||||
"third_party/upb/upb/collections/array.h", |
||||
"third_party/upb/upb/collections/map_gencode_util.h", |
||||
"third_party/upb/upb/collections/map_internal.h", |
||||
"third_party/upb/upb/collections/map_sorter_internal.h", |
||||
"third_party/upb/upb/collections/map.h", |
||||
"third_party/upb/upb/collections/message_value.h", |
||||
"third_party/upb/upb/decode.h", |
||||
"third_party/upb/upb/def.h", |
||||
"third_party/upb/upb/def.hpp", |
||||
"third_party/upb/upb/encode.h", |
||||
"third_party/upb/upb/extension_registry.h", |
||||
"third_party/upb/upb/hash/common.h", |
||||
"third_party/upb/upb/hash/int_table.h", |
||||
"third_party/upb/upb/hash/str_table.h", |
||||
"third_party/upb/upb/json_decode.h", |
||||
"third_party/upb/upb/json_encode.h", |
||||
"third_party/upb/upb/json/decode.h", |
||||
"third_party/upb/upb/json/encode.h", |
||||
"third_party/upb/upb/lex/atoi.h", |
||||
"third_party/upb/upb/lex/round_trip.h", |
||||
"third_party/upb/upb/lex/strtod.h", |
||||
"third_party/upb/upb/lex/unicode.h", |
||||
"third_party/upb/upb/map.h", |
||||
"third_party/upb/upb/mem/alloc.h", |
||||
"third_party/upb/upb/mem/arena_internal.h", |
||||
"third_party/upb/upb/mem/arena.h", |
||||
"third_party/upb/upb/message/accessors_internal.h", |
||||
"third_party/upb/upb/message/accessors.h", |
||||
"third_party/upb/upb/message/extension_internal.h", |
||||
"third_party/upb/upb/message/internal.h", |
||||
"third_party/upb/upb/message/message.h", |
||||
"third_party/upb/upb/mini_table.h", |
||||
"third_party/upb/upb/mini_table/common_internal.h", |
||||
"third_party/upb/upb/mini_table/common.h", |
||||
"third_party/upb/upb/mini_table/decode.h", |
||||
"third_party/upb/upb/mini_table/encode_internal.h", |
||||
"third_party/upb/upb/mini_table/encode_internal.hpp", |
||||
"third_party/upb/upb/mini_table/enum_internal.h", |
||||
"third_party/upb/upb/mini_table/extension_internal.h", |
||||
"third_party/upb/upb/mini_table/extension_registry.h", |
||||
"third_party/upb/upb/mini_table/field_internal.h", |
||||
"third_party/upb/upb/mini_table/file_internal.h", |
||||
"third_party/upb/upb/mini_table/message_internal.h", |
||||
"third_party/upb/upb/mini_table/sub_internal.h", |
||||
"third_party/upb/upb/mini_table/types.h", |
||||
"third_party/upb/upb/msg.h", |
||||
"third_party/upb/upb/port/atomic.h", |
||||
"third_party/upb/upb/port/def.inc", |
||||
"third_party/upb/upb/port/undef.inc", |
||||
"third_party/upb/upb/port/vsnprintf_compat.h", |
||||
"third_party/upb/upb/reflection.h", |
||||
"third_party/upb/upb/reflection.hpp", |
||||
"third_party/upb/upb/reflection/common.h", |
||||
"third_party/upb/upb/reflection/def_builder_internal.h", |
||||
"third_party/upb/upb/reflection/def_pool_internal.h", |
||||
"third_party/upb/upb/reflection/def_pool.h", |
||||
"third_party/upb/upb/reflection/def_type.h", |
||||
"third_party/upb/upb/reflection/def.h", |
||||
"third_party/upb/upb/reflection/def.hpp", |
||||
"third_party/upb/upb/reflection/desc_state_internal.h", |
||||
"third_party/upb/upb/reflection/enum_def_internal.h", |
||||
"third_party/upb/upb/reflection/enum_def.h", |
||||
"third_party/upb/upb/reflection/enum_reserved_range_internal.h", |
||||
"third_party/upb/upb/reflection/enum_reserved_range.h", |
||||
"third_party/upb/upb/reflection/enum_value_def_internal.h", |
||||
"third_party/upb/upb/reflection/enum_value_def.h", |
||||
"third_party/upb/upb/reflection/extension_range_internal.h", |
||||
"third_party/upb/upb/reflection/extension_range.h", |
||||
"third_party/upb/upb/reflection/field_def_internal.h", |
||||
"third_party/upb/upb/reflection/field_def.h", |
||||
"third_party/upb/upb/reflection/file_def_internal.h", |
||||
"third_party/upb/upb/reflection/file_def.h", |
||||
"third_party/upb/upb/reflection/message_def_internal.h", |
||||
"third_party/upb/upb/reflection/message_def.h", |
||||
"third_party/upb/upb/reflection/message_reserved_range_internal.h", |
||||
"third_party/upb/upb/reflection/message_reserved_range.h", |
||||
"third_party/upb/upb/reflection/message.h", |
||||
"third_party/upb/upb/reflection/message.hpp", |
||||
"third_party/upb/upb/reflection/method_def_internal.h", |
||||
"third_party/upb/upb/reflection/method_def.h", |
||||
"third_party/upb/upb/reflection/oneof_def_internal.h", |
||||
"third_party/upb/upb/reflection/oneof_def.h", |
||||
"third_party/upb/upb/reflection/service_def_internal.h", |
||||
"third_party/upb/upb/reflection/service_def.h", |
||||
"third_party/upb/upb/status.h", |
||||
"third_party/upb/upb/string_view.h", |
||||
"third_party/upb/upb/text_encode.h", |
||||
"third_party/upb/upb/text/encode.h", |
||||
"third_party/upb/upb/upb.h", |
||||
"third_party/upb/upb/upb.hpp", |
||||
"third_party/upb/upb/wire/common_internal.h", |
||||
"third_party/upb/upb/wire/common.h", |
||||
"third_party/upb/upb/wire/decode_fast.h", |
||||
"third_party/upb/upb/wire/decode_internal.h", |
||||
"third_party/upb/upb/wire/decode.h", |
||||
"third_party/upb/upb/wire/encode.h", |
||||
"third_party/upb/upb/wire/eps_copy_input_stream.h", |
||||
"third_party/upb/upb/wire/reader.h", |
||||
"third_party/upb/upb/wire/swap_internal.h", |
||||
"third_party/upb/upb/wire/types.h", |
||||
"src/core/ext/upb-generated/google/protobuf/descriptor.upb.h", |
||||
"src/core/ext/upbdefs-generated/google/protobuf/descriptor.upbdefs.h", |
||||
], |
||||
"secure": False, |
||||
} |
||||
] |
||||
except: |
||||
pass |
||||
|
||||
print(yaml.dump(out)) |
||||
@ -1,58 +0,0 @@ |
||||
#!/usr/bin/env python2.7 |
||||
|
||||
# Copyright 2015 gRPC authors. |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
import re |
||||
import os |
||||
import sys |
||||
import yaml |
||||
|
||||
os.chdir(os.path.dirname(sys.argv[0]) + "/../..") |
||||
|
||||
out = {} |
||||
|
||||
try: |
||||
with open("third_party/zlib/CMakeLists.txt") as f: |
||||
cmake = f.read() |
||||
|
||||
def cmpath(x): |
||||
return "third_party/zlib/%s" % x.replace( |
||||
"${CMAKE_CURRENT_BINARY_DIR}/", "" |
||||
) |
||||
|
||||
def cmvar(name): |
||||
regex = r"set\(\s*" |
||||
regex += name |
||||
regex += r"([^)]*)\)" |
||||
return [cmpath(x) for x in re.search(regex, cmake).group(1).split()] |
||||
|
||||
out["libs"] = [ |
||||
{ |
||||
"name": "z", |
||||
"zlib": True, |
||||
"defaults": "zlib", |
||||
"build": "private", |
||||
"language": "c", |
||||
"secure": False, |
||||
"src": sorted(cmvar("ZLIB_SRCS")), |
||||
"headers": sorted( |
||||
cmvar("ZLIB_PUBLIC_HDRS") + cmvar("ZLIB_PRIVATE_HDRS") |
||||
), |
||||
} |
||||
] |
||||
except: |
||||
pass |
||||
|
||||
print(yaml.dump(out)) |
||||
Loading…
Reference in new issue