From 7ad674ba046290d94f4021d98d943779b1fab6f4 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Fri, 2 Feb 2024 16:55:56 -0800 Subject: [PATCH] [Build] Workaround for protobuf-at-head test (#35798) - Updated `third_party/utf8_range` to 25.1 to be aligned with other protobuf (protobuf, upb, and utf8_range need to be updated at the same time as they're in the same github repo) - Added a workaround to `extract_metadata_from_bazel_xml.py` to handle `utf8_range` migration. It used to be `@utf8_range//:utf8_range` and it will be `@com_google_protobuf//third_party/utf8_range:utf8_range` from the next Protobuf release. So the script needs this hack to support both. Closes #35798 PiperOrigin-RevId: 603822623 --- .../.github/workflows/bazel_tests.yml | 9 ++++++- third_party/utf8_range/BUILD.bazel | 7 +++++ third_party/utf8_range/CMakeLists.txt | 5 ++++ .../utf8_range/cmake/utf8_range.pc.cmake | 11 ++++++++ third_party/utf8_range/utf8_validity.cc | 2 ++ .../extract_metadata_from_bazel_xml.py | 27 +++++++++++++------ 6 files changed, 52 insertions(+), 9 deletions(-) create mode 100644 third_party/utf8_range/cmake/utf8_range.pc.cmake diff --git a/third_party/utf8_range/.github/workflows/bazel_tests.yml b/third_party/utf8_range/.github/workflows/bazel_tests.yml index ae3e970419c..adf711859aa 100644 --- a/third_party/utf8_range/.github/workflows/bazel_tests.yml +++ b/third_party/utf8_range/.github/workflows/bazel_tests.yml @@ -29,8 +29,15 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Setup Bazel + run: | + echo "BAZEL=bazel" >> $GITHUB_ENV + echo "USE_BAZEL_VERSION=5.1.1" >> $GITHUB_ENV + - name: Setup Bazel (macOS) + if: runner.os == 'macOS' + run: echo "BAZEL=bazelisk" >> $GITHUB_ENV - name: Install dependencies run: sudo apt update && sudo apt install -y ${{ matrix.install }} if: matrix.install != '' - name: Run tests - run: cd ${{ github.workspace }} && CC=${{ matrix.CC }} bazel test --test_output=errors ... ${{ matrix.flags }} + run: cd ${{ github.workspace }} && CC=${{ matrix.CC }} ${{ env.BAZEL }} test --test_output=errors ... ${{ matrix.flags }} diff --git a/third_party/utf8_range/BUILD.bazel b/third_party/utf8_range/BUILD.bazel index 8582f1bdc48..b9ba04e17c6 100644 --- a/third_party/utf8_range/BUILD.bazel +++ b/third_party/utf8_range/BUILD.bazel @@ -4,6 +4,12 @@ # license that can be found in the LICENSE file or at # https://opensource.org/licenses/MIT. +DEFAULT_VISIBILITY = [ + "//third_party/protobuf:__subpackages__", + "//:__subpackages__", + "//util/utf8/internal:__subpackages__", +] + package(default_visibility = ["//visibility:public"]) licenses(["notice"]) @@ -21,6 +27,7 @@ filegroup( ], visibility = [ "@com_google_protobuf//:__subpackages__", + "@upb//:__subpackages__", ], ) diff --git a/third_party/utf8_range/CMakeLists.txt b/third_party/utf8_range/CMakeLists.txt index 69a4d899077..344952d38cf 100644 --- a/third_party/utf8_range/CMakeLists.txt +++ b/third_party/utf8_range/CMakeLists.txt @@ -73,6 +73,11 @@ if (utf8_range_ENABLE_INSTALL) install(FILES "${PROJECT_BINARY_DIR}/${PROJECT_NAME}-config.cmake" DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}" ) + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/cmake/utf8_range.pc.cmake + ${CMAKE_CURRENT_BINARY_DIR}/utf8_range.pc @ONLY) + install( + FILES ${CMAKE_CURRENT_BINARY_DIR}/utf8_range.pc + DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") # Install public headers explicitly. install(FILES utf8_range.h utf8_validity.h diff --git a/third_party/utf8_range/cmake/utf8_range.pc.cmake b/third_party/utf8_range/cmake/utf8_range.pc.cmake new file mode 100644 index 00000000000..64b4b770b29 --- /dev/null +++ b/third_party/utf8_range/cmake/utf8_range.pc.cmake @@ -0,0 +1,11 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=@CMAKE_INSTALL_PREFIX@ +libdir=@CMAKE_INSTALL_FULL_LIBDIR@ +includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ + +Name: UTF8 Range +Description: Google's UTF8 Library +Version: 1.0 +Requires: absl_strings +Libs: -L${libdir} -lutf8_validity -lutf8_range @CMAKE_THREAD_LIBS_INIT@ +Cflags: -I${includedir} diff --git a/third_party/utf8_range/utf8_validity.cc b/third_party/utf8_range/utf8_validity.cc index db811993f81..9e945766732 100644 --- a/third_party/utf8_range/utf8_validity.cc +++ b/third_party/utf8_range/utf8_validity.cc @@ -133,6 +133,7 @@ size_t ValidUTF8Span(const char* data, const char* end) { return err_pos + (1 - ReturnPosition); } +#ifdef __SSE4_1__ /* Returns the number of bytes needed to skip backwards to get to the first byte of codepoint. */ @@ -148,6 +149,7 @@ inline int CodepointSkipBackwards(int32_t codepoint_word) { } return 0; } +#endif // __SSE4_1__ /* Skipping over ASCII as much as possible, per 8 bytes. It is intentional as most strings to check for validity consist only of 1 byte codepoints. diff --git a/tools/buildgen/extract_metadata_from_bazel_xml.py b/tools/buildgen/extract_metadata_from_bazel_xml.py index 2a289c5e2bb..23c41909ff4 100755 --- a/tools/buildgen/extract_metadata_from_bazel_xml.py +++ b/tools/buildgen/extract_metadata_from_bazel_xml.py @@ -99,11 +99,13 @@ EXTERNAL_PROTO_LIBRARIES = { # For that we need mapping from external repo name to a corresponding # path to a git submodule. EXTERNAL_SOURCE_PREFIXES = { - "@utf8_range": "third_party/utf8_range", - "@com_googlesource_code_re2": "third_party/re2", - "@com_google_googletest": "third_party/googletest", - "@com_google_protobuf": "third_party/upb", - "@zlib": "third_party/zlib", + # TODO(veblush): Remove @utf8_range// item once protobuf is upgraded to 26.x + "@utf8_range//": "third_party/utf8_range", + "@com_googlesource_code_re2//": "third_party/re2", + "@com_google_googletest//": "third_party/googletest", + "@com_google_protobuf//upb": "third_party/upb/upb", + "@com_google_protobuf//third_party/utf8_range": "third_party/utf8_range", + "@zlib//": "third_party/zlib", } @@ -209,9 +211,9 @@ def _try_extract_source_file_path(label: str) -> str: # This is an external source file. We are only interested in sources # for some of the external libraries. for lib_name, prefix in EXTERNAL_SOURCE_PREFIXES.items(): - if label.startswith(lib_name + "//"): + if label.startswith(lib_name): return ( - label.replace("%s//" % lib_name, prefix + "/") + label.replace("%s" % lib_name, prefix) .replace(":", "/") .replace("//", "/") ) @@ -1100,7 +1102,7 @@ _BUILD_EXTRA_METADATA = { "build": "all", "_RENAME": "upb_textformat_lib", }, - "@utf8_range//:utf8_range": { + "@com_google_protobuf//third_party/utf8_range:utf8_range": { "language": "c", "build": "all", # rename to utf8_range_lib is necessary for now to avoid clash with utf8_range target in protobuf's cmake @@ -1400,6 +1402,15 @@ tests = _exclude_unwanted_cc_tests(_extract_cc_tests(bazel_rules)) # only very little "extra metadata" would be needed and/or it would be trivial # to generate it automatically. all_extra_metadata = {} +# TODO(veblush): Remove this workaround once protobuf is upgraded to 26.x +if "@com_google_protobuf//third_party/utf8_range:utf8_range" not in bazel_rules: + md = _BUILD_EXTRA_METADATA[ + "@com_google_protobuf//third_party/utf8_range:utf8_range" + ] + del _BUILD_EXTRA_METADATA[ + "@com_google_protobuf//third_party/utf8_range:utf8_range" + ] + _BUILD_EXTRA_METADATA["@utf8_range//:utf8_range"] = md all_extra_metadata.update(_BUILD_EXTRA_METADATA) all_extra_metadata.update( _generate_build_extra_metadata_for_tests(tests, bazel_rules)