Move utf8_range into the protobuf repo

This makes third_party/utf8_range no longer a Git subtree, but instead the
permanent location and source of truth for utf8_range. It is also now
incorporated into the @com_google_protobuf Bazel repo. Utf8_range still has its
own separate CMake build for now, though.

PiperOrigin-RevId: 580682733
pull/14693/head
Adam Cozzette 1 year ago committed by Copybara-Service
parent e6f8b9d102
commit 5ca84008fc
  1. 12
      .github/workflows/test_cpp.yml
  2. 4
      WORKSPACE
  3. 12
      php/BUILD.bazel
  4. 8
      pkg/cc_dist_library.bzl
  5. 8
      protobuf_deps.bzl
  6. 2
      python/BUILD.bazel
  7. 2
      python/dist/BUILD.bazel
  8. 15
      ruby/BUILD.bazel
  9. 6
      ruby/Rakefile
  10. 4
      ruby/ext/google/protobuf_c/BUILD.bazel
  11. 4
      src/google/protobuf/BUILD.bazel
  12. 2
      src/google/protobuf/json/BUILD.bazel
  13. 13
      third_party/utf8_range/BUILD.bazel
  14. 31
      third_party/utf8_range/CONTRIBUTING.md
  15. 48
      third_party/utf8_range/WORKSPACE
  16. 2
      third_party/utf8_range/fuzz/BUILD.bazel
  17. 0
      third_party/utf8_range/github/.bazelrc
  18. 0
      third_party/utf8_range/github/.github/workflows/bazel_tests.yml
  19. 0
      third_party/utf8_range/github/.github/workflows/cmake_tests.yml
  20. 0
      third_party/utf8_range/github/.gitignore
  21. 25
      third_party/utf8_range/rules_fuzzing.patch
  22. 2
      third_party/utf8_range/utf8_validity_test.cc
  23. 11
      third_party/utf8_range/workspace_deps.bzl
  24. 6
      upb/BUILD
  25. 2
      upb/cmake/BUILD.bazel
  26. 6
      upb/cmake/make_cmakelists.py
  27. 2
      upb/wire/BUILD
  28. 8
      update_subtrees.sh

@ -27,14 +27,14 @@ jobs:
include:
# Set defaults
- image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize@sha256:04cd765285bc52cbbf51d66c8c66d8603579cf0f19cc42df26b09d2c270541fb
- targets: //pkg/... //src/... @com_google_protobuf_examples//... @utf8_range//...
- targets: //pkg/... //src/... @com_google_protobuf_examples//... //third_party/utf8_range/...
# Override cases with custom images
- config: { name: "TCMalloc" }
image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/tcmalloc@sha256:bd39119d74b8a3fad4ae335d4cf5294e70384676331b7e19949459fc7a8d8328"
targets: "//src/... @utf8_range//..."
targets: "//src/... //third_party/utf8_range/..."
- config: { name: "aarch64" }
targets: "//src/... //src/google/protobuf/compiler:protoc_aarch64_test @utf8_range//..."
targets: "//src/... //src/google/protobuf/compiler:protoc_aarch64_test //third_party/utf8_range/..."
image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-63dd26c0c7a808d92673a3e52e848189d4ab0f17"
name: Linux ${{ matrix.config.name }}
runs-on: ${{ matrix.config.runner || 'ubuntu-latest' }}
@ -69,7 +69,7 @@ jobs:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/gcc:${{ matrix.version }}-6.3.0-63dd26c0c7a808d92673a3e52e848189d4ab0f17
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: cpp_linux/gcc-${{ matrix.version }}
bazel: test //pkg/... //src/... @com_google_protobuf_examples//... @utf8_range//...
bazel: test //pkg/... //src/... @com_google_protobuf_examples//... //third_party/utf8_range/...
linux-release:
strategy:
@ -316,13 +316,13 @@ jobs:
- name: MacOS
os: macos-12
cache_key: macos-12
bazel: test //src/... @utf8_range//...
bazel: test //src/... //third_party/utf8_range/...
- name: MacOS Apple Silicon (build only)
os: macos-12
cache_key: macos-12-arm
# Current github runners are all Intel based, so just build/compile
# for Apple Silicon to detect issues there.
bazel: build --cpu=darwin_arm64 //src/... @utf8_range//...
bazel: build --cpu=darwin_arm64 //src/... //third_party/utf8_range/...
- name: Windows
os: windows-2022
cache_key: windows-2022

@ -161,10 +161,6 @@ load("@pip_deps//:requirements.bzl", "install_deps")
install_deps()
load("@utf8_range//:workspace_deps.bzl", "utf8_range_deps")
utf8_range_deps()
http_archive(
name = "rules_fuzzing",
sha256 = "ff52ef4845ab00e95d29c02a9e32e9eff4e0a4c9c8a6bcf8407a2f19eb3f9190",

@ -4,10 +4,10 @@
load("@rules_pkg//:mappings.bzl", "pkg_filegroup", "pkg_files", "strip_prefix")
load("@rules_pkg//:pkg.bzl", "pkg_tar")
load("//upb/cmake:build_defs.bzl", "staleness_test")
load("//:protobuf_version.bzl", "PROTOBUF_PHP_VERSION", "PROTOC_VERSION")
load("//build_defs:internal_shell.bzl", "inline_sh_binary")
load("//conformance:defs.bzl", "conformance_test")
load("//:protobuf_version.bzl", "PROTOBUF_PHP_VERSION", "PROTOC_VERSION")
load("//upb/cmake:build_defs.bzl", "staleness_test")
filegroup(
name = "source_files",
@ -32,7 +32,7 @@ inline_sh_binary(
name = "build_extension",
cmd = """
mkdir -p php/ext/google/protobuf/third_party/utf8_range
cp external/utf8_range/* php/ext/google/protobuf/third_party/utf8_range
cp third_party/utf8_range/* php/ext/google/protobuf/third_party/utf8_range
pushd php/ext/google/protobuf
phpize
@ -46,7 +46,7 @@ genrule(
name = "extension",
srcs = [
":source_files",
"@utf8_range//:utf8_range_srcs",
"//third_party/utf8_range:utf8_range_srcs",
],
outs = ["protobuf.so"],
cmd = """
@ -157,8 +157,8 @@ pkg_files(
pkg_files(
name = "utf8_range_files",
srcs = [
"@utf8_range//:LICENSE",
"@utf8_range//:utf8_range_srcs",
"//third_party/utf8_range:LICENSE",
"//third_party/utf8_range:utf8_range_srcs",
],
prefix = "third_party/utf8_range",
)

@ -137,9 +137,11 @@ def _flatten_target_files(targets):
return depset(transitive = [
target.files
for target in targets
# Filter out targets from external workspaces
if target.label.workspace_name == "" or
target.label.workspace_name == "com_google_protobuf"
# Filter out targets from external workspaces. We also filter out
# utf8_range since that has a separate CMake build for now.
if (target.label.workspace_name == "" or
target.label.workspace_name == "com_google_protobuf") and
not target.label.package.startswith("third_party/utf8_range")
])
def _get_transitive_sources(targets, attr, deps):

@ -68,14 +68,6 @@ def protobuf_deps():
build_file = Label("//:third_party/jsoncpp.BUILD"),
)
if not native.existing_rule("utf8_range"):
_github_archive(
name = "utf8_range",
repo = "https://github.com/protocolbuffers/utf8_range",
commit = "0e378bdb224cc8d4653b0db963b474839c2bb43c",
sha256 = "9b8db33325e5b6105c04369cd04dbbd9186e0577141543714243e66b8f045664",
)
if not native.existing_rule("rules_cc"):
_github_archive(
name = "rules_cc",

@ -203,6 +203,6 @@ py_extension(
"//upb/util:compare",
"//upb/util:def_to_proto",
"//upb/util:required_fields",
"@utf8_range",
"//third_party/utf8_range",
],
)

@ -217,7 +217,7 @@ pkg_files(
pkg_files(
name = "utf8_range_source_files",
srcs = ["@utf8_range//:utf8_range_srcs"],
srcs = ["//third_party/utf8_range:utf8_range_srcs"],
prefix = "utf8_range",
)

@ -6,10 +6,9 @@ load("@bazel_skylib//lib:selects.bzl", "selects")
load("@bazel_skylib//rules:common_settings.bzl", "string_flag")
load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix")
load("@rules_ruby//ruby:defs.bzl", "ruby_library")
load("//build_defs:internal_shell.bzl", "inline_sh_binary")
load("//:protobuf.bzl", "internal_ruby_proto_library")
load("//conformance:defs.bzl", "conformance_test")
load("//:protobuf_version.bzl", "PROTOBUF_RUBY_VERSION")
load("//conformance:defs.bzl", "conformance_test")
################################################################################
# Ruby Runtime
@ -100,8 +99,8 @@ ruby_library(
genrule(
name = "jruby_release",
srcs = [
"@utf8_range//:utf8_range_srcs",
"@utf8_range//:LICENSE",
"//third_party/utf8_range:utf8_range_srcs",
"//third_party/utf8_range:LICENSE",
"//ruby/lib/google:copy_jar",
"//ruby/lib/google:dist_files",
"//ruby/ext/google/protobuf_c:dist_files",
@ -116,7 +115,7 @@ genrule(
cp --parents -L "$$src" tmp
done
mkdir -p "tmp/ruby/ext/google/protobuf_c/third_party/utf8_range"
for utf in $(execpaths @utf8_range//:utf8_range_srcs) $(execpath @utf8_range//:LICENSE); do
for utf in $(execpaths //third_party/utf8_range:utf8_range_srcs) $(execpath //third_party/utf8_range:LICENSE); do
mv "tmp/$$utf" "tmp/ruby/ext/google/protobuf_c/third_party/utf8_range"
done
for wkt in $(execpaths //:well_known_ruby_protos); do
@ -139,8 +138,8 @@ genrule(
genrule(
name = "ruby_release",
srcs = [
"@utf8_range//:utf8_range_srcs",
"@utf8_range//:LICENSE",
"//third_party/utf8_range:utf8_range_srcs",
"//third_party/utf8_range:LICENSE",
"//:well_known_ruby_protos",
"//ruby/ext/google/protobuf_c:dist_files",
"//ruby/lib/google:dist_files",
@ -154,7 +153,7 @@ genrule(
cp --parents -L "$$src" "tmp"
done
mkdir -p "tmp/ruby/ext/google/protobuf_c/third_party/utf8_range"
for utf in $(execpaths @utf8_range//:utf8_range_srcs) $(execpath @utf8_range//:LICENSE); do
for utf in $(execpaths //third_party/utf8_range:utf8_range_srcs) $(execpath //third_party/utf8_range:LICENSE); do
mv "tmp/$$utf" "tmp/ruby/ext/google/protobuf_c/third_party/utf8_range"
done
for wkt in $(execpaths //:well_known_ruby_protos); do

@ -73,11 +73,7 @@ task :copy_third_party do
unless File.exist? 'ext/google/protobuf_c/third_party/utf8_range'
FileUtils.mkdir_p 'ext/google/protobuf_c/third_party/utf8_range'
# We need utf8_range in-tree.
if ENV['BAZEL'] == 'true'
utf8_root = '../external/utf8_range'
else
utf8_root = '../third_party/utf8_range'
end
utf8_root = '../third_party/utf8_range'
%w[
utf8_range.h naive.c range2-neon.c range2-neon.c range2-sse.c LICENSE
].each do |file|

@ -53,8 +53,8 @@ cc_library(
"//conditions:default": [],
}),
deps = [
"//third_party/utf8_range",
"@rules_ruby//ruby/runtime:headers",
"@utf8_range",
],
alwayslink = True,
)
@ -79,7 +79,7 @@ cc_library(
"//conditions:default": [],
}),
deps = [
"@utf8_range",
"//third_party/utf8_range",
],
)

@ -437,6 +437,7 @@ cc_library(
":varint_shuffle",
"//src/google/protobuf/io",
"//src/google/protobuf/stubs:lite",
"//third_party/utf8_range:utf8_validity",
"@com_google_absl//absl/base",
"@com_google_absl//absl/container:btree",
"@com_google_absl//absl/container:flat_hash_set",
@ -448,7 +449,6 @@ cc_library(
"@com_google_absl//absl/strings:internal",
"@com_google_absl//absl/synchronization",
"@com_google_absl//absl/time",
"@utf8_range//:utf8_validity",
],
)
@ -522,6 +522,7 @@ cc_library(
"//src/google/protobuf/io:printer",
"//src/google/protobuf/io:tokenizer",
"//src/google/protobuf/stubs",
"//third_party/utf8_range:utf8_validity",
"@com_google_absl//absl/base",
"@com_google_absl//absl/base:dynamic_annotations",
"@com_google_absl//absl/container:btree",
@ -533,7 +534,6 @@ cc_library(
"@com_google_absl//absl/strings:internal",
"@com_google_absl//absl/synchronization",
"@com_google_absl//absl/time",
"@utf8_range//:utf8_validity",
],
)

@ -96,6 +96,7 @@ cc_library(
"//src/google/protobuf:protobuf_lite",
"//src/google/protobuf/io",
"//src/google/protobuf/util:type_resolver_util",
"//third_party/utf8_range:utf8_validity",
"@com_google_absl//absl/container:flat_hash_map",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
@ -103,7 +104,6 @@ cc_library(
"@com_google_absl//absl/types:optional",
"@com_google_absl//absl/types:span",
"@com_google_absl//absl/types:variant",
"@utf8_range//:utf8_validity",
],
)

@ -5,8 +5,8 @@
# https://opensource.org/licenses/MIT.
DEFAULT_VISIBILITY = [
"//third_party/protobuf:__subpackages__",
"//:__subpackages__",
"//src/google/protobuf:__subpackages__",
"//third_party/utf8_range:__subpackages__",
"//util/utf8/internal:__subpackages__",
]
@ -19,7 +19,7 @@ exports_files([
"rules_fuzzing.patch",
])
# TODO(b/252876197) Remove this once callers have been Bazelified.
# TODO Remove this once callers have been Bazelified.
filegroup(
name = "utf8_range_srcs",
srcs = [
@ -28,10 +28,7 @@ filegroup(
"range2-sse.c",
"utf8_range.h",
],
visibility = [
"@com_google_protobuf//:__subpackages__",
"@upb//:__subpackages__",
],
visibility = ["//:__subpackages__"],
)
cc_library(
@ -42,12 +39,14 @@ cc_library(
"range2-sse.c",
],
hdrs = ["utf8_range.h"],
strip_include_prefix = "/third_party/utf8_range",
)
cc_library(
name = "utf8_validity",
srcs = ["utf8_validity.cc"],
hdrs = ["utf8_validity.h"],
strip_include_prefix = "/third_party/utf8_range",
deps = [
"@com_google_absl//absl/strings",
],

@ -1,31 +0,0 @@
# How to Contribute
This repository is currently a read-only clone of internal Google code for use
in open-source projects. We don't currently have a mechanism to upstream
changes, but if you'd like to contribute, please reach out to us to discuss your
proposed changes.
## Contributor License Agreement
Contributions to this project must be accompanied by a Contributor License
Agreement (CLA). You (or your employer) retain the copyright to your
contribution; this simply gives us permission to use and redistribute your
contributions as part of the project. Head over to
<https://cla.developers.google.com/> to see your current agreements on file or
to sign a new one.
You generally only need to submit a CLA once, so if you've already submitted one
(even if it was for a different project), you probably don't need to do it
again.
## Code Reviews
All submissions, including submissions by project members, require review. We
use GitHub pull requests for this purpose. Consult
[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
information on using pull requests.
## Community Guidelines
This project follows
[Google's Open Source Community Guidelines](https://opensource.google/conduct/).

@ -1,48 +0,0 @@
workspace(name = "utf8_range")
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load("//:workspace_deps.bzl", "utf8_range_deps")
utf8_range_deps()
http_archive(
name = "rules_python",
sha256 = "9d04041ac92a0985e344235f5d946f71ac543f1b1565f2cdbc9a2aaee8adf55b",
strip_prefix = "rules_python-0.26.0",
url = "https://github.com/bazelbuild/rules_python/releases/download/0.26.0/rules_python-0.26.0.tar.gz",
)
load("@rules_python//python:repositories.bzl", "py_repositories")
py_repositories()
http_archive(
name = "com_google_googletest",
sha256 = "81964fe578e9bd7c94dfdb09c8e4d6e6759e19967e397dbea48d1c10e45d0df2",
strip_prefix = "googletest-release-1.12.1",
urls = [
"https://mirror.bazel.build/github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz",
"https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz",
],
)
http_archive(
name = "rules_fuzzing",
sha256 = "ff52ef4845ab00e95d29c02a9e32e9eff4e0a4c9c8a6bcf8407a2f19eb3f9190",
strip_prefix = "rules_fuzzing-0.4.1",
urls = ["https://github.com/bazelbuild/rules_fuzzing/releases/download/v0.4.1/rules_fuzzing-0.4.1.zip"],
patches = ["//:rules_fuzzing.patch"],
patch_args = ["-p1"],
)
load("@rules_fuzzing//fuzzing:repositories.bzl", "rules_fuzzing_dependencies")
rules_fuzzing_dependencies()
load("@rules_fuzzing//fuzzing:init.bzl", "rules_fuzzing_init")
rules_fuzzing_init()
load("@fuzzing_py_deps//:requirements.bzl", "install_deps")
install_deps()

@ -5,5 +5,5 @@ cc_fuzz_test(
testonly = 1,
srcs = ["utf8_validity_fuzzer.cc"],
dicts = ["utf8_fuzzer.dict"],
deps = ["//:utf8_validity"],
deps = ["//third_party/utf8_range:utf8_validity"],
)

@ -1,25 +0,0 @@
diff --git a/fuzzing/tools/validate_dict.py b/fuzzing/tools/validate_dict.py
index d561e68..52cbcb8 100644
--- a/fuzzing/tools/validate_dict.py
+++ b/fuzzing/tools/validate_dict.py
@@ -19,7 +19,7 @@ Validates and merges a set of fuzzing dictionary files into a single output.
from absl import app
from absl import flags
-from dict_validation import validate_line
+from fuzzing.tools.dict_validation import validate_line
from sys import stderr
FLAGS = flags.FLAGS
diff --git a/fuzzing/requirements.txt b/fuzzing/requirements.txt
index 01482d4..4b36f4f 100644
--- a/fuzzing/requirements.txt
+++ b/fuzzing/requirements.txt
@@ -1,5 +1,5 @@
# Python requirements for the tools supporting the fuzzing rules. These are
# installed automatically through the WORKSPACE configuration.
-absl-py==0.11.0 --hash=sha256:b3d9eb5119ff6e0a0125f6dabf2f9fae02f8acae7be70576002fac27235611c5
-six==1.15.0 --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced
+absl-py==2.0.0 --hash=sha256:9a28abb62774ae4e8edbe2dd4c49ffcd45a6a848952a5eccc6a49f3f0fc1e2f3
+six==1.15.0

@ -1,6 +1,6 @@
#include "utf8_validity.h"
#include "gtest/gtest.h"
#include <gtest/gtest.h>
#include "absl/strings/string_view.h"
namespace utf8_range {

@ -1,11 +0,0 @@
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
def utf8_range_deps():
maybe(
http_archive,
name = "com_google_absl",
url = "https://github.com/abseil/abseil-cpp/archive/8c0b94e793a66495e0b1f34a5eb26bd7dc672db0.zip",
strip_prefix = "abseil-cpp-8c0b94e793a66495e0b1f34a5eb26bd7dc672db0",
sha256 = "b9f490fae1c0d89a19073a081c3c588452461e5586e4ae31bc50a8f36339135e",
)

@ -433,7 +433,7 @@ cc_library(
srcs = ["upb.c"],
hdrs = ["upb.h"],
copts = UPB_DEFAULT_COPTS,
deps = ["@utf8_range"],
deps = ["//third_party/utf8_range"],
)
upb_amalgamation(
@ -485,7 +485,7 @@ cc_library(
srcs = ["php-upb.c"],
hdrs = ["php-upb.h"],
copts = UPB_DEFAULT_COPTS,
deps = ["@utf8_range"],
deps = ["//third_party/utf8_range"],
)
upb_amalgamation(
@ -536,7 +536,7 @@ cc_library(
srcs = ["ruby-upb.c"],
hdrs = ["ruby-upb.h"],
copts = UPB_DEFAULT_COPTS,
deps = ["@utf8_range"],
deps = ["//third_party/utf8_range"],
)
exports_files(

@ -86,6 +86,7 @@ sh_test(
data = [
":copy_protos",
":gen_cmakelists",
"//third_party/utf8_range:utf8_range_srcs",
"//upb:source_files",
"//upb/base:source_files",
"//upb/collections:source_files",
@ -99,7 +100,6 @@ sh_test(
"//upb/reflection:source_files",
"//upb/text:source_files",
"//upb/wire:source_files",
"@utf8_range//:utf8_range_srcs",
],
target_compatible_with = select({
"@platforms//os:windows": ["@platforms//:incompatible"],

@ -292,11 +292,9 @@ class Converter(object):
endif()
if(NOT TARGET utf8_range)
if(EXISTS ../../external/utf8_range)
if(EXISTS ../../third_party/utf8_range)
# utf8_range is already installed
include_directories(../external/utf8_range)
elseif(EXISTS ../../utf8_range)
include_directories(../../utf8_range)
include_directories(../../third_party/utf8_range)
else()
include(FetchContent)
FetchContent_Declare(

@ -60,7 +60,7 @@ cc_library(
"//upb:mini_table",
"//upb:mini_table_internal",
"//upb:port",
"@utf8_range",
"//third_party/utf8_range",
],
)

@ -1,8 +0,0 @@
#!/bin/bash -eux
set -eux
cd $(dirname $0)
git subtree pull --prefix third_party/utf8_range \
https://github.com/protocolbuffers/utf8_range.git main --squash
Loading…
Cancel
Save