diff --git a/.gitmodules b/.gitmodules index 7a46a17f1a..db938cf479 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,3 +9,6 @@ [submodule "third_party/jsoncpp"] path = third_party/jsoncpp url = https://github.com/open-source-parsers/jsoncpp.git +[submodule "third_party/utf8_range"] + path = third_party/utf8_range + url = https://github.com/protocolbuffers/utf8_range.git diff --git a/WORKSPACE b/WORKSPACE index 82f523063a..fb152a24a5 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -80,6 +80,9 @@ upb_deps() load("@upb//bazel:system_python.bzl", "system_python") system_python(name = "local_config_python") +load("@utf8_range//:workspace_deps.bzl", "utf8_range_deps") +utf8_range_deps() + bind( name = "python_headers", actual = "@local_config_python//:python_headers", diff --git a/kokoro/release/ruby/linux/ruby/ruby_build.sh b/kokoro/release/ruby/linux/ruby/ruby_build.sh index 12d270eabd..34a85abc00 100755 --- a/kokoro/release/ruby/linux/ruby/ruby_build.sh +++ b/kokoro/release/ruby/linux/ruby/ruby_build.sh @@ -10,6 +10,9 @@ bazel build //:protoc cp bazel-bin/protoc . export PROTOC=$PWD/protoc +# Pull in dependencies. +git submodule update --init --recursive + umask 0022 pushd ruby gem install bundler -v 2.1.4 diff --git a/kokoro/release/ruby/macos/ruby/ruby_build.sh b/kokoro/release/ruby/macos/ruby/ruby_build.sh index bbfc631197..d13c12045e 100755 --- a/kokoro/release/ruby/macos/ruby/ruby_build.sh +++ b/kokoro/release/ruby/macos/ruby/ruby_build.sh @@ -6,6 +6,9 @@ set -ex bazel build //:protoc export PROTOC=$PWD/bazel-bin/protoc +# Pull in dependencies. +git submodule update --init --recursive + umask 0022 pushd ruby bundle update && bundle exec rake gem:native diff --git a/php/BUILD.bazel b/php/BUILD.bazel index 3c97ad8db2..ac33702ab7 100644 --- a/php/BUILD.bazel +++ b/php/BUILD.bazel @@ -34,7 +34,7 @@ inline_sh_binary( name = "build_extension", cmd = """ mkdir -p php/ext/google/protobuf/third_party/utf8_range - cp third_party/utf8_range/* php/ext/google/protobuf/third_party/utf8_range + cp external/utf8_range/* php/ext/google/protobuf/third_party/utf8_range pushd php/ext/google/protobuf phpize @@ -48,7 +48,7 @@ genrule( name = "extension", srcs = [ ":source_files", - "//third_party/utf8_range:all_files", + "@utf8_range//:utf8_range_srcs", ], tools = [":build_extension"], outs = ["protobuf.so"], diff --git a/php/tests/compile_extension.sh b/php/tests/compile_extension.sh index dc42aa32ee..a645ed60da 100755 --- a/php/tests/compile_extension.sh +++ b/php/tests/compile_extension.sh @@ -4,12 +4,15 @@ set -ex cd $(dirname $0)/.. +# Pull in dependencies. +git submodule update --init --recursive + # utf8_range has to live in the base third_party directory. # We copy it into the ext/google/protobuf directory for the build # (and for the release to PECL). rm -rf ext/google/protobuf/third_party mkdir -p ext/google/protobuf/third_party/utf8_range -cp ../third_party/utf8_range/* ext/google/protobuf/third_party/utf8_range +cp -r ../third_party/utf8_range/* ext/google/protobuf/third_party/utf8_range echo "Copied utf8_range from ../third_party -> ext/google/protobuf/third_party" diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index c514dcbd9b..95e71242de 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -63,6 +63,14 @@ def protobuf_deps(): urls = ["https://github.com/open-source-parsers/jsoncpp/archive/refs/tags/1.9.4.tar.gz"], ) + if not native.existing_rule("utf8_range"): + _github_archive( + name = "utf8_range", + repo = "https://github.com/protocolbuffers/utf8_range", + commit = "45fbf543fec00020a08650791a37575319a3ea1d", + sha256 = "dd93db062025f563068abaa224549e9d341434b5851e959c7853dfa263c96416", + ) + if not native.existing_rule("rules_cc"): _github_archive( name = "rules_cc", diff --git a/ruby/BUILD.bazel b/ruby/BUILD.bazel index 5e15d9241a..d4114b57e2 100644 --- a/ruby/BUILD.bazel +++ b/ruby/BUILD.bazel @@ -71,7 +71,7 @@ filegroup( name = "protobuf", srcs = [ ":srcs", - "//third_party/utf8_range:all_files", + "@utf8_range//:utf8_range_srcs", ] + select({ ":java_ruby": [":protobuf_java"], "@bazel_tools//src/conditions:darwin": [":protobuf_c_mac"], diff --git a/ruby/Rakefile b/ruby/Rakefile index 89a00e1ddb..e6b774c5f3 100644 --- a/ruby/Rakefile +++ b/ruby/Rakefile @@ -84,12 +84,16 @@ if RUBY_PLATFORM == "java" else unless ENV['IN_DOCKER'] == 'true' # We need utf8_range in-tree. + if ENV['BAZEL'] == 'true' + utf8_root = '../external/utf8_range' + else + utf8_root = '../third_party/utf8_range' + end FileUtils.mkdir_p("ext/google/protobuf_c/third_party/utf8_range") - FileUtils.cp("../third_party/utf8_range/utf8_range.h", "ext/google/protobuf_c/third_party/utf8_range") - FileUtils.cp("../third_party/utf8_range/naive.c", "ext/google/protobuf_c/third_party/utf8_range") - FileUtils.cp("../third_party/utf8_range/range2-neon.c", "ext/google/protobuf_c/third_party/utf8_range") - FileUtils.cp("../third_party/utf8_range/range2-sse.c", "ext/google/protobuf_c/third_party/utf8_range") - FileUtils.cp("../third_party/utf8_range/LICENSE", "ext/google/protobuf_c/third_party/utf8_range") + FileUtils.cp(utf8_root+"/utf8_range.h", "ext/google/protobuf_c/third_party/utf8_range") + FileUtils.cp(utf8_root+"/naive.c", "ext/google/protobuf_c/third_party/utf8_range") + FileUtils.cp(utf8_root+"/range2-neon.c", "ext/google/protobuf_c/third_party/utf8_range") + FileUtils.cp(utf8_root+"/range2-sse.c", "ext/google/protobuf_c/third_party/utf8_range") end Rake::ExtensionTask.new("protobuf_c", spec) do |ext| diff --git a/ruby/internal.bzl b/ruby/internal.bzl index 74a2d10ef8..10ae10824c 100644 --- a/ruby/internal.bzl +++ b/ruby/internal.bzl @@ -23,7 +23,7 @@ def internal_ruby_extension( ":srcs", ":test_ruby_protos", ":tests", - "//third_party/utf8_range:all_files", + "@utf8_range//:utf8_range_srcs", ], tags = ["manual"], outs = [extension], diff --git a/third_party/utf8_range b/third_party/utf8_range new file mode 160000 index 0000000000..45fbf543fe --- /dev/null +++ b/third_party/utf8_range @@ -0,0 +1 @@ +Subproject commit 45fbf543fec00020a08650791a37575319a3ea1d diff --git a/third_party/utf8_range/BUILD.bazel b/third_party/utf8_range/BUILD.bazel deleted file mode 100644 index 693f390d80..0000000000 --- a/third_party/utf8_range/BUILD.bazel +++ /dev/null @@ -1,37 +0,0 @@ -load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") - -# Pulled from: https://github.com/cyb70289/utf8 - -cc_library( - name = "utf8_range", - hdrs = ["utf8_range.h"], - srcs = [ - "naive.c", - "range2-neon.c", - "range2-sse.c", - ], - visibility = ["//:__pkg__"], -) - -filegroup( - name = "all_files", - srcs = glob([ - "*.h", - "*.c", - ]) + [ - "BUILD.bazel", - "LICENSE", - ], - visibility = [ - "//cmake:__pkg__", - "//php:__pkg__", - "//ruby:__pkg__", - ], -) - -pkg_files( - name = "dist_files", - srcs = [":all_files"], - strip_prefix = strip_prefix.from_root(""), - visibility = ["//pkg:__pkg__"], -) diff --git a/third_party/utf8_range/LICENSE b/third_party/utf8_range/LICENSE deleted file mode 100644 index c1f087ecc8..0000000000 --- a/third_party/utf8_range/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2019 Yibo Cai - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/third_party/utf8_range/naive.c b/third_party/utf8_range/naive.c deleted file mode 100644 index 9b7e5bb4be..0000000000 --- a/third_party/utf8_range/naive.c +++ /dev/null @@ -1,92 +0,0 @@ -#include - -/* - * http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94 - * - * Table 3-7. Well-Formed UTF-8 Byte Sequences - * - * +--------------------+------------+-------------+------------+-------------+ - * | Code Points | First Byte | Second Byte | Third Byte | Fourth Byte | - * +--------------------+------------+-------------+------------+-------------+ - * | U+0000..U+007F | 00..7F | | | | - * +--------------------+------------+-------------+------------+-------------+ - * | U+0080..U+07FF | C2..DF | 80..BF | | | - * +--------------------+------------+-------------+------------+-------------+ - * | U+0800..U+0FFF | E0 | A0..BF | 80..BF | | - * +--------------------+------------+-------------+------------+-------------+ - * | U+1000..U+CFFF | E1..EC | 80..BF | 80..BF | | - * +--------------------+------------+-------------+------------+-------------+ - * | U+D000..U+D7FF | ED | 80..9F | 80..BF | | - * +--------------------+------------+-------------+------------+-------------+ - * | U+E000..U+FFFF | EE..EF | 80..BF | 80..BF | | - * +--------------------+------------+-------------+------------+-------------+ - * | U+10000..U+3FFFF | F0 | 90..BF | 80..BF | 80..BF | - * +--------------------+------------+-------------+------------+-------------+ - * | U+40000..U+FFFFF | F1..F3 | 80..BF | 80..BF | 80..BF | - * +--------------------+------------+-------------+------------+-------------+ - * | U+100000..U+10FFFF | F4 | 80..8F | 80..BF | 80..BF | - * +--------------------+------------+-------------+------------+-------------+ - */ - -/* Return 0 - success, >0 - index(1 based) of first error char */ -int utf8_naive(const unsigned char *data, int len) -{ - int err_pos = 1; - - while (len) { - int bytes; - const unsigned char byte1 = data[0]; - - /* 00..7F */ - if (byte1 <= 0x7F) { - bytes = 1; - /* C2..DF, 80..BF */ - } else if (len >= 2 && byte1 >= 0xC2 && byte1 <= 0xDF && - (signed char)data[1] <= (signed char)0xBF) { - bytes = 2; - } else if (len >= 3) { - const unsigned char byte2 = data[1]; - - /* Is byte2, byte3 between 0x80 ~ 0xBF */ - const int byte2_ok = (signed char)byte2 <= (signed char)0xBF; - const int byte3_ok = (signed char)data[2] <= (signed char)0xBF; - - if (byte2_ok && byte3_ok && - /* E0, A0..BF, 80..BF */ - ((byte1 == 0xE0 && byte2 >= 0xA0) || - /* E1..EC, 80..BF, 80..BF */ - (byte1 >= 0xE1 && byte1 <= 0xEC) || - /* ED, 80..9F, 80..BF */ - (byte1 == 0xED && byte2 <= 0x9F) || - /* EE..EF, 80..BF, 80..BF */ - (byte1 >= 0xEE && byte1 <= 0xEF))) { - bytes = 3; - } else if (len >= 4) { - /* Is byte4 between 0x80 ~ 0xBF */ - const int byte4_ok = (signed char)data[3] <= (signed char)0xBF; - - if (byte2_ok && byte3_ok && byte4_ok && - /* F0, 90..BF, 80..BF, 80..BF */ - ((byte1 == 0xF0 && byte2 >= 0x90) || - /* F1..F3, 80..BF, 80..BF, 80..BF */ - (byte1 >= 0xF1 && byte1 <= 0xF3) || - /* F4, 80..8F, 80..BF, 80..BF */ - (byte1 == 0xF4 && byte2 <= 0x8F))) { - bytes = 4; - } else { - return err_pos; - } - } else { - return err_pos; - } - } else { - return err_pos; - } - - len -= bytes; - err_pos += bytes; - data += bytes; - } - - return 0; -} diff --git a/third_party/utf8_range/range2-neon.c b/third_party/utf8_range/range2-neon.c deleted file mode 100644 index 683396540c..0000000000 --- a/third_party/utf8_range/range2-neon.c +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Process 2x16 bytes in each iteration. - * Comments removed for brevity. See range-neon.c for details. - */ -#if defined(__aarch64__) && defined(__ARM_NEON) - -#include -#include -#include - -int utf8_naive(const unsigned char *data, int len); - -static const uint8_t _first_len_tbl[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3, -}; - -static const uint8_t _first_range_tbl[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, -}; - -static const uint8_t _range_min_tbl[] = { - 0x00, 0x80, 0x80, 0x80, 0xA0, 0x80, 0x90, 0x80, - 0xC2, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, -}; -static const uint8_t _range_max_tbl[] = { - 0x7F, 0xBF, 0xBF, 0xBF, 0xBF, 0x9F, 0xBF, 0x8F, - 0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -}; - -static const uint8_t _range_adjust_tbl[] = { - 2, 3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, -}; - -/* Return 0 on success, -1 on error */ -int utf8_range2(const unsigned char *data, int len) -{ - if (len >= 32) { - uint8x16_t prev_input = vdupq_n_u8(0); - uint8x16_t prev_first_len = vdupq_n_u8(0); - - const uint8x16_t first_len_tbl = vld1q_u8(_first_len_tbl); - const uint8x16_t first_range_tbl = vld1q_u8(_first_range_tbl); - const uint8x16_t range_min_tbl = vld1q_u8(_range_min_tbl); - const uint8x16_t range_max_tbl = vld1q_u8(_range_max_tbl); - const uint8x16x2_t range_adjust_tbl = vld2q_u8(_range_adjust_tbl); - - const uint8x16_t const_1 = vdupq_n_u8(1); - const uint8x16_t const_2 = vdupq_n_u8(2); - const uint8x16_t const_e0 = vdupq_n_u8(0xE0); - - uint8x16_t error1 = vdupq_n_u8(0); - uint8x16_t error2 = vdupq_n_u8(0); - uint8x16_t error3 = vdupq_n_u8(0); - uint8x16_t error4 = vdupq_n_u8(0); - - while (len >= 32) { - /******************* two blocks interleaved **********************/ - -#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ < 8) - /* gcc doesn't support vldq1_u8_x2 until version 8 */ - const uint8x16_t input_a = vld1q_u8(data); - const uint8x16_t input_b = vld1q_u8(data + 16); -#else - /* Forces a double load on Clang */ - const uint8x16x2_t input_pair = vld1q_u8_x2(data); - const uint8x16_t input_a = input_pair.val[0]; - const uint8x16_t input_b = input_pair.val[1]; -#endif - - const uint8x16_t high_nibbles_a = vshrq_n_u8(input_a, 4); - const uint8x16_t high_nibbles_b = vshrq_n_u8(input_b, 4); - - const uint8x16_t first_len_a = - vqtbl1q_u8(first_len_tbl, high_nibbles_a); - const uint8x16_t first_len_b = - vqtbl1q_u8(first_len_tbl, high_nibbles_b); - - uint8x16_t range_a = vqtbl1q_u8(first_range_tbl, high_nibbles_a); - uint8x16_t range_b = vqtbl1q_u8(first_range_tbl, high_nibbles_b); - - range_a = - vorrq_u8(range_a, vextq_u8(prev_first_len, first_len_a, 15)); - range_b = - vorrq_u8(range_b, vextq_u8(first_len_a, first_len_b, 15)); - - uint8x16_t tmp1_a, tmp2_a, tmp1_b, tmp2_b; - tmp1_a = vextq_u8(prev_first_len, first_len_a, 14); - tmp1_a = vqsubq_u8(tmp1_a, const_1); - range_a = vorrq_u8(range_a, tmp1_a); - - tmp1_b = vextq_u8(first_len_a, first_len_b, 14); - tmp1_b = vqsubq_u8(tmp1_b, const_1); - range_b = vorrq_u8(range_b, tmp1_b); - - tmp2_a = vextq_u8(prev_first_len, first_len_a, 13); - tmp2_a = vqsubq_u8(tmp2_a, const_2); - range_a = vorrq_u8(range_a, tmp2_a); - - tmp2_b = vextq_u8(first_len_a, first_len_b, 13); - tmp2_b = vqsubq_u8(tmp2_b, const_2); - range_b = vorrq_u8(range_b, tmp2_b); - - uint8x16_t shift1_a = vextq_u8(prev_input, input_a, 15); - uint8x16_t pos_a = vsubq_u8(shift1_a, const_e0); - range_a = vaddq_u8(range_a, vqtbl2q_u8(range_adjust_tbl, pos_a)); - - uint8x16_t shift1_b = vextq_u8(input_a, input_b, 15); - uint8x16_t pos_b = vsubq_u8(shift1_b, const_e0); - range_b = vaddq_u8(range_b, vqtbl2q_u8(range_adjust_tbl, pos_b)); - - uint8x16_t minv_a = vqtbl1q_u8(range_min_tbl, range_a); - uint8x16_t maxv_a = vqtbl1q_u8(range_max_tbl, range_a); - - uint8x16_t minv_b = vqtbl1q_u8(range_min_tbl, range_b); - uint8x16_t maxv_b = vqtbl1q_u8(range_max_tbl, range_b); - - error1 = vorrq_u8(error1, vcltq_u8(input_a, minv_a)); - error2 = vorrq_u8(error2, vcgtq_u8(input_a, maxv_a)); - - error3 = vorrq_u8(error3, vcltq_u8(input_b, minv_b)); - error4 = vorrq_u8(error4, vcgtq_u8(input_b, maxv_b)); - - /************************ next iteration *************************/ - prev_input = input_b; - prev_first_len = first_len_b; - - data += 32; - len -= 32; - } - error1 = vorrq_u8(error1, error2); - error1 = vorrq_u8(error1, error3); - error1 = vorrq_u8(error1, error4); - - if (vmaxvq_u8(error1)) - return -1; - - uint32_t token4; - vst1q_lane_u32(&token4, vreinterpretq_u32_u8(prev_input), 3); - - const int8_t *token = (const int8_t *)&token4; - int lookahead = 0; - if (token[3] > (int8_t)0xBF) - lookahead = 1; - else if (token[2] > (int8_t)0xBF) - lookahead = 2; - else if (token[1] > (int8_t)0xBF) - lookahead = 3; - - data -= lookahead; - len += lookahead; - } - - return utf8_naive(data, len); -} - -#endif diff --git a/third_party/utf8_range/range2-sse.c b/third_party/utf8_range/range2-sse.c deleted file mode 100644 index f3deb86006..0000000000 --- a/third_party/utf8_range/range2-sse.c +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Process 2x16 bytes in each iteration. - * Comments removed for brevity. See range-sse.c for details. - */ -#ifdef __SSE4_1__ - -#include -#include -#include - -int utf8_naive(const unsigned char *data, int len); - -static const int8_t _first_len_tbl[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3, -}; - -static const int8_t _first_range_tbl[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, -}; - -static const int8_t _range_min_tbl[] = { - 0x00, 0x80, 0x80, 0x80, 0xA0, 0x80, 0x90, 0x80, - 0xC2, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, -}; -static const int8_t _range_max_tbl[] = { - 0x7F, 0xBF, 0xBF, 0xBF, 0xBF, 0x9F, 0xBF, 0x8F, - 0xF4, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, -}; - -static const int8_t _df_ee_tbl[] = { - 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, -}; -static const int8_t _ef_fe_tbl[] = { - 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; - -/* Return 0 on success, -1 on error */ -int utf8_range2(const unsigned char *data, int len) -{ - if (len >= 32) { - __m128i prev_input = _mm_set1_epi8(0); - __m128i prev_first_len = _mm_set1_epi8(0); - - const __m128i first_len_tbl = - _mm_loadu_si128((const __m128i *)_first_len_tbl); - const __m128i first_range_tbl = - _mm_loadu_si128((const __m128i *)_first_range_tbl); - const __m128i range_min_tbl = - _mm_loadu_si128((const __m128i *)_range_min_tbl); - const __m128i range_max_tbl = - _mm_loadu_si128((const __m128i *)_range_max_tbl); - const __m128i df_ee_tbl = - _mm_loadu_si128((const __m128i *)_df_ee_tbl); - const __m128i ef_fe_tbl = - _mm_loadu_si128((const __m128i *)_ef_fe_tbl); - - __m128i error = _mm_set1_epi8(0); - - while (len >= 32) { - /***************************** block 1 ****************************/ - const __m128i input_a = _mm_loadu_si128((const __m128i *)data); - - __m128i high_nibbles = - _mm_and_si128(_mm_srli_epi16(input_a, 4), _mm_set1_epi8(0x0F)); - - __m128i first_len_a = _mm_shuffle_epi8(first_len_tbl, high_nibbles); - - __m128i range_a = _mm_shuffle_epi8(first_range_tbl, high_nibbles); - - range_a = _mm_or_si128( - range_a, _mm_alignr_epi8(first_len_a, prev_first_len, 15)); - - __m128i tmp; - tmp = _mm_alignr_epi8(first_len_a, prev_first_len, 14); - tmp = _mm_subs_epu8(tmp, _mm_set1_epi8(1)); - range_a = _mm_or_si128(range_a, tmp); - - tmp = _mm_alignr_epi8(first_len_a, prev_first_len, 13); - tmp = _mm_subs_epu8(tmp, _mm_set1_epi8(2)); - range_a = _mm_or_si128(range_a, tmp); - - __m128i shift1, pos, range2; - shift1 = _mm_alignr_epi8(input_a, prev_input, 15); - pos = _mm_sub_epi8(shift1, _mm_set1_epi8(0xEF)); - tmp = _mm_subs_epu8(pos, _mm_set1_epi8(0xF0)); - range2 = _mm_shuffle_epi8(df_ee_tbl, tmp); - tmp = _mm_adds_epu8(pos, _mm_set1_epi8(0x70)); - range2 = _mm_add_epi8(range2, _mm_shuffle_epi8(ef_fe_tbl, tmp)); - - range_a = _mm_add_epi8(range_a, range2); - - __m128i minv = _mm_shuffle_epi8(range_min_tbl, range_a); - __m128i maxv = _mm_shuffle_epi8(range_max_tbl, range_a); - - tmp = _mm_or_si128( - _mm_cmplt_epi8(input_a, minv), - _mm_cmpgt_epi8(input_a, maxv) - ); - error = _mm_or_si128(error, tmp); - - /***************************** block 2 ****************************/ - const __m128i input_b = _mm_loadu_si128((const __m128i *)(data+16)); - - high_nibbles = - _mm_and_si128(_mm_srli_epi16(input_b, 4), _mm_set1_epi8(0x0F)); - - __m128i first_len_b = _mm_shuffle_epi8(first_len_tbl, high_nibbles); - - __m128i range_b = _mm_shuffle_epi8(first_range_tbl, high_nibbles); - - range_b = _mm_or_si128( - range_b, _mm_alignr_epi8(first_len_b, first_len_a, 15)); - - - tmp = _mm_alignr_epi8(first_len_b, first_len_a, 14); - tmp = _mm_subs_epu8(tmp, _mm_set1_epi8(1)); - range_b = _mm_or_si128(range_b, tmp); - - tmp = _mm_alignr_epi8(first_len_b, first_len_a, 13); - tmp = _mm_subs_epu8(tmp, _mm_set1_epi8(2)); - range_b = _mm_or_si128(range_b, tmp); - - shift1 = _mm_alignr_epi8(input_b, input_a, 15); - pos = _mm_sub_epi8(shift1, _mm_set1_epi8(0xEF)); - tmp = _mm_subs_epu8(pos, _mm_set1_epi8(0xF0)); - range2 = _mm_shuffle_epi8(df_ee_tbl, tmp); - tmp = _mm_adds_epu8(pos, _mm_set1_epi8(0x70)); - range2 = _mm_add_epi8(range2, _mm_shuffle_epi8(ef_fe_tbl, tmp)); - - range_b = _mm_add_epi8(range_b, range2); - - minv = _mm_shuffle_epi8(range_min_tbl, range_b); - maxv = _mm_shuffle_epi8(range_max_tbl, range_b); - - - tmp = _mm_or_si128( - _mm_cmplt_epi8(input_b, minv), - _mm_cmpgt_epi8(input_b, maxv) - ); - error = _mm_or_si128(error, tmp); - - /************************ next iteration **************************/ - prev_input = input_b; - prev_first_len = first_len_b; - - data += 32; - len -= 32; - } - - if (!_mm_testz_si128(error, error)) - return -1; - - int32_t token4 = _mm_extract_epi32(prev_input, 3); - const int8_t *token = (const int8_t *)&token4; - int lookahead = 0; - if (token[3] > (int8_t)0xBF) - lookahead = 1; - else if (token[2] > (int8_t)0xBF) - lookahead = 2; - else if (token[1] > (int8_t)0xBF) - lookahead = 3; - - data -= lookahead; - len += lookahead; - } - - return utf8_naive(data, len); -} - -#endif diff --git a/third_party/utf8_range/utf8_range.h b/third_party/utf8_range/utf8_range.h deleted file mode 100644 index 23b32a1c77..0000000000 --- a/third_party/utf8_range/utf8_range.h +++ /dev/null @@ -1,9 +0,0 @@ - -#if ((defined(__ARM_NEON) && defined(__aarch64__)) || defined(__SSE4_1__)) && !defined(TRUFFLERUBY) -int utf8_range2(const unsigned char* data, int len); -#else -int utf8_naive(const unsigned char* data, int len); -static inline int utf8_range2(const unsigned char* data, int len) { - return utf8_naive(data, len); -} -#endif