protobuf/third_party/utf8_range/utf8_range_neon.inc

#include <arm_neon.h>

/* This code is almost the same as SSE implementation, please reference
 * utf8-range-sse.inc for detailed explanation.
 * The only difference is the range adjustment step. NEON code is more
 * straightforward.
 */

static FORCE_INLINE_ATTR inline size_t utf8_range_ValidateUTF8Simd(
    const char* data_original, const char* data, const char* end,
    int return_position) {
  const uint8x16_t first_len_tbl = {
      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3,
  };
  const uint8x16_t first_range_tbl = {
      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8,
  };
  const uint8x16_t range_min_tbl = {
      0x00, 0x80, 0x80, 0x80, 0xA0, 0x80, 0x90, 0x80,
      0xC2, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
  };
  const uint8x16_t range_max_tbl = {
      0x7F, 0xBF, 0xBF, 0xBF, 0xBF, 0x9F, 0xBF, 0x8F,
      0xF4, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  };
  /* Range adjustment in NEON uint8x16x2 table. Note that lanes are interleaved
   * in register. The table below is plotted vertically to ease understanding.
   * The 1st column is for E0~EF, 2nd column for F0~FF.
   */
  // clang-format off
  const uint8_t range_adjust_tbl_data[] = {
    /* index -> 0~15  16~31 <- index */
    /*  E0 -> */ 2,     3, /* <- F0  */
                 0,     0,
                 0,     0,
                 0,     0,
                 0,     4, /* <- F4  */
                 0,     0,
                 0,     0,
                 0,     0,
                 0,     0,
                 0,     0,
                 0,     0,
                 0,     0,
                 0,     0,
    /*  ED -> */ 3,     0,
                 0,     0,
                 0,     0,
  };
  // clang-format on
  const uint8x16x2_t range_adjust_tbl = vld2q_u8(range_adjust_tbl_data);

  const uint8x16_t const_1 = vdupq_n_u8(1);
  const uint8x16_t const_2 = vdupq_n_u8(2);
  const uint8x16_t const_e0 = vdupq_n_u8(0xE0);

  uint8x16_t prev_input = vdupq_n_u8(0);
  uint8x16_t prev_first_len = vdupq_n_u8(0);
  uint8x16_t error = vdupq_n_u8(0);

  while (end - data >= 16) {
    const uint8x16_t input = vld1q_u8((const uint8_t*)data);

    const uint8x16_t high_nibbles = vshrq_n_u8(input, 4);

    const uint8x16_t first_len = vqtbl1q_u8(first_len_tbl, high_nibbles);

    uint8x16_t range = vqtbl1q_u8(first_range_tbl, high_nibbles);

    range = vorrq_u8(range, vextq_u8(prev_first_len, first_len, 15));

    uint8x16_t shift2 = vextq_u8(prev_first_len, first_len, 14);
    shift2 = vqsubq_u8(shift2, const_1);
    range = vorrq_u8(range, shift2);

    uint8x16_t shift3 = vextq_u8(prev_first_len, first_len, 13);
    shift3 = vqsubq_u8(shift3, const_2);
    range = vorrq_u8(range, shift3);

    uint8x16_t shift1 = vextq_u8(prev_input, input, 15);
    shift1 = vsubq_u8(shift1, const_e0);
    range = vaddq_u8(range, vqtbl2q_u8(range_adjust_tbl, shift1));

    const uint8x16_t min_range = vqtbl1q_u8(range_min_tbl, range);
    const uint8x16_t max_range = vqtbl1q_u8(range_max_tbl, range);

    if (return_position) {
      error = vcltq_u8(input, min_range);
      error = vorrq_u8(error, vcgtq_u8(input, max_range));
      if (vmaxvq_u32(vreinterpretq_u32_u8(error))) {
        break;
      }
    } else {
      error = vorrq_u8(error, vcltq_u8(input, min_range));
      error = vorrq_u8(error, vcgtq_u8(input, max_range));
    }

    prev_input = input;
    prev_first_len = first_len;

    data += 16;
  }

  if (return_position && data == data_original) {
    return utf8_range_ValidateUTF8Naive(data, end, return_position);
  }
  const int32_t prev = vgetq_lane_s32(vreinterpretq_s32_u8(prev_input), 3);
  data -= utf8_range_CodepointSkipBackwards(prev);
  if (return_position) {
    return (data - data_original) +
           utf8_range_ValidateUTF8Naive(data, end, return_position);
  }
  if (vmaxvq_u32(vreinterpretq_u32_u8(error))) {
    return 0;
  }
  return utf8_range_ValidateUTF8Naive(data, end, return_position);
}