mirror of https://github.com/madler/zlib.git
This commit introduces an optimized version of the longest_match function for Power processors. It uses VSX instructions to match 16 bytes at a time on each comparison, instead of one by one. Author: Matheus Castanho <msc@linux.ibm.com>pull/459/head
Matheus Castanho
5 years ago
parent
4ec46f57bd
commit
44d19e3374
7 changed files with 238 additions and 3 deletions
@ -0,0 +1,194 @@ |
|||||||
|
/* Copyright (C) 2019 Matheus Castanho <msc@linux.ibm.com>, IBM
|
||||||
|
* For conditions of distribution and use, see copyright notice in zlib.h |
||||||
|
*/ |
||||||
|
|
||||||
|
#include <altivec.h> |
||||||
|
#include "../../deflate.h" |
||||||
|
|
||||||
|
local inline int vec_match OF((Bytef* scan, Bytef* match)) |
||||||
|
__attribute__((always_inline)); |
||||||
|
|
||||||
|
local inline int vec_match(Bytef* scan, Bytef* match) |
||||||
|
{ |
||||||
|
vector unsigned char vscan, vmatch, vc; |
||||||
|
int len; |
||||||
|
|
||||||
|
vscan = *((vector unsigned char *) scan); |
||||||
|
vmatch = *((vector unsigned char *) match); |
||||||
|
|
||||||
|
/* Compare 16 bytes at a time.
|
||||||
|
* Each byte of vc will be either all ones or all zeroes, |
||||||
|
* depending on the result of the comparison |
||||||
|
*/ |
||||||
|
vc = (vector unsigned char) vec_cmpne(vscan,vmatch); |
||||||
|
|
||||||
|
/* Since the index of matching bytes will contain only zeroes
|
||||||
|
* on vc (since we used cmpne), counting the number of consecutive |
||||||
|
* bytes where LSB == 0 is the same as counting the length of the match. |
||||||
|
* |
||||||
|
* There was an issue in the way the vec_cnttz_lsbb builtin was implemented |
||||||
|
* that got fixed on GCC 12, but now we have to use different builtins |
||||||
|
* depending on the compiler version. To avoid that, let's use inline asm to |
||||||
|
* generate the exact instruction we need. |
||||||
|
*/ |
||||||
|
#ifdef __LITTLE_ENDIAN__ |
||||||
|
asm volatile("vctzlsbb %0, %1\n\t" : "=r" (len) : "v" (vc)); |
||||||
|
#else |
||||||
|
asm volatile("vclzlsbb %0, %1\n\t" : "=r" (len) : "v" (vc)); |
||||||
|
#endif |
||||||
|
|
||||||
|
return len; |
||||||
|
} |
||||||
|
|
||||||
|
uInt ZLIB_INTERNAL _longest_match_power9(deflate_state *s, IPos cur_match) |
||||||
|
{ |
||||||
|
unsigned chain_length = s->max_chain_length;/* max hash chain length */ |
||||||
|
register Bytef *scan = s->window + s->strstart; /* current string */ |
||||||
|
register Bytef *match; /* matched string */ |
||||||
|
register int len; /* length of current match */ |
||||||
|
int best_len = (int)s->prev_length; /* best match length so far */ |
||||||
|
int nice_match = s->nice_match; /* stop if match long enough */ |
||||||
|
int mbytes; /* matched bytes inside loop */ |
||||||
|
IPos limit = s->strstart > (IPos)MAX_DIST(s) ? |
||||||
|
s->strstart - (IPos)MAX_DIST(s) : 0; |
||||||
|
/* Stop when cur_match becomes <= limit. To simplify the code,
|
||||||
|
* we prevent matches with the string of window index 0. |
||||||
|
*/ |
||||||
|
Posf *prev = s->prev; |
||||||
|
uInt wmask = s->w_mask; |
||||||
|
|
||||||
|
#if (MAX_MATCH == 258) |
||||||
|
/* Compare the last two bytes at once. */ |
||||||
|
register Bytef *strend2 = s->window + s->strstart + MAX_MATCH - 2; |
||||||
|
register ush scan_end = *(ushf*)(scan+best_len-1); |
||||||
|
#else |
||||||
|
register Bytef *strend = s->window + s->strstart + MAX_MATCH; |
||||||
|
register Byte scan_end1 = scan[best_len-1]; |
||||||
|
register Byte scan_end = scan[best_len]; |
||||||
|
#endif |
||||||
|
|
||||||
|
/* The code is optimized for HASH_BITS >= 8 and MAX_MATCH-2 multiple of 16.
|
||||||
|
* It is easy to get rid of this optimization if necessary. |
||||||
|
*/ |
||||||
|
Assert(s->hash_bits >= 8 && MAX_MATCH == 258, "Code too clever"); |
||||||
|
|
||||||
|
/* Do not waste too much time if we already have a good match: */ |
||||||
|
if (s->prev_length >= s->good_match) { |
||||||
|
chain_length >>= 2; |
||||||
|
} |
||||||
|
/* Do not look for matches beyond the end of the input. This is necessary
|
||||||
|
* to make deflate deterministic. |
||||||
|
*/ |
||||||
|
if ((uInt)nice_match > s->lookahead) nice_match = (int)s->lookahead; |
||||||
|
|
||||||
|
Assert((ulg)s->strstart <= s->window_size-MIN_LOOKAHEAD, "need lookahead"); |
||||||
|
|
||||||
|
do { |
||||||
|
Assert(cur_match < s->strstart, "no future"); |
||||||
|
match = s->window + cur_match; |
||||||
|
|
||||||
|
/* Skip to next match if the match length cannot increase
|
||||||
|
* or if the match length is less than 2. Note that the checks below |
||||||
|
* for insufficient lookahead only occur occasionally for performance |
||||||
|
* reasons. Therefore uninitialized memory will be accessed, and |
||||||
|
* conditional jumps will be made that depend on those values. |
||||||
|
* However the length of the match is limited to the lookahead, so |
||||||
|
* the output of deflate is not affected by the uninitialized values. |
||||||
|
*/ |
||||||
|
|
||||||
|
/* MAX_MATCH - 2 should be a multiple of 16 for this optimization to work. */ |
||||||
|
#if (MAX_MATCH == 258) |
||||||
|
|
||||||
|
/* Compare ending (2 bytes) and beginning of potential match.
|
||||||
|
* |
||||||
|
* On Power processors, loading a 16-byte vector takes only 1 extra |
||||||
|
* cycle compared to a regular byte load. So instead of comparing the |
||||||
|
* first two bytes and then the rest later if they match, we can compare |
||||||
|
* the first 16 at once, and when we have a match longer than 2, we will |
||||||
|
* already have the result of comparing the first 16 bytes saved in mbytes. |
||||||
|
*/ |
||||||
|
if (*(ushf*)(match+best_len-1) != scan_end || |
||||||
|
(mbytes = vec_match(scan,match)) < 3) continue; |
||||||
|
|
||||||
|
scan += mbytes; |
||||||
|
match += mbytes; |
||||||
|
|
||||||
|
/* In case when we may have a match longer than 16, we perform further
|
||||||
|
* comparisons in chunks of 16 and keep going while all bytes match. |
||||||
|
*/ |
||||||
|
while(mbytes == 16) { |
||||||
|
mbytes = vec_match(scan,match); |
||||||
|
scan += mbytes; |
||||||
|
match += mbytes; |
||||||
|
|
||||||
|
/* We also have to limit the maximum match based on MAX_MATCH.
|
||||||
|
* Since we are comparing 16 bytes at a time and MAX_MATCH == 258 (to |
||||||
|
* comply with default implementation), we should stop comparing when |
||||||
|
* we have matched 256 bytes, which happens when scan == strend2. |
||||||
|
* In this ("rare") case, we have to check the remaining 2 bytes |
||||||
|
* individually using common load and compare operations. |
||||||
|
*/ |
||||||
|
if(scan >= strend2) { |
||||||
|
if(*scan == *match) { |
||||||
|
if(*++scan == *++match) |
||||||
|
scan++; |
||||||
|
} |
||||||
|
break; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
Assert(scan <= s->window+(unsigned)(s->window_size-1), "wild scan"); |
||||||
|
|
||||||
|
len = (MAX_MATCH - 2) - (int)(strend2 - scan); |
||||||
|
scan = strend2 - (MAX_MATCH - 2); |
||||||
|
|
||||||
|
#else /* MAX_MATCH == 258 */ |
||||||
|
|
||||||
|
if (match[best_len] != scan_end || |
||||||
|
match[best_len-1] != scan_end1 || |
||||||
|
*match != *scan || |
||||||
|
*++match != scan[1]) continue; |
||||||
|
|
||||||
|
/* The check at best_len-1 can be removed because it will be made
|
||||||
|
* again later. (This heuristic is not always a win.) |
||||||
|
* It is not necessary to compare scan[2] and match[2] since they |
||||||
|
* are always equal when the other bytes match, given that |
||||||
|
* the hash keys are equal and that HASH_BITS >= 8. |
||||||
|
*/ |
||||||
|
scan += 2, match++; |
||||||
|
Assert(*scan == *match, "match[2]?"); |
||||||
|
|
||||||
|
/* We check for insufficient lookahead only every 8th comparison;
|
||||||
|
* the 256th check will be made at strstart+258. |
||||||
|
*/ |
||||||
|
do { |
||||||
|
} while (*++scan == *++match && *++scan == *++match && |
||||||
|
*++scan == *++match && *++scan == *++match && |
||||||
|
*++scan == *++match && *++scan == *++match && |
||||||
|
*++scan == *++match && *++scan == *++match && |
||||||
|
scan < strend); |
||||||
|
|
||||||
|
Assert(scan <= s->window+(unsigned)(s->window_size-1), "wild scan"); |
||||||
|
|
||||||
|
len = MAX_MATCH - (int)(strend - scan); |
||||||
|
scan = strend - MAX_MATCH; |
||||||
|
|
||||||
|
#endif /* MAX_MATCH == 258 */ |
||||||
|
|
||||||
|
if (len > best_len) { |
||||||
|
s->match_start = cur_match; |
||||||
|
best_len = len; |
||||||
|
if (len >= nice_match) break; |
||||||
|
#if (MAX_MATCH == 258) |
||||||
|
scan_end = *(ushf*)(scan+best_len-1); |
||||||
|
#else |
||||||
|
scan_end1 = scan[best_len-1]; |
||||||
|
scan_end = scan[best_len]; |
||||||
|
#endif |
||||||
|
} |
||||||
|
} while ((cur_match = prev[cur_match & wmask]) > limit |
||||||
|
&& --chain_length != 0); |
||||||
|
|
||||||
|
if ((uInt)best_len <= s->lookahead) return (uInt)best_len; |
||||||
|
return s->lookahead; |
||||||
|
} |
||||||
@ -0,0 +1,15 @@ |
|||||||
|
/* Copyright (C) 2019 Matheus Castanho <msc@linux.ibm.com>, IBM
|
||||||
|
* For conditions of distribution and use, see copyright notice in zlib.h |
||||||
|
*/ |
||||||
|
|
||||||
|
#include "../gcc/zifunc.h" |
||||||
|
#include "power.h" |
||||||
|
|
||||||
|
Z_IFUNC(longest_match) { |
||||||
|
#ifdef Z_POWER9 |
||||||
|
if (__builtin_cpu_supports("arch_3_00")) |
||||||
|
return _longest_match_power9; |
||||||
|
#endif |
||||||
|
|
||||||
|
return longest_match_default; |
||||||
|
} |
||||||
Loading…
Reference in new issue