mirror of https://github.com/madler/zlib.git
Mark Adler
13 years ago
parent
639be99788
commit
f6194ef39a
45 changed files with 3642 additions and 852 deletions
@ -0,0 +1,357 @@ |
||||
/* |
||||
* match.S -- optimized version of longest_match() |
||||
* based on the similar work by Gilles Vollant, and Brian Raiter, written 1998 |
||||
* |
||||
* This is free software; you can redistribute it and/or modify it
|
||||
* under the terms of the BSD License. Use by owners of Che Guevarra |
||||
* parafernalia is prohibited, where possible, and highly discouraged |
||||
* elsewhere. |
||||
*/ |
||||
|
||||
#ifndef NO_UNDERLINE |
||||
# define match_init _match_init |
||||
# define longest_match _longest_match |
||||
#endif |
||||
|
||||
#define scanend ebx |
||||
#define scanendw bx |
||||
#define chainlenwmask edx /* high word: current chain len low word: s->wmask */ |
||||
#define curmatch rsi |
||||
#define curmatchd esi |
||||
#define windowbestlen r8 |
||||
#define scanalign r9 |
||||
#define scanalignd r9d |
||||
#define window r10 |
||||
#define bestlen r11 |
||||
#define bestlend r11d |
||||
#define scanstart r12d |
||||
#define scanstartw r12w |
||||
#define scan r13 |
||||
#define nicematch r14d |
||||
#define limit r15 |
||||
#define limitd r15d |
||||
#define prev rcx |
||||
|
||||
/* |
||||
* The 258 is a "magic number, not a parameter -- changing it |
||||
* breaks the hell loose |
||||
*/ |
||||
#define MAX_MATCH (258) |
||||
#define MIN_MATCH (3) |
||||
#define MIN_LOOKAHEAD (MAX_MATCH + MIN_MATCH + 1) |
||||
#define MAX_MATCH_8 ((MAX_MATCH + 7) & ~7) |
||||
|
||||
/* stack frame offsets */ |
||||
#define LocalVarsSize (112) |
||||
#define _chainlenwmask ( 8-LocalVarsSize)(%rsp) |
||||
#define _windowbestlen (16-LocalVarsSize)(%rsp) |
||||
#define save_r14 (24-LocalVarsSize)(%rsp) |
||||
#define save_rsi (32-LocalVarsSize)(%rsp) |
||||
#define save_rbx (40-LocalVarsSize)(%rsp) |
||||
#define save_r12 (56-LocalVarsSize)(%rsp) |
||||
#define save_r13 (64-LocalVarsSize)(%rsp) |
||||
#define save_r15 (80-LocalVarsSize)(%rsp) |
||||
|
||||
/* |
||||
* On AMD64 the first argument of a function (in our case -- the pointer to |
||||
* deflate_state structure) is passed in %rdi, hence our offsets below are |
||||
* all off of that. |
||||
*/ |
||||
#ifndef STRUCT_OFFSET |
||||
# define STRUCT_OFFSET (0) |
||||
#endif |
||||
#define dsWSize ( 56 + STRUCT_OFFSET)(%rdi) |
||||
#define dsWMask ( 64 + STRUCT_OFFSET)(%rdi) |
||||
#define dsWindow ( 72 + STRUCT_OFFSET)(%rdi) |
||||
#define dsPrev ( 88 + STRUCT_OFFSET)(%rdi) |
||||
#define dsMatchLen (136 + STRUCT_OFFSET)(%rdi) |
||||
#define dsPrevMatch (140 + STRUCT_OFFSET)(%rdi) |
||||
#define dsStrStart (148 + STRUCT_OFFSET)(%rdi) |
||||
#define dsMatchStart (152 + STRUCT_OFFSET)(%rdi) |
||||
#define dsLookahead (156 + STRUCT_OFFSET)(%rdi) |
||||
#define dsPrevLen (160 + STRUCT_OFFSET)(%rdi) |
||||
#define dsMaxChainLen (164 + STRUCT_OFFSET)(%rdi) |
||||
#define dsGoodMatch (180 + STRUCT_OFFSET)(%rdi) |
||||
#define dsNiceMatch (184 + STRUCT_OFFSET)(%rdi) |
||||
|
||||
.globl match_init, longest_match |
||||
|
||||
.text |
||||
|
||||
/* uInt longest_match(deflate_state *deflatestate, IPos curmatch) */ |
||||
|
||||
longest_match: |
||||
/* |
||||
* Retrieve the function arguments. %curmatch will hold cur_match |
||||
* throughout the entire function (passed via rsi on amd64). |
||||
* rdi will hold the pointer to the deflate_state (first arg on amd64) |
||||
*/ |
||||
mov %rsi, save_rsi |
||||
mov %rbx, save_rbx |
||||
mov %r12, save_r12 |
||||
mov %r13, save_r13 |
||||
mov %r14, save_r14 |
||||
mov %r15, save_r15 |
||||
|
||||
/* uInt wmask = s->w_mask; */ |
||||
/* unsigned chain_length = s->max_chain_length; */ |
||||
/* if (s->prev_length >= s->good_match) { */ |
||||
/* chain_length >>= 2; */ |
||||
/* } */ |
||||
|
||||
movl dsPrevLen, %eax |
||||
movl dsGoodMatch, %ebx |
||||
cmpl %ebx, %eax |
||||
movl dsWMask, %eax |
||||
movl dsMaxChainLen, %chainlenwmask |
||||
jl LastMatchGood |
||||
shrl $2, %chainlenwmask |
||||
LastMatchGood: |
||||
|
||||
/* chainlen is decremented once beforehand so that the function can */ |
||||
/* use the sign flag instead of the zero flag for the exit test. */ |
||||
/* It is then shifted into the high word, to make room for the wmask */ |
||||
/* value, which it will always accompany. */ |
||||
|
||||
decl %chainlenwmask |
||||
shll $16, %chainlenwmask |
||||
orl %eax, %chainlenwmask |
||||
|
||||
/* if ((uInt)nice_match > s->lookahead) nice_match = s->lookahead; */ |
||||
|
||||
movl dsNiceMatch, %eax |
||||
movl dsLookahead, %ebx |
||||
cmpl %eax, %ebx |
||||
jl LookaheadLess |
||||
movl %eax, %ebx |
||||
LookaheadLess: movl %ebx, %nicematch |
||||
|
||||
/* register Bytef *scan = s->window + s->strstart; */ |
||||
|
||||
mov dsWindow, %window |
||||
movl dsStrStart, %limitd |
||||
lea (%limit, %window), %scan |
||||
|
||||
/* Determine how many bytes the scan ptr is off from being */ |
||||
/* dword-aligned. */ |
||||
|
||||
mov %scan, %scanalign |
||||
negl %scanalignd |
||||
andl $3, %scanalignd |
||||
|
||||
/* IPos limit = s->strstart > (IPos)MAX_DIST(s) ? */ |
||||
/* s->strstart - (IPos)MAX_DIST(s) : NIL; */ |
||||
|
||||
movl dsWSize, %eax |
||||
subl $MIN_LOOKAHEAD, %eax |
||||
xorl %ecx, %ecx |
||||
subl %eax, %limitd |
||||
cmovng %ecx, %limitd |
||||
|
||||
/* int best_len = s->prev_length; */ |
||||
|
||||
movl dsPrevLen, %bestlend |
||||
|
||||
/* Store the sum of s->window + best_len in %windowbestlen locally, and in memory. */ |
||||
|
||||
lea (%window, %bestlen), %windowbestlen |
||||
mov %windowbestlen, _windowbestlen |
||||
|
||||
/* register ush scan_start = *(ushf*)scan; */ |
||||
/* register ush scan_end = *(ushf*)(scan+best_len-1); */ |
||||
/* Posf *prev = s->prev; */ |
||||
|
||||
movzwl (%scan), %scanstart |
||||
movzwl -1(%scan, %bestlen), %scanend |
||||
mov dsPrev, %prev |
||||
|
||||
/* Jump into the main loop. */ |
||||
|
||||
movl %chainlenwmask, _chainlenwmask |
||||
jmp LoopEntry |
||||
|
||||
.balign 16
|
||||
|
||||
/* do { |
||||
* match = s->window + cur_match;
|
||||
* if (*(ushf*)(match+best_len-1) != scan_end || |
||||
* *(ushf*)match != scan_start) continue;
|
||||
* [...] |
||||
* } while ((cur_match = prev[cur_match & wmask]) > limit |
||||
* && --chain_length != 0);
|
||||
* |
||||
* Here is the inner loop of the function. The function will spend the |
||||
* majority of its time in this loop, and majority of that time will |
||||
* be spent in the first ten instructions. |
||||
*/ |
||||
LookupLoop: |
||||
andl %chainlenwmask, %curmatchd |
||||
movzwl (%prev, %curmatch, 2), %curmatchd |
||||
cmpl %limitd, %curmatchd |
||||
jbe LeaveNow |
||||
subl $0x00010000, %chainlenwmask |
||||
js LeaveNow |
||||
LoopEntry: cmpw -1(%windowbestlen, %curmatch), %scanendw |
||||
jne LookupLoop |
||||
cmpw %scanstartw, (%window, %curmatch) |
||||
jne LookupLoop |
||||
|
||||
/* Store the current value of chainlen. */ |
||||
movl %chainlenwmask, _chainlenwmask |
||||
|
||||
/* %scan is the string under scrutiny, and %prev to the string we */ |
||||
/* are hoping to match it up with. In actuality, %esi and %edi are */ |
||||
/* both pointed (MAX_MATCH_8 - scanalign) bytes ahead, and %edx is */ |
||||
/* initialized to -(MAX_MATCH_8 - scanalign). */ |
||||
|
||||
mov $(-MAX_MATCH_8), %rdx |
||||
lea (%curmatch, %window), %windowbestlen |
||||
lea MAX_MATCH_8(%windowbestlen, %scanalign), %windowbestlen |
||||
lea MAX_MATCH_8(%scan, %scanalign), %prev |
||||
|
||||
/* the prefetching below makes very little difference... */ |
||||
prefetcht1 (%windowbestlen, %rdx) |
||||
prefetcht1 (%prev, %rdx) |
||||
|
||||
/* |
||||
* Test the strings for equality, 8 bytes at a time. At the end, |
||||
* adjust %rdx so that it is offset to the exact byte that mismatched. |
||||
* |
||||
* It should be confessed that this loop usually does not represent |
||||
* much of the total running time. Replacing it with a more |
||||
* straightforward "rep cmpsb" would not drastically degrade |
||||
* performance -- unrolling it, for example, makes no difference. |
||||
*/ |
||||
#undef USE_SSE /* works, but is 6-7% slower, than non-SSE... */ |
||||
LoopCmps: |
||||
#ifdef USE_SSE |
||||
/* Preload the SSE registers */ |
||||
movdqu (%windowbestlen, %rdx), %xmm1 |
||||
movdqu (%prev, %rdx), %xmm2 |
||||
pcmpeqb %xmm2, %xmm1 |
||||
movdqu 16(%windowbestlen, %rdx), %xmm3 |
||||
movdqu 16(%prev, %rdx), %xmm4 |
||||
pcmpeqb %xmm4, %xmm3 |
||||
movdqu 32(%windowbestlen, %rdx), %xmm5 |
||||
movdqu 32(%prev, %rdx), %xmm6 |
||||
pcmpeqb %xmm6, %xmm5 |
||||
movdqu 48(%windowbestlen, %rdx), %xmm7 |
||||
movdqu 48(%prev, %rdx), %xmm8 |
||||
pcmpeqb %xmm8, %xmm7 |
||||
|
||||
/* Check the comparisions' results */ |
||||
pmovmskb %xmm1, %rax |
||||
notw %ax |
||||
bsfw %ax, %ax |
||||
jnz LeaveLoopCmps |
||||
add $16, %rdx |
||||
pmovmskb %xmm3, %rax |
||||
notw %ax |
||||
bsfw %ax, %ax |
||||
jnz LeaveLoopCmps |
||||
add $16, %rdx |
||||
pmovmskb %xmm5, %rax |
||||
notw %ax |
||||
bsfw %ax, %ax |
||||
jnz LeaveLoopCmps |
||||
add $16, %rdx |
||||
pmovmskb %xmm7, %rax |
||||
notw %ax |
||||
bsfw %ax, %ax |
||||
jnz LeaveLoopCmps |
||||
add $16, %rdx |
||||
jmp LoopCmps |
||||
LeaveLoopCmps: add %rax, %rdx |
||||
#else |
||||
mov (%windowbestlen, %rdx), %rax |
||||
xor (%prev, %rdx), %rax |
||||
jnz LeaveLoopCmps |
||||
add $8, %rdx |
||||
jnz LoopCmps |
||||
jmp LenMaximum |
||||
# if 0 |
||||
/* |
||||
* This three-liner is tantalizingly simple, but bsf is a slow instruction, |
||||
* and the complicated alternative down below is quite a bit faster. Sad... |
||||
*/ |
||||
LeaveLoopCmps: bsf %rax, %rax /* find the first non-zero bit */ |
||||
shrl $3, %eax /* divide by 8 to get the byte */ |
||||
add %rax, %rdx |
||||
# else |
||||
LeaveLoopCmps: testl $0xFFFFFFFF, %eax /* Check the first 4 bytes */ |
||||
jnz Check16 |
||||
add $4, %rdx |
||||
shr $32, %rax |
||||
Check16: testw $0xFFFF, %ax |
||||
jnz LenLower |
||||
add $2, %rdx |
||||
shrl $16, %eax |
||||
LenLower: subb $1, %al |
||||
adc $0, %rdx |
||||
# endif |
||||
#endif |
||||
|
||||
/* Calculate the length of the match. If it is longer than MAX_MATCH, */ |
||||
/* then automatically accept it as the best possible match and leave. */ |
||||
|
||||
lea (%prev, %rdx), %rax |
||||
sub %scan, %rax |
||||
cmpl $MAX_MATCH, %eax |
||||
jge LenMaximum |
||||
|
||||
/* If the length of the match is not longer than the best match we */ |
||||
/* have so far, then forget it and return to the lookup loop. */ |
||||
|
||||
cmpl %bestlend, %eax |
||||
jg LongerMatch |
||||
mov _windowbestlen, %windowbestlen |
||||
mov dsPrev, %prev |
||||
movl _chainlenwmask, %edx |
||||
jmp LookupLoop |
||||
|
||||
/* s->match_start = cur_match; */ |
||||
/* best_len = len; */ |
||||
/* if (len >= nice_match) break; */ |
||||
/* scan_end = *(ushf*)(scan+best_len-1); */ |
||||
|
||||
LongerMatch: |
||||
movl %eax, %bestlend |
||||
movl %curmatchd, dsMatchStart |
||||
cmpl %nicematch, %eax |
||||
jge LeaveNow |
||||
|
||||
lea (%window, %bestlen), %windowbestlen |
||||
mov %windowbestlen, _windowbestlen |
||||
|
||||
movzwl -1(%scan, %rax), %scanend |
||||
mov dsPrev, %prev |
||||
movl _chainlenwmask, %chainlenwmask |
||||
jmp LookupLoop |
||||
|
||||
/* Accept the current string, with the maximum possible length. */ |
||||
|
||||
LenMaximum: |
||||
movl $MAX_MATCH, %bestlend |
||||
movl %curmatchd, dsMatchStart |
||||
|
||||
/* if ((uInt)best_len <= s->lookahead) return (uInt)best_len; */ |
||||
/* return s->lookahead; */ |
||||
|
||||
LeaveNow: |
||||
movl dsLookahead, %eax |
||||
cmpl %eax, %bestlend |
||||
cmovngl %bestlend, %eax |
||||
LookaheadRet: |
||||
|
||||
/* Restore the registers and return from whence we came. */ |
||||
|
||||
mov save_rsi, %rsi |
||||
mov save_rbx, %rbx |
||||
mov save_r12, %r12 |
||||
mov save_r13, %r13 |
||||
mov save_r14, %r14 |
||||
mov save_r15, %r15 |
||||
|
||||
ret |
||||
|
||||
match_init: ret |
||||
@ -0,0 +1,126 @@ |
||||
#---------------------------------------------------------------------------------
|
||||
.SUFFIXES: |
||||
#---------------------------------------------------------------------------------
|
||||
|
||||
ifeq ($(strip $(DEVKITARM)),) |
||||
$(error "Please set DEVKITARM in your environment. export DEVKITARM=<path to>devkitARM") |
||||
endif |
||||
|
||||
include $(DEVKITARM)/ds_rules |
||||
|
||||
#---------------------------------------------------------------------------------
|
||||
# TARGET is the name of the output
|
||||
# BUILD is the directory where object files & intermediate files will be placed
|
||||
# SOURCES is a list of directories containing source code
|
||||
# DATA is a list of directories containing data files
|
||||
# INCLUDES is a list of directories containing header files
|
||||
#---------------------------------------------------------------------------------
|
||||
TARGET := $(shell basename $(CURDIR))
|
||||
BUILD := build
|
||||
SOURCES := ../../
|
||||
DATA := data
|
||||
INCLUDES := include
|
||||
|
||||
#---------------------------------------------------------------------------------
|
||||
# options for code generation
|
||||
#---------------------------------------------------------------------------------
|
||||
ARCH := -mthumb -mthumb-interwork
|
||||
|
||||
CFLAGS := -Wall -O2\
|
||||
-march=armv5te -mtune=arm946e-s \
|
||||
-fomit-frame-pointer -ffast-math \
|
||||
$(ARCH)
|
||||
|
||||
CFLAGS += $(INCLUDE) -DARM9
|
||||
CXXFLAGS := $(CFLAGS) -fno-rtti -fno-exceptions
|
||||
|
||||
ASFLAGS := $(ARCH) -march=armv5te -mtune=arm946e-s
|
||||
LDFLAGS = -specs=ds_arm9.specs -g $(ARCH) -Wl,-Map,$(notdir $*.map)
|
||||
|
||||
#---------------------------------------------------------------------------------
|
||||
# list of directories containing libraries, this must be the top level containing
|
||||
# include and lib
|
||||
#---------------------------------------------------------------------------------
|
||||
LIBDIRS := $(LIBNDS)
|
||||
|
||||
#---------------------------------------------------------------------------------
|
||||
# no real need to edit anything past this point unless you need to add additional
|
||||
# rules for different file extensions
|
||||
#---------------------------------------------------------------------------------
|
||||
ifneq ($(BUILD),$(notdir $(CURDIR))) |
||||
#---------------------------------------------------------------------------------
|
||||
|
||||
export OUTPUT := $(CURDIR)/lib/libz.a
|
||||
|
||||
export VPATH := $(foreach dir,$(SOURCES),$(CURDIR)/$(dir)) \
|
||||
$(foreach dir,$(DATA),$(CURDIR)/$(dir))
|
||||
|
||||
export DEPSDIR := $(CURDIR)/$(BUILD)
|
||||
|
||||
CFILES := $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.c)))
|
||||
CPPFILES := $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.cpp)))
|
||||
SFILES := $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.s)))
|
||||
BINFILES := $(foreach dir,$(DATA),$(notdir $(wildcard $(dir)/*.*)))
|
||||
|
||||
#---------------------------------------------------------------------------------
|
||||
# use CXX for linking C++ projects, CC for standard C
|
||||
#---------------------------------------------------------------------------------
|
||||
ifeq ($(strip $(CPPFILES)),) |
||||
#---------------------------------------------------------------------------------
|
||||
export LD := $(CC)
|
||||
#---------------------------------------------------------------------------------
|
||||
else |
||||
#---------------------------------------------------------------------------------
|
||||
export LD := $(CXX)
|
||||
#---------------------------------------------------------------------------------
|
||||
endif |
||||
#---------------------------------------------------------------------------------
|
||||
|
||||
export OFILES := $(addsuffix .o,$(BINFILES)) \
|
||||
$(CPPFILES:.cpp=.o) $(CFILES:.c=.o) $(SFILES:.s=.o)
|
||||
|
||||
export INCLUDE := $(foreach dir,$(INCLUDES),-I$(CURDIR)/$(dir)) \
|
||||
$(foreach dir,$(LIBDIRS),-I$(dir)/include) \
|
||||
-I$(CURDIR)/$(BUILD)
|
||||
|
||||
.PHONY: $(BUILD) clean all |
||||
|
||||
#---------------------------------------------------------------------------------
|
||||
all: $(BUILD) |
||||
@[ -d $@ ] || mkdir -p include
|
||||
@cp ../../*.h include
|
||||
|
||||
lib: |
||||
@[ -d $@ ] || mkdir -p $@
|
||||
|
||||
$(BUILD): lib |
||||
@[ -d $@ ] || mkdir -p $@
|
||||
@$(MAKE) --no-print-directory -C $(BUILD) -f $(CURDIR)/Makefile
|
||||
|
||||
#---------------------------------------------------------------------------------
|
||||
clean: |
||||
@echo clean ...
|
||||
@rm -fr $(BUILD) lib
|
||||
|
||||
#---------------------------------------------------------------------------------
|
||||
else |
||||
|
||||
DEPENDS := $(OFILES:.o=.d)
|
||||
|
||||
#---------------------------------------------------------------------------------
|
||||
# main targets
|
||||
#---------------------------------------------------------------------------------
|
||||
$(OUTPUT) : $(OFILES) |
||||
|
||||
#---------------------------------------------------------------------------------
|
||||
%.bin.o : %.bin |
||||
#---------------------------------------------------------------------------------
|
||||
@echo $(notdir $<)
|
||||
@$(bin2o)
|
||||
|
||||
|
||||
-include $(DEPENDS) |
||||
|
||||
#---------------------------------------------------------------------------------------
|
||||
endif |
||||
#---------------------------------------------------------------------------------------
|
||||
@ -0,0 +1,5 @@ |
||||
This Makefile requires devkitARM (http://www.devkitpro.org/category/devkitarm/) and works inside "contrib/nds". It is based on a devkitARM template. |
||||
|
||||
Eduardo Costa <eduardo.m.costa@gmail.com> |
||||
January 3, 2009 |
||||
|
||||
Binary file not shown.
@ -0,0 +1,569 @@ |
||||
/* enough.c -- determine the maximum size of inflate's Huffman code tables over
|
||||
* all possible valid and complete Huffman codes, subject to a length limit. |
||||
* Copyright (C) 2007, 2008 Mark Adler |
||||
* Version 1.3 17 February 2008 Mark Adler |
||||
*/ |
||||
|
||||
/* Version history:
|
||||
1.0 3 Jan 2007 First version (derived from codecount.c version 1.4) |
||||
1.1 4 Jan 2007 Use faster incremental table usage computation |
||||
Prune examine() search on previously visited states |
||||
1.2 5 Jan 2007 Comments clean up |
||||
As inflate does, decrease root for short codes |
||||
Refuse cases where inflate would increase root |
||||
1.3 17 Feb 2008 Add argument for initial root table size |
||||
Fix bug for initial root table size == max - 1 |
||||
Use a macro to compute the history index |
||||
*/ |
||||
|
||||
/*
|
||||
Examine all possible Huffman codes for a given number of symbols and a |
||||
maximum code length in bits to determine the maximum table size for zilb's |
||||
inflate. Only complete Huffman codes are counted. |
||||
|
||||
Two codes are considered distinct if the vectors of the number of codes per |
||||
length are not identical. So permutations of the symbol assignments result |
||||
in the same code for the counting, as do permutations of the assignments of |
||||
the bit values to the codes (i.e. only canonical codes are counted). |
||||
|
||||
We build a code from shorter to longer lengths, determining how many symbols |
||||
are coded at each length. At each step, we have how many symbols remain to |
||||
be coded, what the last code length used was, and how many bit patterns of |
||||
that length remain unused. Then we add one to the code length and double the |
||||
number of unused patterns to graduate to the next code length. We then |
||||
assign all portions of the remaining symbols to that code length that |
||||
preserve the properties of a correct and eventually complete code. Those |
||||
properties are: we cannot use more bit patterns than are available; and when |
||||
all the symbols are used, there are exactly zero possible bit patterns |
||||
remaining. |
||||
|
||||
The inflate Huffman decoding algorithm uses two-level lookup tables for |
||||
speed. There is a single first-level table to decode codes up to root bits |
||||
in length (root == 9 in the current inflate implementation). The table |
||||
has 1 << root entries and is indexed by the next root bits of input. Codes |
||||
shorter than root bits have replicated table entries, so that the correct |
||||
entry is pointed to regardless of the bits that follow the short code. If |
||||
the code is longer than root bits, then the table entry points to a second- |
||||
level table. The size of that table is determined by the longest code with |
||||
that root-bit prefix. If that longest code has length len, then the table |
||||
has size 1 << (len - root), to index the remaining bits in that set of |
||||
codes. Each subsequent root-bit prefix then has its own sub-table. The |
||||
total number of table entries required by the code is calculated |
||||
incrementally as the number of codes at each bit length is populated. When |
||||
all of the codes are shorter than root bits, then root is reduced to the |
||||
longest code length, resulting in a single, smaller, one-level table. |
||||
|
||||
The inflate algorithm also provides for small values of root (relative to |
||||
the log2 of the number of symbols), where the shortest code has more bits |
||||
than root. In that case, root is increased to the length of the shortest |
||||
code. This program, by design, does not handle that case, so it is verified |
||||
that the number of symbols is less than 2^(root + 1). |
||||
|
||||
In order to speed up the examination (by about ten orders of magnitude for |
||||
the default arguments), the intermediate states in the build-up of a code |
||||
are remembered and previously visited branches are pruned. The memory |
||||
required for this will increase rapidly with the total number of symbols and |
||||
the maximum code length in bits. However this is a very small price to pay |
||||
for the vast speedup. |
||||
|
||||
First, all of the possible Huffman codes are counted, and reachable |
||||
intermediate states are noted by a non-zero count in a saved-results array. |
||||
Second, the intermediate states that lead to (root + 1) bit or longer codes |
||||
are used to look at all sub-codes from those junctures for their inflate |
||||
memory usage. (The amount of memory used is not affected by the number of |
||||
codes of root bits or less in length.) Third, the visited states in the |
||||
construction of those sub-codes and the associated calculation of the table |
||||
size is recalled in order to avoid recalculating from the same juncture. |
||||
Beginning the code examination at (root + 1) bit codes, which is enabled by |
||||
identifying the reachable nodes, accounts for about six of the orders of |
||||
magnitude of improvement for the default arguments. About another four |
||||
orders of magnitude come from not revisiting previous states. Out of |
||||
approximately 2x10^16 possible Huffman codes, only about 2x10^6 sub-codes |
||||
need to be examined to cover all of the possible table memory usage cases |
||||
for the default arguments of 286 symbols limited to 15-bit codes. |
||||
|
||||
Note that an unsigned long long type is used for counting. It is quite easy |
||||
to exceed the capacity of an eight-byte integer with a large number of |
||||
symbols and a large maximum code length, so multiple-precision arithmetic |
||||
would need to replace the unsigned long long arithmetic in that case. This |
||||
program will abort if an overflow occurs. The big_t type identifies where |
||||
the counting takes place. |
||||
|
||||
An unsigned long long type is also used for calculating the number of |
||||
possible codes remaining at the maximum length. This limits the maximum |
||||
code length to the number of bits in a long long minus the number of bits |
||||
needed to represent the symbols in a flat code. The code_t type identifies |
||||
where the bit pattern counting takes place. |
||||
*/ |
||||
|
||||
#include <stdio.h> |
||||
#include <stdlib.h> |
||||
#include <string.h> |
||||
#include <assert.h> |
||||
|
||||
#define local static |
||||
|
||||
/* special data types */ |
||||
typedef unsigned long long big_t; /* type for code counting */ |
||||
typedef unsigned long long code_t; /* type for bit pattern counting */ |
||||
struct tab { /* type for been here check */ |
||||
size_t len; /* length of bit vector in char's */ |
||||
char *vec; /* allocated bit vector */ |
||||
}; |
||||
|
||||
/* The array for saving results, num[], is indexed with this triplet:
|
||||
|
||||
syms: number of symbols remaining to code |
||||
left: number of available bit patterns at length len |
||||
len: number of bits in the codes currently being assigned |
||||
|
||||
Those indices are constrained thusly when saving results: |
||||
|
||||
syms: 3..totsym (totsym == total symbols to code) |
||||
left: 2..syms - 1, but only the evens (so syms == 8 -> 2, 4, 6) |
||||
len: 1..max - 1 (max == maximum code length in bits) |
||||
|
||||
syms == 2 is not saved since that immediately leads to a single code. left |
||||
must be even, since it represents the number of available bit patterns at |
||||
the current length, which is double the number at the previous length. |
||||
left ends at syms-1 since left == syms immediately results in a single code. |
||||
(left > sym is not allowed since that would result in an incomplete code.) |
||||
len is less than max, since the code completes immediately when len == max. |
||||
|
||||
The offset into the array is calculated for the three indices with the |
||||
first one (syms) being outermost, and the last one (len) being innermost. |
||||
We build the array with length max-1 lists for the len index, with syms-3 |
||||
of those for each symbol. There are totsym-2 of those, with each one |
||||
varying in length as a function of sym. See the calculation of index in |
||||
count() for the index, and the calculation of size in main() for the size |
||||
of the array. |
||||
|
||||
For the deflate example of 286 symbols limited to 15-bit codes, the array |
||||
has 284,284 entries, taking up 2.17 MB for an 8-byte big_t. More than |
||||
half of the space allocated for saved results is actually used -- not all |
||||
possible triplets are reached in the generation of valid Huffman codes.
|
||||
*/ |
||||
|
||||
/* The array for tracking visited states, done[], is itself indexed identically
|
||||
to the num[] array as described above for the (syms, left, len) triplet. |
||||
Each element in the array is further indexed by the (mem, rem) doublet, |
||||
where mem is the amount of inflate table space used so far, and rem is the |
||||
remaining unused entries in the current inflate sub-table. Each indexed |
||||
element is simply one bit indicating whether the state has been visited or |
||||
not. Since the ranges for mem and rem are not known a priori, each bit |
||||
vector is of a variable size, and grows as needed to accommodate the visited |
||||
states. mem and rem are used to calculate a single index in a triangular |
||||
array. Since the range of mem is expected in the default case to be about |
||||
ten times larger than the range of rem, the array is skewed to reduce the |
||||
memory usage, with eight times the range for mem than for rem. See the |
||||
calculations for offset and bit in beenhere() for the details. |
||||
|
||||
For the deflate example of 286 symbols limited to 15-bit codes, the bit |
||||
vectors grow to total approximately 21 MB, in addition to the 4.3 MB done[] |
||||
array itself. |
||||
*/ |
||||
|
||||
/* Globals to avoid propagating constants or constant pointers recursively */ |
||||
local int max; /* maximum allowed bit length for the codes */ |
||||
local int root; /* size of base code table in bits */ |
||||
local int large; /* largest code table so far */ |
||||
local size_t size; /* number of elements in num and done */ |
||||
local int *code; /* number of symbols assigned to each bit length */ |
||||
local big_t *num; /* saved results array for code counting */ |
||||
local struct tab *done; /* states already evaluated array */ |
||||
|
||||
/* Index function for num[] and done[] */ |
||||
#define INDEX(i,j,k) (((size_t)((i-1)>>1)*((i-2)>>1)+(j>>1)-1)*(max-1)+k-1) |
||||
|
||||
/* Free allocated space. Uses globals code, num, and done. */ |
||||
local void cleanup(void) |
||||
{ |
||||
size_t n; |
||||
|
||||
if (done != NULL) { |
||||
for (n = 0; n < size; n++) |
||||
if (done[n].len) |
||||
free(done[n].vec); |
||||
free(done); |
||||
} |
||||
if (num != NULL) |
||||
free(num); |
||||
if (code != NULL) |
||||
free(code); |
||||
} |
||||
|
||||
/* Return the number of possible Huffman codes using bit patterns of lengths
|
||||
len through max inclusive, coding syms symbols, with left bit patterns of |
||||
length len unused -- return -1 if there is an overflow in the counting. |
||||
Keep a record of previous results in num to prevent repeating the same |
||||
calculation. Uses the globals max and num. */ |
||||
local big_t count(int syms, int len, int left) |
||||
{ |
||||
big_t sum; /* number of possible codes from this juncture */ |
||||
big_t got; /* value returned from count() */ |
||||
int least; /* least number of syms to use at this juncture */ |
||||
int most; /* most number of syms to use at this juncture */ |
||||
int use; /* number of bit patterns to use in next call */ |
||||
size_t index; /* index of this case in *num */ |
||||
|
||||
/* see if only one possible code */ |
||||
if (syms == left) |
||||
return 1; |
||||
|
||||
/* note and verify the expected state */ |
||||
assert(syms > left && left > 0 && len < max); |
||||
|
||||
/* see if we've done this one already */ |
||||
index = INDEX(syms, left, len); |
||||
got = num[index]; |
||||
if (got) |
||||
return got; /* we have -- return the saved result */ |
||||
|
||||
/* we need to use at least this many bit patterns so that the code won't be
|
||||
incomplete at the next length (more bit patterns than symbols) */ |
||||
least = (left << 1) - syms; |
||||
if (least < 0) |
||||
least = 0; |
||||
|
||||
/* we can use at most this many bit patterns, lest there not be enough
|
||||
available for the remaining symbols at the maximum length (if there were |
||||
no limit to the code length, this would become: most = left - 1) */ |
||||
most = (((code_t)left << (max - len)) - syms) / |
||||
(((code_t)1 << (max - len)) - 1); |
||||
|
||||
/* count all possible codes from this juncture and add them up */ |
||||
sum = 0; |
||||
for (use = least; use <= most; use++) { |
||||
got = count(syms - use, len + 1, (left - use) << 1); |
||||
sum += got; |
||||
if (got == -1 || sum < got) /* overflow */ |
||||
return -1; |
||||
} |
||||
|
||||
/* verify that all recursive calls are productive */ |
||||
assert(sum != 0); |
||||
|
||||
/* save the result and return it */ |
||||
num[index] = sum; |
||||
return sum; |
||||
} |
||||
|
||||
/* Return true if we've been here before, set to true if not. Set a bit in a
|
||||
bit vector to indicate visiting this state. Each (syms,len,left) state |
||||
has a variable size bit vector indexed by (mem,rem). The bit vector is |
||||
lengthened if needed to allow setting the (mem,rem) bit. */ |
||||
local int beenhere(int syms, int len, int left, int mem, int rem) |
||||
{ |
||||
size_t index; /* index for this state's bit vector */ |
||||
size_t offset; /* offset in this state's bit vector */ |
||||
int bit; /* mask for this state's bit */ |
||||
size_t length; /* length of the bit vector in bytes */ |
||||
char *vector; /* new or enlarged bit vector */ |
||||
|
||||
/* point to vector for (syms,left,len), bit in vector for (mem,rem) */ |
||||
index = INDEX(syms, left, len); |
||||
mem -= 1 << root; |
||||
offset = (mem >> 3) + rem; |
||||
offset = ((offset * (offset + 1)) >> 1) + rem; |
||||
bit = 1 << (mem & 7); |
||||
|
||||
/* see if we've been here */ |
||||
length = done[index].len; |
||||
if (offset < length && (done[index].vec[offset] & bit) != 0) |
||||
return 1; /* done this! */ |
||||
|
||||
/* we haven't been here before -- set the bit to show we have now */ |
||||
|
||||
/* see if we need to lengthen the vector in order to set the bit */ |
||||
if (length <= offset) { |
||||
/* if we have one already, enlarge it, zero out the appended space */ |
||||
if (length) { |
||||
do { |
||||
length <<= 1; |
||||
} while (length <= offset); |
||||
vector = realloc(done[index].vec, length); |
||||
if (vector != NULL) |
||||
memset(vector + done[index].len, 0, length - done[index].len); |
||||
} |
||||
|
||||
/* otherwise we need to make a new vector and zero it out */ |
||||
else { |
||||
length = 1 << (len - root); |
||||
while (length <= offset) |
||||
length <<= 1; |
||||
vector = calloc(length, sizeof(char)); |
||||
} |
||||
|
||||
/* in either case, bail if we can't get the memory */ |
||||
if (vector == NULL) { |
||||
fputs("abort: unable to allocate enough memory\n", stderr); |
||||
cleanup(); |
||||
exit(1); |
||||
} |
||||
|
||||
/* install the new vector */ |
||||
done[index].len = length; |
||||
done[index].vec = vector; |
||||
} |
||||
|
||||
/* set the bit */ |
||||
done[index].vec[offset] |= bit; |
||||
return 0; |
||||
} |
||||
|
||||
/* Examine all possible codes from the given node (syms, len, left). Compute
|
||||
the amount of memory required to build inflate's decoding tables, where the |
||||
number of code structures used so far is mem, and the number remaining in |
||||
the current sub-table is rem. Uses the globals max, code, root, large, and |
||||
done. */ |
||||
local void examine(int syms, int len, int left, int mem, int rem) |
||||
{ |
||||
int least; /* least number of syms to use at this juncture */ |
||||
int most; /* most number of syms to use at this juncture */ |
||||
int use; /* number of bit patterns to use in next call */ |
||||
|
||||
/* see if we have a complete code */ |
||||
if (syms == left) { |
||||
/* set the last code entry */ |
||||
code[len] = left; |
||||
|
||||
/* complete computation of memory used by this code */ |
||||
while (rem < left) { |
||||
left -= rem; |
||||
rem = 1 << (len - root); |
||||
mem += rem; |
||||
} |
||||
assert(rem == left); |
||||
|
||||
/* if this is a new maximum, show the entries used and the sub-code */ |
||||
if (mem > large) { |
||||
large = mem; |
||||
printf("max %d: ", mem); |
||||
for (use = root + 1; use <= max; use++) |
||||
if (code[use]) |
||||
printf("%d[%d] ", code[use], use); |
||||
putchar('\n'); |
||||
fflush(stdout); |
||||
} |
||||
|
||||
/* remove entries as we drop back down in the recursion */ |
||||
code[len] = 0; |
||||
return; |
||||
} |
||||
|
||||
/* prune the tree if we can */ |
||||
if (beenhere(syms, len, left, mem, rem)) |
||||
return; |
||||
|
||||
/* we need to use at least this many bit patterns so that the code won't be
|
||||
incomplete at the next length (more bit patterns than symbols) */ |
||||
least = (left << 1) - syms; |
||||
if (least < 0) |
||||
least = 0; |
||||
|
||||
/* we can use at most this many bit patterns, lest there not be enough
|
||||
available for the remaining symbols at the maximum length (if there were |
||||
no limit to the code length, this would become: most = left - 1) */ |
||||
most = (((code_t)left << (max - len)) - syms) / |
||||
(((code_t)1 << (max - len)) - 1); |
||||
|
||||
/* occupy least table spaces, creating new sub-tables as needed */ |
||||
use = least; |
||||
while (rem < use) { |
||||
use -= rem; |
||||
rem = 1 << (len - root); |
||||
mem += rem; |
||||
} |
||||
rem -= use; |
||||
|
||||
/* examine codes from here, updating table space as we go */ |
||||
for (use = least; use <= most; use++) { |
||||
code[len] = use; |
||||
examine(syms - use, len + 1, (left - use) << 1, |
||||
mem + (rem ? 1 << (len - root) : 0), rem << 1); |
||||
if (rem == 0) { |
||||
rem = 1 << (len - root); |
||||
mem += rem; |
||||
} |
||||
rem--; |
||||
} |
||||
|
||||
/* remove entries as we drop back down in the recursion */ |
||||
code[len] = 0; |
||||
} |
||||
|
||||
/* Look at all sub-codes starting with root + 1 bits. Look at only the valid
|
||||
intermediate code states (syms, left, len). For each completed code, |
||||
calculate the amount of memory required by inflate to build the decoding |
||||
tables. Find the maximum amount of memory required and show the code that |
||||
requires that maximum. Uses the globals max, root, and num. */ |
||||
local void enough(int syms) |
||||
{ |
||||
int n; /* number of remaing symbols for this node */ |
||||
int left; /* number of unused bit patterns at this length */ |
||||
size_t index; /* index of this case in *num */ |
||||
|
||||
/* clear code */ |
||||
for (n = 0; n <= max; n++) |
||||
code[n] = 0; |
||||
|
||||
/* look at all (root + 1) bit and longer codes */ |
||||
large = 1 << root; /* base table */ |
||||
if (root < max) /* otherwise, there's only a base table */ |
||||
for (n = 3; n <= syms; n++) |
||||
for (left = 2; left < n; left += 2) |
||||
{ |
||||
/* look at all reachable (root + 1) bit nodes, and the
|
||||
resulting codes (complete at root + 2 or more) */ |
||||
index = INDEX(n, left, root + 1); |
||||
if (root + 1 < max && num[index]) /* reachable node */ |
||||
examine(n, root + 1, left, 1 << root, 0); |
||||
|
||||
/* also look at root bit codes with completions at root + 1
|
||||
bits (not saved in num, since complete), just in case */ |
||||
if (num[index - 1] && n <= left << 1) |
||||
examine((n - left) << 1, root + 1, (n - left) << 1, |
||||
1 << root, 0); |
||||
} |
||||
|
||||
/* done */ |
||||
printf("done: maximum of %d table entries\n", large); |
||||
} |
||||
|
||||
/*
|
||||
Examine and show the total number of possible Huffman codes for a given |
||||
maximum number of symbols, initial root table size, and maximum code length |
||||
in bits -- those are the command arguments in that order. The default |
||||
values are 286, 9, and 15 respectively, for the deflate literal/length code. |
||||
The possible codes are counted for each number of coded symbols from two to |
||||
the maximum. The counts for each of those and the total number of codes are |
||||
shown. The maximum number of inflate table entires is then calculated |
||||
across all possible codes. Each new maximum number of table entries and the |
||||
associated sub-code (starting at root + 1 == 10 bits) is shown. |
||||
|
||||
To count and examine Huffman codes that are not length-limited, provide a |
||||
maximum length equal to the number of symbols minus one. |
||||
|
||||
For the deflate literal/length code, use "enough". For the deflate distance |
||||
code, use "enough 30 6". |
||||
|
||||
This uses the %llu printf format to print big_t numbers, which assumes that |
||||
big_t is an unsigned long long. If the big_t type is changed (for example |
||||
to a multiple precision type), the method of printing will also need to be |
||||
updated. |
||||
*/ |
||||
int main(int argc, char **argv) |
||||
{ |
||||
int syms; /* total number of symbols to code */ |
||||
int n; /* number of symbols to code for this run */ |
||||
big_t got; /* return value of count() */ |
||||
big_t sum; /* accumulated number of codes over n */ |
||||
|
||||
/* set up globals for cleanup() */ |
||||
code = NULL; |
||||
num = NULL; |
||||
done = NULL; |
||||
|
||||
/* get arguments -- default to the deflate literal/length code */ |
||||
syms = 286; |
||||
root = 9; |
||||
max = 15; |
||||
if (argc > 1) { |
||||
syms = atoi(argv[1]); |
||||
if (argc > 2) { |
||||
root = atoi(argv[2]); |
||||
if (argc > 3) |
||||
max = atoi(argv[3]); |
||||
} |
||||
} |
||||
if (argc > 4 || syms < 2 || root < 1 || max < 1) { |
||||
fputs("invalid arguments, need: [sym >= 2 [root >= 1 [max >= 1]]]\n", |
||||
stderr); |
||||
return 1; |
||||
} |
||||
|
||||
/* if not restricting the code length, the longest is syms - 1 */ |
||||
if (max > syms - 1) |
||||
max = syms - 1; |
||||
|
||||
/* determine the number of bits in a code_t */ |
||||
n = 0; |
||||
while (((code_t)1 << n) != 0) |
||||
n++; |
||||
|
||||
/* make sure that the calculation of most will not overflow */ |
||||
if (max > n || syms - 2 >= (((code_t)0 - 1) >> (max - 1))) { |
||||
fputs("abort: code length too long for internal types\n", stderr); |
||||
return 1; |
||||
} |
||||
|
||||
/* reject impossible code requests */ |
||||
if (syms - 1 > ((code_t)1 << max) - 1) { |
||||
fprintf(stderr, "%d symbols cannot be coded in %d bits\n", |
||||
syms, max); |
||||
return 1; |
||||
} |
||||
|
||||
/* allocate code vector */ |
||||
code = calloc(max + 1, sizeof(int)); |
||||
if (code == NULL) { |
||||
fputs("abort: unable to allocate enough memory\n", stderr); |
||||
return 1; |
||||
} |
||||
|
||||
/* determine size of saved results array, checking for overflows,
|
||||
allocate and clear the array (set all to zero with calloc()) */ |
||||
if (syms == 2) /* iff max == 1 */ |
||||
num = NULL; /* won't be saving any results */ |
||||
else { |
||||
size = syms >> 1; |
||||
if (size > ((size_t)0 - 1) / (n = (syms - 1) >> 1) || |
||||
(size *= n, size > ((size_t)0 - 1) / (n = max - 1)) || |
||||
(size *= n, size > ((size_t)0 - 1) / sizeof(big_t)) || |
||||
(num = calloc(size, sizeof(big_t))) == NULL) { |
||||
fputs("abort: unable to allocate enough memory\n", stderr); |
||||
cleanup(); |
||||
return 1; |
||||
} |
||||
} |
||||
|
||||
/* count possible codes for all numbers of symbols, add up counts */ |
||||
sum = 0; |
||||
for (n = 2; n <= syms; n++) { |
||||
got = count(n, 1, 2); |
||||
sum += got; |
||||
if (got == -1 || sum < got) { /* overflow */ |
||||
fputs("abort: can't count that high!\n", stderr); |
||||
cleanup(); |
||||
return 1; |
||||
} |
||||
printf("%llu %d-codes\n", got, n); |
||||
} |
||||
printf("%llu total codes for 2 to %d symbols", sum, syms); |
||||
if (max < syms - 1) |
||||
printf(" (%d-bit length limit)\n", max); |
||||
else |
||||
puts(" (no length limit)"); |
||||
|
||||
/* allocate and clear done array for beenhere() */ |
||||
if (syms == 2) |
||||
done = NULL; |
||||
else if (size > ((size_t)0 - 1) / sizeof(struct tab) || |
||||
(done = calloc(size, sizeof(struct tab))) == NULL) { |
||||
fputs("abort: unable to allocate enough memory\n", stderr); |
||||
cleanup(); |
||||
return 1; |
||||
} |
||||
|
||||
/* find and show maximum inflate table usage */ |
||||
if (root > max) /* reduce root to max length */ |
||||
root = max; |
||||
if (syms < ((code_t)1 << (root + 1))) |
||||
enough(syms); |
||||
else |
||||
puts("cannot handle minimum code lengths > root"); |
||||
|
||||
/* done */ |
||||
cleanup(); |
||||
return 0; |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,452 @@ |
||||
/* pigz.c -- parallel implementation of gzip
|
||||
* Copyright (C) 2007 Mark Adler |
||||
* Version 1.1 28 January 2007 Mark Adler |
||||
*/ |
||||
|
||||
/* Version history:
|
||||
1.0 17 Jan 2007 First version |
||||
1.1 28 Jan 2007 Avoid void * arithmetic (some compilers don't get that) |
||||
Add note about requiring zlib 1.2.3 |
||||
Allow compression level 0 (no compression) |
||||
Completely rewrite parallelism -- add a write thread |
||||
Use deflateSetDictionary() to make use of history |
||||
Tune argument defaults to best performance on four cores |
||||
*/ |
||||
|
||||
/*
|
||||
pigz compresses from stdin to stdout using threads to make use of multiple |
||||
processors and cores. The input is broken up into 128 KB chunks, and each |
||||
is compressed separately. The CRC for each chunk is also calculated |
||||
separately. The compressed chunks are written in order to the output, |
||||
and the overall CRC is calculated from the CRC's of the chunks. |
||||
|
||||
The compressed data format generated is the gzip format using the deflate |
||||
compression method. First a gzip header is written, followed by raw deflate |
||||
partial streams. They are partial, in that they do not have a terminating |
||||
block. At the end, the deflate stream is terminated with a final empty |
||||
static block, and lastly a gzip trailer is written with the CRC and the |
||||
number of input bytes. |
||||
|
||||
Each raw deflate partial stream is terminated by an empty stored block |
||||
(using the Z_SYNC_FLUSH option of zlib), in order to end that partial |
||||
bit stream at a byte boundary. That allows the partial streams to be |
||||
concantenated simply as sequences of bytes. This adds a very small four |
||||
or five byte overhead to the output for each input chunk. |
||||
|
||||
zlib's crc32_combine() routine allows the calcuation of the CRC of the |
||||
entire input using the independent CRC's of the chunks. pigz requires zlib |
||||
version 1.2.3 or later, since that is the first version that provides the |
||||
crc32_combine() function. |
||||
|
||||
pigz uses the POSIX pthread library for thread control and communication. |
||||
*/ |
||||
|
||||
#include <stdio.h> |
||||
#include <stdlib.h> |
||||
#include <string.h> |
||||
#include <pthread.h> |
||||
#include <sys/types.h> |
||||
#include <sys/uio.h> |
||||
#include <unistd.h> |
||||
#include "zlib.h" |
||||
|
||||
#define local static |
||||
|
||||
/* exit with error */ |
||||
local void bail(char *msg) |
||||
{ |
||||
fprintf(stderr, "pigz abort: %s\n", msg); |
||||
exit(1); |
||||
} |
||||
|
||||
/* read up to len bytes into buf, repeating read() calls as needed */ |
||||
local size_t readn(int desc, unsigned char *buf, size_t len) |
||||
{ |
||||
ssize_t ret; |
||||
size_t got; |
||||
|
||||
got = 0; |
||||
while (len) { |
||||
ret = read(desc, buf, len); |
||||
if (ret < 0) |
||||
bail("read error"); |
||||
if (ret == 0) |
||||
break; |
||||
buf += ret; |
||||
len -= ret; |
||||
got += ret; |
||||
} |
||||
return got; |
||||
} |
||||
|
||||
/* write len bytes, repeating write() calls as needed */ |
||||
local void writen(int desc, unsigned char *buf, size_t len) |
||||
{ |
||||
ssize_t ret; |
||||
|
||||
while (len) { |
||||
ret = write(desc, buf, len); |
||||
if (ret < 1) |
||||
bail("write error"); |
||||
buf += ret; |
||||
len -= ret; |
||||
} |
||||
} |
||||
|
||||
/* a flag variable for communication between two threads */ |
||||
struct flag { |
||||
int value; /* value of flag */ |
||||
pthread_mutex_t lock; /* lock for checking and changing flag */ |
||||
pthread_cond_t cond; /* condition for signaling on flag change */ |
||||
}; |
||||
|
||||
/* initialize a flag for use, starting with value val */ |
||||
local void flag_init(struct flag *me, int val) |
||||
{ |
||||
me->value = val; |
||||
pthread_mutex_init(&(me->lock), NULL); |
||||
pthread_cond_init(&(me->cond), NULL); |
||||
} |
||||
|
||||
/* set the flag to val, signal another process that may be waiting for it */ |
||||
local void flag_set(struct flag *me, int val) |
||||
{ |
||||
pthread_mutex_lock(&(me->lock)); |
||||
me->value = val; |
||||
pthread_cond_signal(&(me->cond)); |
||||
pthread_mutex_unlock(&(me->lock)); |
||||
} |
||||
|
||||
/* if it isn't already, wait for some other thread to set the flag to val */ |
||||
local void flag_wait(struct flag *me, int val) |
||||
{ |
||||
pthread_mutex_lock(&(me->lock)); |
||||
while (me->value != val) |
||||
pthread_cond_wait(&(me->cond), &(me->lock)); |
||||
pthread_mutex_unlock(&(me->lock)); |
||||
} |
||||
|
||||
/* if flag is equal to val, wait for some other thread to change it */ |
||||
local void flag_wait_not(struct flag *me, int val) |
||||
{ |
||||
pthread_mutex_lock(&(me->lock)); |
||||
while (me->value == val) |
||||
pthread_cond_wait(&(me->cond), &(me->lock)); |
||||
pthread_mutex_unlock(&(me->lock)); |
||||
} |
||||
|
||||
/* clean up the flag when done with it */ |
||||
local void flag_done(struct flag *me) |
||||
{ |
||||
pthread_cond_destroy(&(me->cond)); |
||||
pthread_mutex_destroy(&(me->lock)); |
||||
} |
||||
|
||||
/* a unit of work to feed to compress_thread() -- it is assumed that the out
|
||||
buffer is large enough to hold the maximum size len bytes could deflate to, |
||||
plus five bytes for the final sync marker */ |
||||
struct work { |
||||
size_t len; /* length of input */ |
||||
unsigned long crc; /* crc of input */ |
||||
unsigned char *buf; /* input */ |
||||
unsigned char *out; /* space for output (guaranteed big enough) */ |
||||
z_stream strm; /* pre-initialized z_stream */ |
||||
struct flag busy; /* busy flag indicating work unit in use */ |
||||
pthread_t comp; /* this compression thread */ |
||||
}; |
||||
|
||||
/* busy flag values */ |
||||
#define IDLE 0 /* compress and writing done -- can start compress */ |
||||
#define COMP 1 /* compress -- input and output buffers in use */ |
||||
#define WRITE 2 /* compress done, writing output -- can read input */ |
||||
|
||||
/* read-only globals (set by main/read thread before others started) */ |
||||
local int ind; /* input file descriptor */ |
||||
local int outd; /* output file descriptor */ |
||||
local int level; /* compression level */ |
||||
local int procs; /* number of compression threads (>= 2) */ |
||||
local size_t size; /* uncompressed input size per thread (>= 32K) */ |
||||
local struct work *jobs; /* work units: jobs[0..procs-1] */ |
||||
|
||||
/* next and previous jobs[] indices */ |
||||
#define NEXT(n) ((n) == procs - 1 ? 0 : (n) + 1) |
||||
#define PREV(n) ((n) == 0 ? procs - 1 : (n) - 1) |
||||
|
||||
/* sliding dictionary size for deflate */ |
||||
#define DICT 32768U |
||||
|
||||
/* largest power of 2 that fits in an unsigned int -- used to limit requests
|
||||
to zlib functions that use unsigned int lengths */ |
||||
#define MAX ((((unsigned)-1) >> 1) + 1) |
||||
|
||||
/* compress thread: compress the input in the provided work unit and compute
|
||||
its crc -- assume that the amount of space at job->out is guaranteed to be |
||||
enough for the compressed output, as determined by the maximum expansion |
||||
of deflate compression -- use the input in the previous work unit (if there |
||||
is one) to set the deflate dictionary for better compression */ |
||||
local void *compress_thread(void *arg) |
||||
{ |
||||
size_t len; /* input length for this work unit */ |
||||
unsigned long crc; /* crc of input data */ |
||||
struct work *prev; /* previous work unit */ |
||||
struct work *job = arg; /* work unit for this thread */ |
||||
z_stream *strm = &(job->strm); /* zlib stream for this work unit */ |
||||
|
||||
/* reset state for a new compressed stream */ |
||||
(void)deflateReset(strm); |
||||
|
||||
/* initialize input, output, and crc */ |
||||
strm->next_in = job->buf; |
||||
strm->next_out = job->out; |
||||
len = job->len; |
||||
crc = crc32(0L, Z_NULL, 0); |
||||
|
||||
/* set dictionary if this isn't the first work unit, and if we will be
|
||||
compressing something (the read thread assures that the dictionary |
||||
data in the previous work unit is still there) */ |
||||
prev = jobs + PREV(job - jobs); |
||||
if (prev->buf != NULL && len != 0) |
||||
deflateSetDictionary(strm, prev->buf + (size - DICT), DICT); |
||||
|
||||
/* run MAX-sized amounts of input through deflate and crc32 -- this loop
|
||||
is needed for those cases where the integer type is smaller than the |
||||
size_t type, or when len is close to the limit of the size_t type */ |
||||
while (len > MAX) { |
||||
strm->avail_in = MAX; |
||||
strm->avail_out = (unsigned)-1; |
||||
crc = crc32(crc, strm->next_in, strm->avail_in); |
||||
(void)deflate(strm, Z_NO_FLUSH); |
||||
len -= MAX; |
||||
} |
||||
|
||||
/* run last piece through deflate and crc32, follow with a sync marker */ |
||||
if (len) { |
||||
strm->avail_in = len; |
||||
strm->avail_out = (unsigned)-1; |
||||
crc = crc32(crc, strm->next_in, strm->avail_in); |
||||
(void)deflate(strm, Z_SYNC_FLUSH); |
||||
} |
||||
|
||||
/* don't need to Z_FINISH, since we'd delete the last two bytes anyway */ |
||||
|
||||
/* return result */ |
||||
job->crc = crc; |
||||
return NULL; |
||||
} |
||||
|
||||
/* put a 4-byte integer into a byte array in LSB order */ |
||||
#define PUT4(a,b) (*(a)=(b),(a)[1]=(b)>>8,(a)[2]=(b)>>16,(a)[3]=(b)>>24) |
||||
|
||||
/* write thread: wait for compression threads to complete, write output in
|
||||
order, also write gzip header and trailer around the compressed data */ |
||||
local void *write_thread(void *arg) |
||||
{ |
||||
int n; /* compress thread index */ |
||||
size_t len; /* length of input processed */ |
||||
unsigned long tot; /* total uncompressed size (overflow ok) */ |
||||
unsigned long crc; /* CRC-32 of uncompressed data */ |
||||
unsigned char wrap[10]; /* gzip header or trailer */ |
||||
|
||||
/* write simple gzip header */ |
||||
memcpy(wrap, "\037\213\10\0\0\0\0\0\0\3", 10); |
||||
wrap[8] = level == 9 ? 2 : (level == 1 ? 4 : 0); |
||||
writen(outd, wrap, 10); |
||||
|
||||
/* process output of compress threads until end of input */
|
||||
tot = 0; |
||||
crc = crc32(0L, Z_NULL, 0); |
||||
n = 0; |
||||
do { |
||||
/* wait for compress thread to start, then wait to complete */ |
||||
flag_wait(&(jobs[n].busy), COMP); |
||||
pthread_join(jobs[n].comp, NULL); |
||||
|
||||
/* now that compress is done, allow read thread to use input buffer */ |
||||
flag_set(&(jobs[n].busy), WRITE); |
||||
|
||||
/* write compressed data and update length and crc */ |
||||
writen(outd, jobs[n].out, jobs[n].strm.next_out - jobs[n].out); |
||||
len = jobs[n].len; |
||||
tot += len; |
||||
crc = crc32_combine(crc, jobs[n].crc, len); |
||||
|
||||
/* release this work unit and go to the next work unit */ |
||||
flag_set(&(jobs[n].busy), IDLE); |
||||
n = NEXT(n); |
||||
|
||||
/* an input buffer less than size in length indicates end of input */ |
||||
} while (len == size); |
||||
|
||||
/* write final static block and gzip trailer (crc and len mod 2^32) */ |
||||
wrap[0] = 3; wrap[1] = 0; |
||||
PUT4(wrap + 2, crc); |
||||
PUT4(wrap + 6, tot); |
||||
writen(outd, wrap, 10); |
||||
return NULL; |
||||
} |
||||
|
||||
/* one-time initialization of a work unit -- this is where we set the deflate
|
||||
compression level and request raw deflate, and also where we set the size |
||||
of the output buffer to guarantee enough space for a worst-case deflate |
||||
ending with a Z_SYNC_FLUSH */ |
||||
local void job_init(struct work *job) |
||||
{ |
||||
int ret; /* deflateInit2() return value */ |
||||
|
||||
job->buf = malloc(size); |
||||
job->out = malloc(size + (size >> 11) + 10); |
||||
job->strm.zfree = Z_NULL; |
||||
job->strm.zalloc = Z_NULL; |
||||
job->strm.opaque = Z_NULL; |
||||
ret = deflateInit2(&(job->strm), level, Z_DEFLATED, -15, 8, |
||||
Z_DEFAULT_STRATEGY); |
||||
if (job->buf == NULL || job->out == NULL || ret != Z_OK) |
||||
bail("not enough memory"); |
||||
} |
||||
|
||||
/* compress ind to outd in the gzip format, using multiple threads for the
|
||||
compression and crc calculation and another thread for writing the output -- |
||||
the read thread is the main thread */ |
||||
local void read_thread(void) |
||||
{ |
||||
int n; /* general index */ |
||||
size_t got; /* amount read */ |
||||
pthread_attr_t attr; /* thread attributes (left at defaults) */ |
||||
pthread_t write; /* write thread */ |
||||
|
||||
/* set defaults (not all pthread implementations default to joinable) */ |
||||
pthread_attr_init(&attr); |
||||
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); |
||||
|
||||
/* allocate and set up work list (individual work units will be initialized
|
||||
as needed, in case the input is short), assure that allocation size |
||||
arithmetic does not overflow */ |
||||
if (size + (size >> 11) + 10 < (size >> 11) + 10 || |
||||
(ssize_t)(size + (size >> 11) + 10) < 0 || |
||||
((size_t)0 - 1) / procs <= sizeof(struct work) || |
||||
(jobs = malloc(procs * sizeof(struct work))) == NULL) |
||||
bail("not enough memory"); |
||||
for (n = 0; n < procs; n++) { |
||||
jobs[n].buf = NULL; |
||||
flag_init(&(jobs[n].busy), IDLE); |
||||
} |
||||
|
||||
/* start write thread */ |
||||
pthread_create(&write, &attr, write_thread, NULL); |
||||
|
||||
/* read from input and start compress threads (write thread will pick up
|
||||
the output of the compress threads) */ |
||||
n = 0; |
||||
do { |
||||
/* initialize this work unit if it's the first time it's used */ |
||||
if (jobs[n].buf == NULL) |
||||
job_init(jobs + n); |
||||
|
||||
/* read input data, but wait for last compress on this work unit to be
|
||||
done, and wait for the dictionary to be used by the last compress on |
||||
the next work unit */ |
||||
flag_wait_not(&(jobs[n].busy), COMP); |
||||
flag_wait_not(&(jobs[NEXT(n)].busy), COMP); |
||||
got = readn(ind, jobs[n].buf, size); |
||||
|
||||
/* start compress thread, but wait for write to be done first */ |
||||
flag_wait(&(jobs[n].busy), IDLE); |
||||
jobs[n].len = got; |
||||
pthread_create(&(jobs[n].comp), &attr, compress_thread, jobs + n); |
||||
|
||||
/* mark work unit so write thread knows compress was started */ |
||||
flag_set(&(jobs[n].busy), COMP); |
||||
|
||||
/* go to the next work unit */ |
||||
n = NEXT(n); |
||||
|
||||
/* do until end of input, indicated by a read less than size */ |
||||
} while (got == size); |
||||
|
||||
/* wait for the write thread to complete -- the write thread will join with
|
||||
all of the compress threads, so this waits for all of the threads to |
||||
complete */ |
||||
pthread_join(write, NULL); |
||||
|
||||
/* free up all requested resources and return */ |
||||
for (n = procs - 1; n >= 0; n--) { |
||||
flag_done(&(jobs[n].busy)); |
||||
(void)deflateEnd(&(jobs[n].strm)); |
||||
free(jobs[n].out); |
||||
free(jobs[n].buf); |
||||
} |
||||
free(jobs); |
||||
pthread_attr_destroy(&attr); |
||||
} |
||||
|
||||
/* Process arguments for level, size, and procs, compress from stdin to
|
||||
stdout in the gzip format. Note that procs must be at least two in |
||||
order to provide a dictionary in one work unit for the other work |
||||
unit, and that size must be at least 32K to store a full dictionary. */ |
||||
int main(int argc, char **argv) |
||||
{ |
||||
int n; /* general index */ |
||||
int get; /* command line parameters to get */ |
||||
char *arg; /* command line argument */ |
||||
|
||||
/* set defaults -- 32 processes and 128K buffers was found to provide
|
||||
good utilization of four cores (about 97%) and balanced the overall |
||||
execution time impact of more threads against more dictionary |
||||
processing for a fixed amount of memory -- the memory usage for these |
||||
settings and full use of all work units (at least 4 MB of input) is |
||||
16.2 MB |
||||
*/ |
||||
level = Z_DEFAULT_COMPRESSION; |
||||
procs = 32; |
||||
size = 131072UL; |
||||
|
||||
/* process command-line arguments */ |
||||
get = 0; |
||||
for (n = 1; n < argc; n++) { |
||||
arg = argv[n]; |
||||
if (*arg == '-') { |
||||
while (*++arg) |
||||
if (*arg >= '0' && *arg <= '9') /* compression level */ |
||||
level = *arg - '0'; |
||||
else if (*arg == 'b') /* chunk size in K */ |
||||
get |= 1; |
||||
else if (*arg == 'p') /* number of processes */ |
||||
get |= 2; |
||||
else if (*arg == 'h') { /* help */ |
||||
fputs("usage: pigz [-0..9] [-b blocksizeinK]", stderr); |
||||
fputs(" [-p processes] < foo > foo.gz\n", stderr); |
||||
return 0; |
||||
} |
||||
else |
||||
bail("invalid option"); |
||||
} |
||||
else if (get & 1) { |
||||
if (get & 2) |
||||
bail("you need to separate the -b and -p options"); |
||||
size = (size_t)(atol(arg)) << 10; /* chunk size */ |
||||
if (size < DICT) |
||||
bail("invalid option"); |
||||
get = 0; |
||||
} |
||||
else if (get & 2) { |
||||
procs = atoi(arg); /* processes */ |
||||
if (procs < 2) |
||||
bail("invalid option"); |
||||
get = 0; |
||||
} |
||||
else |
||||
bail("invalid option (you need to pipe input and output)"); |
||||
} |
||||
if (get) |
||||
bail("missing option argument"); |
||||
|
||||
/* do parallel compression from stdin to stdout (the read thread starts up
|
||||
the write thread and the compression threads, and they all join before |
||||
the read thread returns) */ |
||||
ind = 0; |
||||
outd = 1; |
||||
read_thread(); |
||||
|
||||
/* done */ |
||||
return 0; |
||||
} |
||||
@ -0,0 +1,152 @@ |
||||
#!/usr/bin/perl |
||||
|
||||
# Transform K&R C function definitions into ANSI equivalent. |
||||
# |
||||
# Author: Paul Marquess |
||||
# Version: 1.0 |
||||
# Date: 3 October 2006 |
||||
|
||||
# TODO |
||||
# |
||||
# Asumes no function pointer parameters. unless they are typedefed. |
||||
# Assumes no literal strings that look like function definitions |
||||
# Assumes functions start at the beginning of a line |
||||
|
||||
use strict; |
||||
use warnings; |
||||
|
||||
local $/; |
||||
$_ = <>; |
||||
|
||||
my $sp = qr{ \s* (?: /\* .*? \*/ )? \s* }x; # assume no nested comments |
||||
|
||||
my $d1 = qr{ $sp (?: [\w\*\s]+ $sp)* $sp \w+ $sp [\[\]\s]* $sp }x ; |
||||
my $decl = qr{ $sp (?: \w+ $sp )+ $d1 }xo ; |
||||
my $dList = qr{ $sp $decl (?: $sp , $d1 )* $sp ; $sp }xo ; |
||||
|
||||
|
||||
while (s/^ |
||||
( # Start $1 |
||||
( # Start $2 |
||||
.*? # Minimal eat content |
||||
( ^ \w [\w\s\*]+ ) # $3 -- function name |
||||
\s* # optional whitespace |
||||
) # $2 - Matched up to before parameter list |
||||
|
||||
\( \s* # Literal "(" + optional whitespace |
||||
( [^\)]+ ) # $4 - one or more anythings except ")" |
||||
\s* \) # optional whitespace surrounding a Literal ")" |
||||
|
||||
( (?: $dList )+ ) # $5 |
||||
|
||||
$sp ^ { # literal "{" at start of line |
||||
) # Remember to $1 |
||||
//xsom |
||||
) |
||||
{ |
||||
my $all = $1 ; |
||||
my $prefix = $2; |
||||
my $param_list = $4 ; |
||||
my $params = $5; |
||||
|
||||
StripComments($params); |
||||
StripComments($param_list); |
||||
$param_list =~ s/^\s+//; |
||||
$param_list =~ s/\s+$//; |
||||
|
||||
my $i = 0 ; |
||||
my %pList = map { $_ => $i++ } |
||||
split /\s*,\s*/, $param_list; |
||||
my $pMatch = '(\b' . join('|', keys %pList) . '\b)\W*$' ; |
||||
|
||||
my @params = split /\s*;\s*/, $params; |
||||
my @outParams = (); |
||||
foreach my $p (@params) |
||||
{ |
||||
if ($p =~ /,/) |
||||
{ |
||||
my @bits = split /\s*,\s*/, $p; |
||||
my $first = shift @bits; |
||||
$first =~ s/^\s*//; |
||||
push @outParams, $first; |
||||
$first =~ /^(\w+\s*)/; |
||||
my $type = $1 ; |
||||
push @outParams, map { $type . $_ } @bits; |
||||
} |
||||
else |
||||
{ |
||||
$p =~ s/^\s+//; |
||||
push @outParams, $p; |
||||
} |
||||
} |
||||
|
||||
|
||||
my %tmp = map { /$pMatch/; $_ => $pList{$1} } |
||||
@outParams ; |
||||
|
||||
@outParams = map { " $_" } |
||||
sort { $tmp{$a} <=> $tmp{$b} } |
||||
@outParams ; |
||||
|
||||
print $prefix ; |
||||
print "(\n" . join(",\n", @outParams) . ")\n"; |
||||
print "{" ; |
||||
|
||||
} |
||||
|
||||
# Output any trailing code. |
||||
print ; |
||||
exit 0; |
||||
|
||||
|
||||
sub StripComments |
||||
{ |
||||
|
||||
no warnings; |
||||
|
||||
# Strip C & C++ coments |
||||
# From the perlfaq |
||||
$_[0] =~ |
||||
|
||||
s{ |
||||
/\* ## Start of /* ... */ comment |
||||
[^*]*\*+ ## Non-* followed by 1-or-more *'s |
||||
( |
||||
[^/*][^*]*\*+ |
||||
)* ## 0-or-more things which don't start with / |
||||
## but do end with '*' |
||||
/ ## End of /* ... */ comment |
||||
|
||||
| ## OR C++ Comment |
||||
// ## Start of C++ comment // |
||||
[^\n]* ## followed by 0-or-more non end of line characters |
||||
|
||||
| ## OR various things which aren't comments: |
||||
|
||||
( |
||||
" ## Start of " ... " string |
||||
( |
||||
\\. ## Escaped char |
||||
| ## OR |
||||
[^"\\] ## Non "\ |
||||
)* |
||||
" ## End of " ... " string |
||||
|
||||
| ## OR |
||||
|
||||
' ## Start of ' ... ' string |
||||
( |
||||
\\. ## Escaped char |
||||
| ## OR |
||||
[^'\\] ## Non '\ |
||||
)* |
||||
' ## End of ' ... ' string |
||||
|
||||
| ## OR |
||||
|
||||
. ## Anything other char |
||||
[^/"'\\]* ## Chars which doesn't start a comment, string or escape |
||||
) |
||||
}{$2}gxs; |
||||
|
||||
} |
||||
Loading…
Reference in new issue