mirror of https://github.com/madler/zlib.git
Mark Adler
13 years ago
parent
639be99788
commit
f6194ef39a
45 changed files with 3642 additions and 852 deletions
@ -0,0 +1,357 @@ |
|||||||
|
/* |
||||||
|
* match.S -- optimized version of longest_match() |
||||||
|
* based on the similar work by Gilles Vollant, and Brian Raiter, written 1998 |
||||||
|
* |
||||||
|
* This is free software; you can redistribute it and/or modify it
|
||||||
|
* under the terms of the BSD License. Use by owners of Che Guevarra |
||||||
|
* parafernalia is prohibited, where possible, and highly discouraged |
||||||
|
* elsewhere. |
||||||
|
*/ |
||||||
|
|
||||||
|
#ifndef NO_UNDERLINE |
||||||
|
# define match_init _match_init |
||||||
|
# define longest_match _longest_match |
||||||
|
#endif |
||||||
|
|
||||||
|
#define scanend ebx |
||||||
|
#define scanendw bx |
||||||
|
#define chainlenwmask edx /* high word: current chain len low word: s->wmask */ |
||||||
|
#define curmatch rsi |
||||||
|
#define curmatchd esi |
||||||
|
#define windowbestlen r8 |
||||||
|
#define scanalign r9 |
||||||
|
#define scanalignd r9d |
||||||
|
#define window r10 |
||||||
|
#define bestlen r11 |
||||||
|
#define bestlend r11d |
||||||
|
#define scanstart r12d |
||||||
|
#define scanstartw r12w |
||||||
|
#define scan r13 |
||||||
|
#define nicematch r14d |
||||||
|
#define limit r15 |
||||||
|
#define limitd r15d |
||||||
|
#define prev rcx |
||||||
|
|
||||||
|
/* |
||||||
|
* The 258 is a "magic number, not a parameter -- changing it |
||||||
|
* breaks the hell loose |
||||||
|
*/ |
||||||
|
#define MAX_MATCH (258) |
||||||
|
#define MIN_MATCH (3) |
||||||
|
#define MIN_LOOKAHEAD (MAX_MATCH + MIN_MATCH + 1) |
||||||
|
#define MAX_MATCH_8 ((MAX_MATCH + 7) & ~7) |
||||||
|
|
||||||
|
/* stack frame offsets */ |
||||||
|
#define LocalVarsSize (112) |
||||||
|
#define _chainlenwmask ( 8-LocalVarsSize)(%rsp) |
||||||
|
#define _windowbestlen (16-LocalVarsSize)(%rsp) |
||||||
|
#define save_r14 (24-LocalVarsSize)(%rsp) |
||||||
|
#define save_rsi (32-LocalVarsSize)(%rsp) |
||||||
|
#define save_rbx (40-LocalVarsSize)(%rsp) |
||||||
|
#define save_r12 (56-LocalVarsSize)(%rsp) |
||||||
|
#define save_r13 (64-LocalVarsSize)(%rsp) |
||||||
|
#define save_r15 (80-LocalVarsSize)(%rsp) |
||||||
|
|
||||||
|
/* |
||||||
|
* On AMD64 the first argument of a function (in our case -- the pointer to |
||||||
|
* deflate_state structure) is passed in %rdi, hence our offsets below are |
||||||
|
* all off of that. |
||||||
|
*/ |
||||||
|
#ifndef STRUCT_OFFSET |
||||||
|
# define STRUCT_OFFSET (0) |
||||||
|
#endif |
||||||
|
#define dsWSize ( 56 + STRUCT_OFFSET)(%rdi) |
||||||
|
#define dsWMask ( 64 + STRUCT_OFFSET)(%rdi) |
||||||
|
#define dsWindow ( 72 + STRUCT_OFFSET)(%rdi) |
||||||
|
#define dsPrev ( 88 + STRUCT_OFFSET)(%rdi) |
||||||
|
#define dsMatchLen (136 + STRUCT_OFFSET)(%rdi) |
||||||
|
#define dsPrevMatch (140 + STRUCT_OFFSET)(%rdi) |
||||||
|
#define dsStrStart (148 + STRUCT_OFFSET)(%rdi) |
||||||
|
#define dsMatchStart (152 + STRUCT_OFFSET)(%rdi) |
||||||
|
#define dsLookahead (156 + STRUCT_OFFSET)(%rdi) |
||||||
|
#define dsPrevLen (160 + STRUCT_OFFSET)(%rdi) |
||||||
|
#define dsMaxChainLen (164 + STRUCT_OFFSET)(%rdi) |
||||||
|
#define dsGoodMatch (180 + STRUCT_OFFSET)(%rdi) |
||||||
|
#define dsNiceMatch (184 + STRUCT_OFFSET)(%rdi) |
||||||
|
|
||||||
|
.globl match_init, longest_match |
||||||
|
|
||||||
|
.text |
||||||
|
|
||||||
|
/* uInt longest_match(deflate_state *deflatestate, IPos curmatch) */ |
||||||
|
|
||||||
|
longest_match: |
||||||
|
/* |
||||||
|
* Retrieve the function arguments. %curmatch will hold cur_match |
||||||
|
* throughout the entire function (passed via rsi on amd64). |
||||||
|
* rdi will hold the pointer to the deflate_state (first arg on amd64) |
||||||
|
*/ |
||||||
|
mov %rsi, save_rsi |
||||||
|
mov %rbx, save_rbx |
||||||
|
mov %r12, save_r12 |
||||||
|
mov %r13, save_r13 |
||||||
|
mov %r14, save_r14 |
||||||
|
mov %r15, save_r15 |
||||||
|
|
||||||
|
/* uInt wmask = s->w_mask; */ |
||||||
|
/* unsigned chain_length = s->max_chain_length; */ |
||||||
|
/* if (s->prev_length >= s->good_match) { */ |
||||||
|
/* chain_length >>= 2; */ |
||||||
|
/* } */ |
||||||
|
|
||||||
|
movl dsPrevLen, %eax |
||||||
|
movl dsGoodMatch, %ebx |
||||||
|
cmpl %ebx, %eax |
||||||
|
movl dsWMask, %eax |
||||||
|
movl dsMaxChainLen, %chainlenwmask |
||||||
|
jl LastMatchGood |
||||||
|
shrl $2, %chainlenwmask |
||||||
|
LastMatchGood: |
||||||
|
|
||||||
|
/* chainlen is decremented once beforehand so that the function can */ |
||||||
|
/* use the sign flag instead of the zero flag for the exit test. */ |
||||||
|
/* It is then shifted into the high word, to make room for the wmask */ |
||||||
|
/* value, which it will always accompany. */ |
||||||
|
|
||||||
|
decl %chainlenwmask |
||||||
|
shll $16, %chainlenwmask |
||||||
|
orl %eax, %chainlenwmask |
||||||
|
|
||||||
|
/* if ((uInt)nice_match > s->lookahead) nice_match = s->lookahead; */ |
||||||
|
|
||||||
|
movl dsNiceMatch, %eax |
||||||
|
movl dsLookahead, %ebx |
||||||
|
cmpl %eax, %ebx |
||||||
|
jl LookaheadLess |
||||||
|
movl %eax, %ebx |
||||||
|
LookaheadLess: movl %ebx, %nicematch |
||||||
|
|
||||||
|
/* register Bytef *scan = s->window + s->strstart; */ |
||||||
|
|
||||||
|
mov dsWindow, %window |
||||||
|
movl dsStrStart, %limitd |
||||||
|
lea (%limit, %window), %scan |
||||||
|
|
||||||
|
/* Determine how many bytes the scan ptr is off from being */ |
||||||
|
/* dword-aligned. */ |
||||||
|
|
||||||
|
mov %scan, %scanalign |
||||||
|
negl %scanalignd |
||||||
|
andl $3, %scanalignd |
||||||
|
|
||||||
|
/* IPos limit = s->strstart > (IPos)MAX_DIST(s) ? */ |
||||||
|
/* s->strstart - (IPos)MAX_DIST(s) : NIL; */ |
||||||
|
|
||||||
|
movl dsWSize, %eax |
||||||
|
subl $MIN_LOOKAHEAD, %eax |
||||||
|
xorl %ecx, %ecx |
||||||
|
subl %eax, %limitd |
||||||
|
cmovng %ecx, %limitd |
||||||
|
|
||||||
|
/* int best_len = s->prev_length; */ |
||||||
|
|
||||||
|
movl dsPrevLen, %bestlend |
||||||
|
|
||||||
|
/* Store the sum of s->window + best_len in %windowbestlen locally, and in memory. */ |
||||||
|
|
||||||
|
lea (%window, %bestlen), %windowbestlen |
||||||
|
mov %windowbestlen, _windowbestlen |
||||||
|
|
||||||
|
/* register ush scan_start = *(ushf*)scan; */ |
||||||
|
/* register ush scan_end = *(ushf*)(scan+best_len-1); */ |
||||||
|
/* Posf *prev = s->prev; */ |
||||||
|
|
||||||
|
movzwl (%scan), %scanstart |
||||||
|
movzwl -1(%scan, %bestlen), %scanend |
||||||
|
mov dsPrev, %prev |
||||||
|
|
||||||
|
/* Jump into the main loop. */ |
||||||
|
|
||||||
|
movl %chainlenwmask, _chainlenwmask |
||||||
|
jmp LoopEntry |
||||||
|
|
||||||
|
.balign 16
|
||||||
|
|
||||||
|
/* do { |
||||||
|
* match = s->window + cur_match;
|
||||||
|
* if (*(ushf*)(match+best_len-1) != scan_end || |
||||||
|
* *(ushf*)match != scan_start) continue;
|
||||||
|
* [...] |
||||||
|
* } while ((cur_match = prev[cur_match & wmask]) > limit |
||||||
|
* && --chain_length != 0);
|
||||||
|
* |
||||||
|
* Here is the inner loop of the function. The function will spend the |
||||||
|
* majority of its time in this loop, and majority of that time will |
||||||
|
* be spent in the first ten instructions. |
||||||
|
*/ |
||||||
|
LookupLoop: |
||||||
|
andl %chainlenwmask, %curmatchd |
||||||
|
movzwl (%prev, %curmatch, 2), %curmatchd |
||||||
|
cmpl %limitd, %curmatchd |
||||||
|
jbe LeaveNow |
||||||
|
subl $0x00010000, %chainlenwmask |
||||||
|
js LeaveNow |
||||||
|
LoopEntry: cmpw -1(%windowbestlen, %curmatch), %scanendw |
||||||
|
jne LookupLoop |
||||||
|
cmpw %scanstartw, (%window, %curmatch) |
||||||
|
jne LookupLoop |
||||||
|
|
||||||
|
/* Store the current value of chainlen. */ |
||||||
|
movl %chainlenwmask, _chainlenwmask |
||||||
|
|
||||||
|
/* %scan is the string under scrutiny, and %prev to the string we */ |
||||||
|
/* are hoping to match it up with. In actuality, %esi and %edi are */ |
||||||
|
/* both pointed (MAX_MATCH_8 - scanalign) bytes ahead, and %edx is */ |
||||||
|
/* initialized to -(MAX_MATCH_8 - scanalign). */ |
||||||
|
|
||||||
|
mov $(-MAX_MATCH_8), %rdx |
||||||
|
lea (%curmatch, %window), %windowbestlen |
||||||
|
lea MAX_MATCH_8(%windowbestlen, %scanalign), %windowbestlen |
||||||
|
lea MAX_MATCH_8(%scan, %scanalign), %prev |
||||||
|
|
||||||
|
/* the prefetching below makes very little difference... */ |
||||||
|
prefetcht1 (%windowbestlen, %rdx) |
||||||
|
prefetcht1 (%prev, %rdx) |
||||||
|
|
||||||
|
/* |
||||||
|
* Test the strings for equality, 8 bytes at a time. At the end, |
||||||
|
* adjust %rdx so that it is offset to the exact byte that mismatched. |
||||||
|
* |
||||||
|
* It should be confessed that this loop usually does not represent |
||||||
|
* much of the total running time. Replacing it with a more |
||||||
|
* straightforward "rep cmpsb" would not drastically degrade |
||||||
|
* performance -- unrolling it, for example, makes no difference. |
||||||
|
*/ |
||||||
|
#undef USE_SSE /* works, but is 6-7% slower, than non-SSE... */ |
||||||
|
LoopCmps: |
||||||
|
#ifdef USE_SSE |
||||||
|
/* Preload the SSE registers */ |
||||||
|
movdqu (%windowbestlen, %rdx), %xmm1 |
||||||
|
movdqu (%prev, %rdx), %xmm2 |
||||||
|
pcmpeqb %xmm2, %xmm1 |
||||||
|
movdqu 16(%windowbestlen, %rdx), %xmm3 |
||||||
|
movdqu 16(%prev, %rdx), %xmm4 |
||||||
|
pcmpeqb %xmm4, %xmm3 |
||||||
|
movdqu 32(%windowbestlen, %rdx), %xmm5 |
||||||
|
movdqu 32(%prev, %rdx), %xmm6 |
||||||
|
pcmpeqb %xmm6, %xmm5 |
||||||
|
movdqu 48(%windowbestlen, %rdx), %xmm7 |
||||||
|
movdqu 48(%prev, %rdx), %xmm8 |
||||||
|
pcmpeqb %xmm8, %xmm7 |
||||||
|
|
||||||
|
/* Check the comparisions' results */ |
||||||
|
pmovmskb %xmm1, %rax |
||||||
|
notw %ax |
||||||
|
bsfw %ax, %ax |
||||||
|
jnz LeaveLoopCmps |
||||||
|
add $16, %rdx |
||||||
|
pmovmskb %xmm3, %rax |
||||||
|
notw %ax |
||||||
|
bsfw %ax, %ax |
||||||
|
jnz LeaveLoopCmps |
||||||
|
add $16, %rdx |
||||||
|
pmovmskb %xmm5, %rax |
||||||
|
notw %ax |
||||||
|
bsfw %ax, %ax |
||||||
|
jnz LeaveLoopCmps |
||||||
|
add $16, %rdx |
||||||
|
pmovmskb %xmm7, %rax |
||||||
|
notw %ax |
||||||
|
bsfw %ax, %ax |
||||||
|
jnz LeaveLoopCmps |
||||||
|
add $16, %rdx |
||||||
|
jmp LoopCmps |
||||||
|
LeaveLoopCmps: add %rax, %rdx |
||||||
|
#else |
||||||
|
mov (%windowbestlen, %rdx), %rax |
||||||
|
xor (%prev, %rdx), %rax |
||||||
|
jnz LeaveLoopCmps |
||||||
|
add $8, %rdx |
||||||
|
jnz LoopCmps |
||||||
|
jmp LenMaximum |
||||||
|
# if 0 |
||||||
|
/* |
||||||
|
* This three-liner is tantalizingly simple, but bsf is a slow instruction, |
||||||
|
* and the complicated alternative down below is quite a bit faster. Sad... |
||||||
|
*/ |
||||||
|
LeaveLoopCmps: bsf %rax, %rax /* find the first non-zero bit */ |
||||||
|
shrl $3, %eax /* divide by 8 to get the byte */ |
||||||
|
add %rax, %rdx |
||||||
|
# else |
||||||
|
LeaveLoopCmps: testl $0xFFFFFFFF, %eax /* Check the first 4 bytes */ |
||||||
|
jnz Check16 |
||||||
|
add $4, %rdx |
||||||
|
shr $32, %rax |
||||||
|
Check16: testw $0xFFFF, %ax |
||||||
|
jnz LenLower |
||||||
|
add $2, %rdx |
||||||
|
shrl $16, %eax |
||||||
|
LenLower: subb $1, %al |
||||||
|
adc $0, %rdx |
||||||
|
# endif |
||||||
|
#endif |
||||||
|
|
||||||
|
/* Calculate the length of the match. If it is longer than MAX_MATCH, */ |
||||||
|
/* then automatically accept it as the best possible match and leave. */ |
||||||
|
|
||||||
|
lea (%prev, %rdx), %rax |
||||||
|
sub %scan, %rax |
||||||
|
cmpl $MAX_MATCH, %eax |
||||||
|
jge LenMaximum |
||||||
|
|
||||||
|
/* If the length of the match is not longer than the best match we */ |
||||||
|
/* have so far, then forget it and return to the lookup loop. */ |
||||||
|
|
||||||
|
cmpl %bestlend, %eax |
||||||
|
jg LongerMatch |
||||||
|
mov _windowbestlen, %windowbestlen |
||||||
|
mov dsPrev, %prev |
||||||
|
movl _chainlenwmask, %edx |
||||||
|
jmp LookupLoop |
||||||
|
|
||||||
|
/* s->match_start = cur_match; */ |
||||||
|
/* best_len = len; */ |
||||||
|
/* if (len >= nice_match) break; */ |
||||||
|
/* scan_end = *(ushf*)(scan+best_len-1); */ |
||||||
|
|
||||||
|
LongerMatch: |
||||||
|
movl %eax, %bestlend |
||||||
|
movl %curmatchd, dsMatchStart |
||||||
|
cmpl %nicematch, %eax |
||||||
|
jge LeaveNow |
||||||
|
|
||||||
|
lea (%window, %bestlen), %windowbestlen |
||||||
|
mov %windowbestlen, _windowbestlen |
||||||
|
|
||||||
|
movzwl -1(%scan, %rax), %scanend |
||||||
|
mov dsPrev, %prev |
||||||
|
movl _chainlenwmask, %chainlenwmask |
||||||
|
jmp LookupLoop |
||||||
|
|
||||||
|
/* Accept the current string, with the maximum possible length. */ |
||||||
|
|
||||||
|
LenMaximum: |
||||||
|
movl $MAX_MATCH, %bestlend |
||||||
|
movl %curmatchd, dsMatchStart |
||||||
|
|
||||||
|
/* if ((uInt)best_len <= s->lookahead) return (uInt)best_len; */ |
||||||
|
/* return s->lookahead; */ |
||||||
|
|
||||||
|
LeaveNow: |
||||||
|
movl dsLookahead, %eax |
||||||
|
cmpl %eax, %bestlend |
||||||
|
cmovngl %bestlend, %eax |
||||||
|
LookaheadRet: |
||||||
|
|
||||||
|
/* Restore the registers and return from whence we came. */ |
||||||
|
|
||||||
|
mov save_rsi, %rsi |
||||||
|
mov save_rbx, %rbx |
||||||
|
mov save_r12, %r12 |
||||||
|
mov save_r13, %r13 |
||||||
|
mov save_r14, %r14 |
||||||
|
mov save_r15, %r15 |
||||||
|
|
||||||
|
ret |
||||||
|
|
||||||
|
match_init: ret |
||||||
@ -0,0 +1,126 @@ |
|||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
.SUFFIXES: |
||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
ifeq ($(strip $(DEVKITARM)),) |
||||||
|
$(error "Please set DEVKITARM in your environment. export DEVKITARM=<path to>devkitARM") |
||||||
|
endif |
||||||
|
|
||||||
|
include $(DEVKITARM)/ds_rules |
||||||
|
|
||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
# TARGET is the name of the output
|
||||||
|
# BUILD is the directory where object files & intermediate files will be placed
|
||||||
|
# SOURCES is a list of directories containing source code
|
||||||
|
# DATA is a list of directories containing data files
|
||||||
|
# INCLUDES is a list of directories containing header files
|
||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
TARGET := $(shell basename $(CURDIR))
|
||||||
|
BUILD := build
|
||||||
|
SOURCES := ../../
|
||||||
|
DATA := data
|
||||||
|
INCLUDES := include
|
||||||
|
|
||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
# options for code generation
|
||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
ARCH := -mthumb -mthumb-interwork
|
||||||
|
|
||||||
|
CFLAGS := -Wall -O2\
|
||||||
|
-march=armv5te -mtune=arm946e-s \
|
||||||
|
-fomit-frame-pointer -ffast-math \
|
||||||
|
$(ARCH)
|
||||||
|
|
||||||
|
CFLAGS += $(INCLUDE) -DARM9
|
||||||
|
CXXFLAGS := $(CFLAGS) -fno-rtti -fno-exceptions
|
||||||
|
|
||||||
|
ASFLAGS := $(ARCH) -march=armv5te -mtune=arm946e-s
|
||||||
|
LDFLAGS = -specs=ds_arm9.specs -g $(ARCH) -Wl,-Map,$(notdir $*.map)
|
||||||
|
|
||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
# list of directories containing libraries, this must be the top level containing
|
||||||
|
# include and lib
|
||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
LIBDIRS := $(LIBNDS)
|
||||||
|
|
||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
# no real need to edit anything past this point unless you need to add additional
|
||||||
|
# rules for different file extensions
|
||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
ifneq ($(BUILD),$(notdir $(CURDIR))) |
||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
export OUTPUT := $(CURDIR)/lib/libz.a
|
||||||
|
|
||||||
|
export VPATH := $(foreach dir,$(SOURCES),$(CURDIR)/$(dir)) \
|
||||||
|
$(foreach dir,$(DATA),$(CURDIR)/$(dir))
|
||||||
|
|
||||||
|
export DEPSDIR := $(CURDIR)/$(BUILD)
|
||||||
|
|
||||||
|
CFILES := $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.c)))
|
||||||
|
CPPFILES := $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.cpp)))
|
||||||
|
SFILES := $(foreach dir,$(SOURCES),$(notdir $(wildcard $(dir)/*.s)))
|
||||||
|
BINFILES := $(foreach dir,$(DATA),$(notdir $(wildcard $(dir)/*.*)))
|
||||||
|
|
||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
# use CXX for linking C++ projects, CC for standard C
|
||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
ifeq ($(strip $(CPPFILES)),) |
||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
export LD := $(CC)
|
||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
else |
||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
export LD := $(CXX)
|
||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
endif |
||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
export OFILES := $(addsuffix .o,$(BINFILES)) \
|
||||||
|
$(CPPFILES:.cpp=.o) $(CFILES:.c=.o) $(SFILES:.s=.o)
|
||||||
|
|
||||||
|
export INCLUDE := $(foreach dir,$(INCLUDES),-I$(CURDIR)/$(dir)) \
|
||||||
|
$(foreach dir,$(LIBDIRS),-I$(dir)/include) \
|
||||||
|
-I$(CURDIR)/$(BUILD)
|
||||||
|
|
||||||
|
.PHONY: $(BUILD) clean all |
||||||
|
|
||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
all: $(BUILD) |
||||||
|
@[ -d $@ ] || mkdir -p include
|
||||||
|
@cp ../../*.h include
|
||||||
|
|
||||||
|
lib: |
||||||
|
@[ -d $@ ] || mkdir -p $@
|
||||||
|
|
||||||
|
$(BUILD): lib |
||||||
|
@[ -d $@ ] || mkdir -p $@
|
||||||
|
@$(MAKE) --no-print-directory -C $(BUILD) -f $(CURDIR)/Makefile
|
||||||
|
|
||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
clean: |
||||||
|
@echo clean ...
|
||||||
|
@rm -fr $(BUILD) lib
|
||||||
|
|
||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
else |
||||||
|
|
||||||
|
DEPENDS := $(OFILES:.o=.d)
|
||||||
|
|
||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
# main targets
|
||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
$(OUTPUT) : $(OFILES) |
||||||
|
|
||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
%.bin.o : %.bin |
||||||
|
#---------------------------------------------------------------------------------
|
||||||
|
@echo $(notdir $<)
|
||||||
|
@$(bin2o)
|
||||||
|
|
||||||
|
|
||||||
|
-include $(DEPENDS) |
||||||
|
|
||||||
|
#---------------------------------------------------------------------------------------
|
||||||
|
endif |
||||||
|
#---------------------------------------------------------------------------------------
|
||||||
@ -0,0 +1,5 @@ |
|||||||
|
This Makefile requires devkitARM (http://www.devkitpro.org/category/devkitarm/) and works inside "contrib/nds". It is based on a devkitARM template. |
||||||
|
|
||||||
|
Eduardo Costa <eduardo.m.costa@gmail.com> |
||||||
|
January 3, 2009 |
||||||
|
|
||||||
Binary file not shown.
@ -0,0 +1,569 @@ |
|||||||
|
/* enough.c -- determine the maximum size of inflate's Huffman code tables over
|
||||||
|
* all possible valid and complete Huffman codes, subject to a length limit. |
||||||
|
* Copyright (C) 2007, 2008 Mark Adler |
||||||
|
* Version 1.3 17 February 2008 Mark Adler |
||||||
|
*/ |
||||||
|
|
||||||
|
/* Version history:
|
||||||
|
1.0 3 Jan 2007 First version (derived from codecount.c version 1.4) |
||||||
|
1.1 4 Jan 2007 Use faster incremental table usage computation |
||||||
|
Prune examine() search on previously visited states |
||||||
|
1.2 5 Jan 2007 Comments clean up |
||||||
|
As inflate does, decrease root for short codes |
||||||
|
Refuse cases where inflate would increase root |
||||||
|
1.3 17 Feb 2008 Add argument for initial root table size |
||||||
|
Fix bug for initial root table size == max - 1 |
||||||
|
Use a macro to compute the history index |
||||||
|
*/ |
||||||
|
|
||||||
|
/*
|
||||||
|
Examine all possible Huffman codes for a given number of symbols and a |
||||||
|
maximum code length in bits to determine the maximum table size for zilb's |
||||||
|
inflate. Only complete Huffman codes are counted. |
||||||
|
|
||||||
|
Two codes are considered distinct if the vectors of the number of codes per |
||||||
|
length are not identical. So permutations of the symbol assignments result |
||||||
|
in the same code for the counting, as do permutations of the assignments of |
||||||
|
the bit values to the codes (i.e. only canonical codes are counted). |
||||||
|
|
||||||
|
We build a code from shorter to longer lengths, determining how many symbols |
||||||
|
are coded at each length. At each step, we have how many symbols remain to |
||||||
|
be coded, what the last code length used was, and how many bit patterns of |
||||||
|
that length remain unused. Then we add one to the code length and double the |
||||||
|
number of unused patterns to graduate to the next code length. We then |
||||||
|
assign all portions of the remaining symbols to that code length that |
||||||
|
preserve the properties of a correct and eventually complete code. Those |
||||||
|
properties are: we cannot use more bit patterns than are available; and when |
||||||
|
all the symbols are used, there are exactly zero possible bit patterns |
||||||
|
remaining. |
||||||
|
|
||||||
|
The inflate Huffman decoding algorithm uses two-level lookup tables for |
||||||
|
speed. There is a single first-level table to decode codes up to root bits |
||||||
|
in length (root == 9 in the current inflate implementation). The table |
||||||
|
has 1 << root entries and is indexed by the next root bits of input. Codes |
||||||
|
shorter than root bits have replicated table entries, so that the correct |
||||||
|
entry is pointed to regardless of the bits that follow the short code. If |
||||||
|
the code is longer than root bits, then the table entry points to a second- |
||||||
|
level table. The size of that table is determined by the longest code with |
||||||
|
that root-bit prefix. If that longest code has length len, then the table |
||||||
|
has size 1 << (len - root), to index the remaining bits in that set of |
||||||
|
codes. Each subsequent root-bit prefix then has its own sub-table. The |
||||||
|
total number of table entries required by the code is calculated |
||||||
|
incrementally as the number of codes at each bit length is populated. When |
||||||
|
all of the codes are shorter than root bits, then root is reduced to the |
||||||
|
longest code length, resulting in a single, smaller, one-level table. |
||||||
|
|
||||||
|
The inflate algorithm also provides for small values of root (relative to |
||||||
|
the log2 of the number of symbols), where the shortest code has more bits |
||||||
|
than root. In that case, root is increased to the length of the shortest |
||||||
|
code. This program, by design, does not handle that case, so it is verified |
||||||
|
that the number of symbols is less than 2^(root + 1). |
||||||
|
|
||||||
|
In order to speed up the examination (by about ten orders of magnitude for |
||||||
|
the default arguments), the intermediate states in the build-up of a code |
||||||
|
are remembered and previously visited branches are pruned. The memory |
||||||
|
required for this will increase rapidly with the total number of symbols and |
||||||
|
the maximum code length in bits. However this is a very small price to pay |
||||||
|
for the vast speedup. |
||||||
|
|
||||||
|
First, all of the possible Huffman codes are counted, and reachable |
||||||
|
intermediate states are noted by a non-zero count in a saved-results array. |
||||||
|
Second, the intermediate states that lead to (root + 1) bit or longer codes |
||||||
|
are used to look at all sub-codes from those junctures for their inflate |
||||||
|
memory usage. (The amount of memory used is not affected by the number of |
||||||
|
codes of root bits or less in length.) Third, the visited states in the |
||||||
|
construction of those sub-codes and the associated calculation of the table |
||||||
|
size is recalled in order to avoid recalculating from the same juncture. |
||||||
|
Beginning the code examination at (root + 1) bit codes, which is enabled by |
||||||
|
identifying the reachable nodes, accounts for about six of the orders of |
||||||
|
magnitude of improvement for the default arguments. About another four |
||||||
|
orders of magnitude come from not revisiting previous states. Out of |
||||||
|
approximately 2x10^16 possible Huffman codes, only about 2x10^6 sub-codes |
||||||
|
need to be examined to cover all of the possible table memory usage cases |
||||||
|
for the default arguments of 286 symbols limited to 15-bit codes. |
||||||
|
|
||||||
|
Note that an unsigned long long type is used for counting. It is quite easy |
||||||
|
to exceed the capacity of an eight-byte integer with a large number of |
||||||
|
symbols and a large maximum code length, so multiple-precision arithmetic |
||||||
|
would need to replace the unsigned long long arithmetic in that case. This |
||||||
|
program will abort if an overflow occurs. The big_t type identifies where |
||||||
|
the counting takes place. |
||||||
|
|
||||||
|
An unsigned long long type is also used for calculating the number of |
||||||
|
possible codes remaining at the maximum length. This limits the maximum |
||||||
|
code length to the number of bits in a long long minus the number of bits |
||||||
|
needed to represent the symbols in a flat code. The code_t type identifies |
||||||
|
where the bit pattern counting takes place. |
||||||
|
*/ |
||||||
|
|
||||||
|
#include <stdio.h> |
||||||
|
#include <stdlib.h> |
||||||
|
#include <string.h> |
||||||
|
#include <assert.h> |
||||||
|
|
||||||
|
#define local static |
||||||
|
|
||||||
|
/* special data types */ |
||||||
|
typedef unsigned long long big_t; /* type for code counting */ |
||||||
|
typedef unsigned long long code_t; /* type for bit pattern counting */ |
||||||
|
struct tab { /* type for been here check */ |
||||||
|
size_t len; /* length of bit vector in char's */ |
||||||
|
char *vec; /* allocated bit vector */ |
||||||
|
}; |
||||||
|
|
||||||
|
/* The array for saving results, num[], is indexed with this triplet:
|
||||||
|
|
||||||
|
syms: number of symbols remaining to code |
||||||
|
left: number of available bit patterns at length len |
||||||
|
len: number of bits in the codes currently being assigned |
||||||
|
|
||||||
|
Those indices are constrained thusly when saving results: |
||||||
|
|
||||||
|
syms: 3..totsym (totsym == total symbols to code) |
||||||
|
left: 2..syms - 1, but only the evens (so syms == 8 -> 2, 4, 6) |
||||||
|
len: 1..max - 1 (max == maximum code length in bits) |
||||||
|
|
||||||
|
syms == 2 is not saved since that immediately leads to a single code. left |
||||||
|
must be even, since it represents the number of available bit patterns at |
||||||
|
the current length, which is double the number at the previous length. |
||||||
|
left ends at syms-1 since left == syms immediately results in a single code. |
||||||
|
(left > sym is not allowed since that would result in an incomplete code.) |
||||||
|
len is less than max, since the code completes immediately when len == max. |
||||||
|
|
||||||
|
The offset into the array is calculated for the three indices with the |
||||||
|
first one (syms) being outermost, and the last one (len) being innermost. |
||||||
|
We build the array with length max-1 lists for the len index, with syms-3 |
||||||
|
of those for each symbol. There are totsym-2 of those, with each one |
||||||
|
varying in length as a function of sym. See the calculation of index in |
||||||
|
count() for the index, and the calculation of size in main() for the size |
||||||
|
of the array. |
||||||
|
|
||||||
|
For the deflate example of 286 symbols limited to 15-bit codes, the array |
||||||
|
has 284,284 entries, taking up 2.17 MB for an 8-byte big_t. More than |
||||||
|
half of the space allocated for saved results is actually used -- not all |
||||||
|
possible triplets are reached in the generation of valid Huffman codes.
|
||||||
|
*/ |
||||||
|
|
||||||
|
/* The array for tracking visited states, done[], is itself indexed identically
|
||||||
|
to the num[] array as described above for the (syms, left, len) triplet. |
||||||
|
Each element in the array is further indexed by the (mem, rem) doublet, |
||||||
|
where mem is the amount of inflate table space used so far, and rem is the |
||||||
|
remaining unused entries in the current inflate sub-table. Each indexed |
||||||
|
element is simply one bit indicating whether the state has been visited or |
||||||
|
not. Since the ranges for mem and rem are not known a priori, each bit |
||||||
|
vector is of a variable size, and grows as needed to accommodate the visited |
||||||
|
states. mem and rem are used to calculate a single index in a triangular |
||||||
|
array. Since the range of mem is expected in the default case to be about |
||||||
|
ten times larger than the range of rem, the array is skewed to reduce the |
||||||
|
memory usage, with eight times the range for mem than for rem. See the |
||||||
|
calculations for offset and bit in beenhere() for the details. |
||||||
|
|
||||||
|
For the deflate example of 286 symbols limited to 15-bit codes, the bit |
||||||
|
vectors grow to total approximately 21 MB, in addition to the 4.3 MB done[] |
||||||
|
array itself. |
||||||
|
*/ |
||||||
|
|
||||||
|
/* Globals to avoid propagating constants or constant pointers recursively */ |
||||||
|
local int max; /* maximum allowed bit length for the codes */ |
||||||
|
local int root; /* size of base code table in bits */ |
||||||
|
local int large; /* largest code table so far */ |
||||||
|
local size_t size; /* number of elements in num and done */ |
||||||
|
local int *code; /* number of symbols assigned to each bit length */ |
||||||
|
local big_t *num; /* saved results array for code counting */ |
||||||
|
local struct tab *done; /* states already evaluated array */ |
||||||
|
|
||||||
|
/* Index function for num[] and done[] */ |
||||||
|
#define INDEX(i,j,k) (((size_t)((i-1)>>1)*((i-2)>>1)+(j>>1)-1)*(max-1)+k-1) |
||||||
|
|
||||||
|
/* Free allocated space. Uses globals code, num, and done. */ |
||||||
|
local void cleanup(void) |
||||||
|
{ |
||||||
|
size_t n; |
||||||
|
|
||||||
|
if (done != NULL) { |
||||||
|
for (n = 0; n < size; n++) |
||||||
|
if (done[n].len) |
||||||
|
free(done[n].vec); |
||||||
|
free(done); |
||||||
|
} |
||||||
|
if (num != NULL) |
||||||
|
free(num); |
||||||
|
if (code != NULL) |
||||||
|
free(code); |
||||||
|
} |
||||||
|
|
||||||
|
/* Return the number of possible Huffman codes using bit patterns of lengths
|
||||||
|
len through max inclusive, coding syms symbols, with left bit patterns of |
||||||
|
length len unused -- return -1 if there is an overflow in the counting. |
||||||
|
Keep a record of previous results in num to prevent repeating the same |
||||||
|
calculation. Uses the globals max and num. */ |
||||||
|
local big_t count(int syms, int len, int left) |
||||||
|
{ |
||||||
|
big_t sum; /* number of possible codes from this juncture */ |
||||||
|
big_t got; /* value returned from count() */ |
||||||
|
int least; /* least number of syms to use at this juncture */ |
||||||
|
int most; /* most number of syms to use at this juncture */ |
||||||
|
int use; /* number of bit patterns to use in next call */ |
||||||
|
size_t index; /* index of this case in *num */ |
||||||
|
|
||||||
|
/* see if only one possible code */ |
||||||
|
if (syms == left) |
||||||
|
return 1; |
||||||
|
|
||||||
|
/* note and verify the expected state */ |
||||||
|
assert(syms > left && left > 0 && len < max); |
||||||
|
|
||||||
|
/* see if we've done this one already */ |
||||||
|
index = INDEX(syms, left, len); |
||||||
|
got = num[index]; |
||||||
|
if (got) |
||||||
|
return got; /* we have -- return the saved result */ |
||||||
|
|
||||||
|
/* we need to use at least this many bit patterns so that the code won't be
|
||||||
|
incomplete at the next length (more bit patterns than symbols) */ |
||||||
|
least = (left << 1) - syms; |
||||||
|
if (least < 0) |
||||||
|
least = 0; |
||||||
|
|
||||||
|
/* we can use at most this many bit patterns, lest there not be enough
|
||||||
|
available for the remaining symbols at the maximum length (if there were |
||||||
|
no limit to the code length, this would become: most = left - 1) */ |
||||||
|
most = (((code_t)left << (max - len)) - syms) / |
||||||
|
(((code_t)1 << (max - len)) - 1); |
||||||
|
|
||||||
|
/* count all possible codes from this juncture and add them up */ |
||||||
|
sum = 0; |
||||||
|
for (use = least; use <= most; use++) { |
||||||
|
got = count(syms - use, len + 1, (left - use) << 1); |
||||||
|
sum += got; |
||||||
|
if (got == -1 || sum < got) /* overflow */ |
||||||
|
return -1; |
||||||
|
} |
||||||
|
|
||||||
|
/* verify that all recursive calls are productive */ |
||||||
|
assert(sum != 0); |
||||||
|
|
||||||
|
/* save the result and return it */ |
||||||
|
num[index] = sum; |
||||||
|
return sum; |
||||||
|
} |
||||||
|
|
||||||
|
/* Return true if we've been here before, set to true if not. Set a bit in a
|
||||||
|
bit vector to indicate visiting this state. Each (syms,len,left) state |
||||||
|
has a variable size bit vector indexed by (mem,rem). The bit vector is |
||||||
|
lengthened if needed to allow setting the (mem,rem) bit. */ |
||||||
|
local int beenhere(int syms, int len, int left, int mem, int rem) |
||||||
|
{ |
||||||
|
size_t index; /* index for this state's bit vector */ |
||||||
|
size_t offset; /* offset in this state's bit vector */ |
||||||
|
int bit; /* mask for this state's bit */ |
||||||
|
size_t length; /* length of the bit vector in bytes */ |
||||||
|
char *vector; /* new or enlarged bit vector */ |
||||||
|
|
||||||
|
/* point to vector for (syms,left,len), bit in vector for (mem,rem) */ |
||||||
|
index = INDEX(syms, left, len); |
||||||
|
mem -= 1 << root; |
||||||
|
offset = (mem >> 3) + rem; |
||||||
|
offset = ((offset * (offset + 1)) >> 1) + rem; |
||||||
|
bit = 1 << (mem & 7); |
||||||
|
|
||||||
|
/* see if we've been here */ |
||||||
|
length = done[index].len; |
||||||
|
if (offset < length && (done[index].vec[offset] & bit) != 0) |
||||||
|
return 1; /* done this! */ |
||||||
|
|
||||||
|
/* we haven't been here before -- set the bit to show we have now */ |
||||||
|
|
||||||
|
/* see if we need to lengthen the vector in order to set the bit */ |
||||||
|
if (length <= offset) { |
||||||
|
/* if we have one already, enlarge it, zero out the appended space */ |
||||||
|
if (length) { |
||||||
|
do { |
||||||
|
length <<= 1; |
||||||
|
} while (length <= offset); |
||||||
|
vector = realloc(done[index].vec, length); |
||||||
|
if (vector != NULL) |
||||||
|
memset(vector + done[index].len, 0, length - done[index].len); |
||||||
|
} |
||||||
|
|
||||||
|
/* otherwise we need to make a new vector and zero it out */ |
||||||
|
else { |
||||||
|
length = 1 << (len - root); |
||||||
|
while (length <= offset) |
||||||
|
length <<= 1; |
||||||
|
vector = calloc(length, sizeof(char)); |
||||||
|
} |
||||||
|
|
||||||
|
/* in either case, bail if we can't get the memory */ |
||||||
|
if (vector == NULL) { |
||||||
|
fputs("abort: unable to allocate enough memory\n", stderr); |
||||||
|
cleanup(); |
||||||
|
exit(1); |
||||||
|
} |
||||||
|
|
||||||
|
/* install the new vector */ |
||||||
|
done[index].len = length; |
||||||
|
done[index].vec = vector; |
||||||
|
} |
||||||
|
|
||||||
|
/* set the bit */ |
||||||
|
done[index].vec[offset] |= bit; |
||||||
|
return 0; |
||||||
|
} |
||||||
|
|
||||||
|
/* Examine all possible codes from the given node (syms, len, left). Compute
|
||||||
|
the amount of memory required to build inflate's decoding tables, where the |
||||||
|
number of code structures used so far is mem, and the number remaining in |
||||||
|
the current sub-table is rem. Uses the globals max, code, root, large, and |
||||||
|
done. */ |
||||||
|
local void examine(int syms, int len, int left, int mem, int rem) |
||||||
|
{ |
||||||
|
int least; /* least number of syms to use at this juncture */ |
||||||
|
int most; /* most number of syms to use at this juncture */ |
||||||
|
int use; /* number of bit patterns to use in next call */ |
||||||
|
|
||||||
|
/* see if we have a complete code */ |
||||||
|
if (syms == left) { |
||||||
|
/* set the last code entry */ |
||||||
|
code[len] = left; |
||||||
|
|
||||||
|
/* complete computation of memory used by this code */ |
||||||
|
while (rem < left) { |
||||||
|
left -= rem; |
||||||
|
rem = 1 << (len - root); |
||||||
|
mem += rem; |
||||||
|
} |
||||||
|
assert(rem == left); |
||||||
|
|
||||||
|
/* if this is a new maximum, show the entries used and the sub-code */ |
||||||
|
if (mem > large) { |
||||||
|
large = mem; |
||||||
|
printf("max %d: ", mem); |
||||||
|
for (use = root + 1; use <= max; use++) |
||||||
|
if (code[use]) |
||||||
|
printf("%d[%d] ", code[use], use); |
||||||
|
putchar('\n'); |
||||||
|
fflush(stdout); |
||||||
|
} |
||||||
|
|
||||||
|
/* remove entries as we drop back down in the recursion */ |
||||||
|
code[len] = 0; |
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
/* prune the tree if we can */ |
||||||
|
if (beenhere(syms, len, left, mem, rem)) |
||||||
|
return; |
||||||
|
|
||||||
|
/* we need to use at least this many bit patterns so that the code won't be
|
||||||
|
incomplete at the next length (more bit patterns than symbols) */ |
||||||
|
least = (left << 1) - syms; |
||||||
|
if (least < 0) |
||||||
|
least = 0; |
||||||
|
|
||||||
|
/* we can use at most this many bit patterns, lest there not be enough
|
||||||
|
available for the remaining symbols at the maximum length (if there were |
||||||
|
no limit to the code length, this would become: most = left - 1) */ |
||||||
|
most = (((code_t)left << (max - len)) - syms) / |
||||||
|
(((code_t)1 << (max - len)) - 1); |
||||||
|
|
||||||
|
/* occupy least table spaces, creating new sub-tables as needed */ |
||||||
|
use = least; |
||||||
|
while (rem < use) { |
||||||
|
use -= rem; |
||||||
|
rem = 1 << (len - root); |
||||||
|
mem += rem; |
||||||
|
} |
||||||
|
rem -= use; |
||||||
|
|
||||||
|
/* examine codes from here, updating table space as we go */ |
||||||
|
for (use = least; use <= most; use++) { |
||||||
|
code[len] = use; |
||||||
|
examine(syms - use, len + 1, (left - use) << 1, |
||||||
|
mem + (rem ? 1 << (len - root) : 0), rem << 1); |
||||||
|
if (rem == 0) { |
||||||
|
rem = 1 << (len - root); |
||||||
|
mem += rem; |
||||||
|
} |
||||||
|
rem--; |
||||||
|
} |
||||||
|
|
||||||
|
/* remove entries as we drop back down in the recursion */ |
||||||
|
code[len] = 0; |
||||||
|
} |
||||||
|
|
||||||
|
/* Look at all sub-codes starting with root + 1 bits. Look at only the valid
|
||||||
|
intermediate code states (syms, left, len). For each completed code, |
||||||
|
calculate the amount of memory required by inflate to build the decoding |
||||||
|
tables. Find the maximum amount of memory required and show the code that |
||||||
|
requires that maximum. Uses the globals max, root, and num. */ |
||||||
|
local void enough(int syms) |
||||||
|
{ |
||||||
|
int n; /* number of remaing symbols for this node */ |
||||||
|
int left; /* number of unused bit patterns at this length */ |
||||||
|
size_t index; /* index of this case in *num */ |
||||||
|
|
||||||
|
/* clear code */ |
||||||
|
for (n = 0; n <= max; n++) |
||||||
|
code[n] = 0; |
||||||
|
|
||||||
|
/* look at all (root + 1) bit and longer codes */ |
||||||
|
large = 1 << root; /* base table */ |
||||||
|
if (root < max) /* otherwise, there's only a base table */ |
||||||
|
for (n = 3; n <= syms; n++) |
||||||
|
for (left = 2; left < n; left += 2) |
||||||
|
{ |
||||||
|
/* look at all reachable (root + 1) bit nodes, and the
|
||||||
|
resulting codes (complete at root + 2 or more) */ |
||||||
|
index = INDEX(n, left, root + 1); |
||||||
|
if (root + 1 < max && num[index]) /* reachable node */ |
||||||
|
examine(n, root + 1, left, 1 << root, 0); |
||||||
|
|
||||||
|
/* also look at root bit codes with completions at root + 1
|
||||||
|
bits (not saved in num, since complete), just in case */ |
||||||
|
if (num[index - 1] && n <= left << 1) |
||||||
|
examine((n - left) << 1, root + 1, (n - left) << 1, |
||||||
|
1 << root, 0); |
||||||
|
} |
||||||
|
|
||||||
|
/* done */ |
||||||
|
printf("done: maximum of %d table entries\n", large); |
||||||
|
} |
||||||
|
|
||||||
|
/*
|
||||||
|
Examine and show the total number of possible Huffman codes for a given |
||||||
|
maximum number of symbols, initial root table size, and maximum code length |
||||||
|
in bits -- those are the command arguments in that order. The default |
||||||
|
values are 286, 9, and 15 respectively, for the deflate literal/length code. |
||||||
|
The possible codes are counted for each number of coded symbols from two to |
||||||
|
the maximum. The counts for each of those and the total number of codes are |
||||||
|
shown. The maximum number of inflate table entires is then calculated |
||||||
|
across all possible codes. Each new maximum number of table entries and the |
||||||
|
associated sub-code (starting at root + 1 == 10 bits) is shown. |
||||||
|
|
||||||
|
To count and examine Huffman codes that are not length-limited, provide a |
||||||
|
maximum length equal to the number of symbols minus one. |
||||||
|
|
||||||
|
For the deflate literal/length code, use "enough". For the deflate distance |
||||||
|
code, use "enough 30 6". |
||||||
|
|
||||||
|
This uses the %llu printf format to print big_t numbers, which assumes that |
||||||
|
big_t is an unsigned long long. If the big_t type is changed (for example |
||||||
|
to a multiple precision type), the method of printing will also need to be |
||||||
|
updated. |
||||||
|
*/ |
||||||
|
int main(int argc, char **argv) |
||||||
|
{ |
||||||
|
int syms; /* total number of symbols to code */ |
||||||
|
int n; /* number of symbols to code for this run */ |
||||||
|
big_t got; /* return value of count() */ |
||||||
|
big_t sum; /* accumulated number of codes over n */ |
||||||
|
|
||||||
|
/* set up globals for cleanup() */ |
||||||
|
code = NULL; |
||||||
|
num = NULL; |
||||||
|
done = NULL; |
||||||
|
|
||||||
|
/* get arguments -- default to the deflate literal/length code */ |
||||||
|
syms = 286; |
||||||
|
root = 9; |
||||||
|
max = 15; |
||||||
|
if (argc > 1) { |
||||||
|
syms = atoi(argv[1]); |
||||||
|
if (argc > 2) { |
||||||
|
root = atoi(argv[2]); |
||||||
|
if (argc > 3) |
||||||
|
max = atoi(argv[3]); |
||||||
|
} |
||||||
|
} |
||||||
|
if (argc > 4 || syms < 2 || root < 1 || max < 1) { |
||||||
|
fputs("invalid arguments, need: [sym >= 2 [root >= 1 [max >= 1]]]\n", |
||||||
|
stderr); |
||||||
|
return 1; |
||||||
|
} |
||||||
|
|
||||||
|
/* if not restricting the code length, the longest is syms - 1 */ |
||||||
|
if (max > syms - 1) |
||||||
|
max = syms - 1; |
||||||
|
|
||||||
|
/* determine the number of bits in a code_t */ |
||||||
|
n = 0; |
||||||
|
while (((code_t)1 << n) != 0) |
||||||
|
n++; |
||||||
|
|
||||||
|
/* make sure that the calculation of most will not overflow */ |
||||||
|
if (max > n || syms - 2 >= (((code_t)0 - 1) >> (max - 1))) { |
||||||
|
fputs("abort: code length too long for internal types\n", stderr); |
||||||
|
return 1; |
||||||
|
} |
||||||
|
|
||||||
|
/* reject impossible code requests */ |
||||||
|
if (syms - 1 > ((code_t)1 << max) - 1) { |
||||||
|
fprintf(stderr, "%d symbols cannot be coded in %d bits\n", |
||||||
|
syms, max); |
||||||
|
return 1; |
||||||
|
} |
||||||
|
|
||||||
|
/* allocate code vector */ |
||||||
|
code = calloc(max + 1, sizeof(int)); |
||||||
|
if (code == NULL) { |
||||||
|
fputs("abort: unable to allocate enough memory\n", stderr); |
||||||
|
return 1; |
||||||
|
} |
||||||
|
|
||||||
|
/* determine size of saved results array, checking for overflows,
|
||||||
|
allocate and clear the array (set all to zero with calloc()) */ |
||||||
|
if (syms == 2) /* iff max == 1 */ |
||||||
|
num = NULL; /* won't be saving any results */ |
||||||
|
else { |
||||||
|
size = syms >> 1; |
||||||
|
if (size > ((size_t)0 - 1) / (n = (syms - 1) >> 1) || |
||||||
|
(size *= n, size > ((size_t)0 - 1) / (n = max - 1)) || |
||||||
|
(size *= n, size > ((size_t)0 - 1) / sizeof(big_t)) || |
||||||
|
(num = calloc(size, sizeof(big_t))) == NULL) { |
||||||
|
fputs("abort: unable to allocate enough memory\n", stderr); |
||||||
|
cleanup(); |
||||||
|
return 1; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/* count possible codes for all numbers of symbols, add up counts */ |
||||||
|
sum = 0; |
||||||
|
for (n = 2; n <= syms; n++) { |
||||||
|
got = count(n, 1, 2); |
||||||
|
sum += got; |
||||||
|
if (got == -1 || sum < got) { /* overflow */ |
||||||
|
fputs("abort: can't count that high!\n", stderr); |
||||||
|
cleanup(); |
||||||
|
return 1; |
||||||
|
} |
||||||
|
printf("%llu %d-codes\n", got, n); |
||||||
|
} |
||||||
|
printf("%llu total codes for 2 to %d symbols", sum, syms); |
||||||
|
if (max < syms - 1) |
||||||
|
printf(" (%d-bit length limit)\n", max); |
||||||
|
else |
||||||
|
puts(" (no length limit)"); |
||||||
|
|
||||||
|
/* allocate and clear done array for beenhere() */ |
||||||
|
if (syms == 2) |
||||||
|
done = NULL; |
||||||
|
else if (size > ((size_t)0 - 1) / sizeof(struct tab) || |
||||||
|
(done = calloc(size, sizeof(struct tab))) == NULL) { |
||||||
|
fputs("abort: unable to allocate enough memory\n", stderr); |
||||||
|
cleanup(); |
||||||
|
return 1; |
||||||
|
} |
||||||
|
|
||||||
|
/* find and show maximum inflate table usage */ |
||||||
|
if (root > max) /* reduce root to max length */ |
||||||
|
root = max; |
||||||
|
if (syms < ((code_t)1 << (root + 1))) |
||||||
|
enough(syms); |
||||||
|
else |
||||||
|
puts("cannot handle minimum code lengths > root"); |
||||||
|
|
||||||
|
/* done */ |
||||||
|
cleanup(); |
||||||
|
return 0; |
||||||
|
} |
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,452 @@ |
|||||||
|
/* pigz.c -- parallel implementation of gzip
|
||||||
|
* Copyright (C) 2007 Mark Adler |
||||||
|
* Version 1.1 28 January 2007 Mark Adler |
||||||
|
*/ |
||||||
|
|
||||||
|
/* Version history:
|
||||||
|
1.0 17 Jan 2007 First version |
||||||
|
1.1 28 Jan 2007 Avoid void * arithmetic (some compilers don't get that) |
||||||
|
Add note about requiring zlib 1.2.3 |
||||||
|
Allow compression level 0 (no compression) |
||||||
|
Completely rewrite parallelism -- add a write thread |
||||||
|
Use deflateSetDictionary() to make use of history |
||||||
|
Tune argument defaults to best performance on four cores |
||||||
|
*/ |
||||||
|
|
||||||
|
/*
|
||||||
|
pigz compresses from stdin to stdout using threads to make use of multiple |
||||||
|
processors and cores. The input is broken up into 128 KB chunks, and each |
||||||
|
is compressed separately. The CRC for each chunk is also calculated |
||||||
|
separately. The compressed chunks are written in order to the output, |
||||||
|
and the overall CRC is calculated from the CRC's of the chunks. |
||||||
|
|
||||||
|
The compressed data format generated is the gzip format using the deflate |
||||||
|
compression method. First a gzip header is written, followed by raw deflate |
||||||
|
partial streams. They are partial, in that they do not have a terminating |
||||||
|
block. At the end, the deflate stream is terminated with a final empty |
||||||
|
static block, and lastly a gzip trailer is written with the CRC and the |
||||||
|
number of input bytes. |
||||||
|
|
||||||
|
Each raw deflate partial stream is terminated by an empty stored block |
||||||
|
(using the Z_SYNC_FLUSH option of zlib), in order to end that partial |
||||||
|
bit stream at a byte boundary. That allows the partial streams to be |
||||||
|
concantenated simply as sequences of bytes. This adds a very small four |
||||||
|
or five byte overhead to the output for each input chunk. |
||||||
|
|
||||||
|
zlib's crc32_combine() routine allows the calcuation of the CRC of the |
||||||
|
entire input using the independent CRC's of the chunks. pigz requires zlib |
||||||
|
version 1.2.3 or later, since that is the first version that provides the |
||||||
|
crc32_combine() function. |
||||||
|
|
||||||
|
pigz uses the POSIX pthread library for thread control and communication. |
||||||
|
*/ |
||||||
|
|
||||||
|
#include <stdio.h> |
||||||
|
#include <stdlib.h> |
||||||
|
#include <string.h> |
||||||
|
#include <pthread.h> |
||||||
|
#include <sys/types.h> |
||||||
|
#include <sys/uio.h> |
||||||
|
#include <unistd.h> |
||||||
|
#include "zlib.h" |
||||||
|
|
||||||
|
#define local static |
||||||
|
|
||||||
|
/* exit with error */ |
||||||
|
local void bail(char *msg) |
||||||
|
{ |
||||||
|
fprintf(stderr, "pigz abort: %s\n", msg); |
||||||
|
exit(1); |
||||||
|
} |
||||||
|
|
||||||
|
/* read up to len bytes into buf, repeating read() calls as needed */ |
||||||
|
local size_t readn(int desc, unsigned char *buf, size_t len) |
||||||
|
{ |
||||||
|
ssize_t ret; |
||||||
|
size_t got; |
||||||
|
|
||||||
|
got = 0; |
||||||
|
while (len) { |
||||||
|
ret = read(desc, buf, len); |
||||||
|
if (ret < 0) |
||||||
|
bail("read error"); |
||||||
|
if (ret == 0) |
||||||
|
break; |
||||||
|
buf += ret; |
||||||
|
len -= ret; |
||||||
|
got += ret; |
||||||
|
} |
||||||
|
return got; |
||||||
|
} |
||||||
|
|
||||||
|
/* write len bytes, repeating write() calls as needed */ |
||||||
|
local void writen(int desc, unsigned char *buf, size_t len) |
||||||
|
{ |
||||||
|
ssize_t ret; |
||||||
|
|
||||||
|
while (len) { |
||||||
|
ret = write(desc, buf, len); |
||||||
|
if (ret < 1) |
||||||
|
bail("write error"); |
||||||
|
buf += ret; |
||||||
|
len -= ret; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/* a flag variable for communication between two threads */ |
||||||
|
struct flag { |
||||||
|
int value; /* value of flag */ |
||||||
|
pthread_mutex_t lock; /* lock for checking and changing flag */ |
||||||
|
pthread_cond_t cond; /* condition for signaling on flag change */ |
||||||
|
}; |
||||||
|
|
||||||
|
/* initialize a flag for use, starting with value val */ |
||||||
|
local void flag_init(struct flag *me, int val) |
||||||
|
{ |
||||||
|
me->value = val; |
||||||
|
pthread_mutex_init(&(me->lock), NULL); |
||||||
|
pthread_cond_init(&(me->cond), NULL); |
||||||
|
} |
||||||
|
|
||||||
|
/* set the flag to val, signal another process that may be waiting for it */ |
||||||
|
local void flag_set(struct flag *me, int val) |
||||||
|
{ |
||||||
|
pthread_mutex_lock(&(me->lock)); |
||||||
|
me->value = val; |
||||||
|
pthread_cond_signal(&(me->cond)); |
||||||
|
pthread_mutex_unlock(&(me->lock)); |
||||||
|
} |
||||||
|
|
||||||
|
/* if it isn't already, wait for some other thread to set the flag to val */ |
||||||
|
local void flag_wait(struct flag *me, int val) |
||||||
|
{ |
||||||
|
pthread_mutex_lock(&(me->lock)); |
||||||
|
while (me->value != val) |
||||||
|
pthread_cond_wait(&(me->cond), &(me->lock)); |
||||||
|
pthread_mutex_unlock(&(me->lock)); |
||||||
|
} |
||||||
|
|
||||||
|
/* if flag is equal to val, wait for some other thread to change it */ |
||||||
|
local void flag_wait_not(struct flag *me, int val) |
||||||
|
{ |
||||||
|
pthread_mutex_lock(&(me->lock)); |
||||||
|
while (me->value == val) |
||||||
|
pthread_cond_wait(&(me->cond), &(me->lock)); |
||||||
|
pthread_mutex_unlock(&(me->lock)); |
||||||
|
} |
||||||
|
|
||||||
|
/* clean up the flag when done with it */ |
||||||
|
local void flag_done(struct flag *me) |
||||||
|
{ |
||||||
|
pthread_cond_destroy(&(me->cond)); |
||||||
|
pthread_mutex_destroy(&(me->lock)); |
||||||
|
} |
||||||
|
|
||||||
|
/* a unit of work to feed to compress_thread() -- it is assumed that the out
|
||||||
|
buffer is large enough to hold the maximum size len bytes could deflate to, |
||||||
|
plus five bytes for the final sync marker */ |
||||||
|
struct work { |
||||||
|
size_t len; /* length of input */ |
||||||
|
unsigned long crc; /* crc of input */ |
||||||
|
unsigned char *buf; /* input */ |
||||||
|
unsigned char *out; /* space for output (guaranteed big enough) */ |
||||||
|
z_stream strm; /* pre-initialized z_stream */ |
||||||
|
struct flag busy; /* busy flag indicating work unit in use */ |
||||||
|
pthread_t comp; /* this compression thread */ |
||||||
|
}; |
||||||
|
|
||||||
|
/* busy flag values */ |
||||||
|
#define IDLE 0 /* compress and writing done -- can start compress */ |
||||||
|
#define COMP 1 /* compress -- input and output buffers in use */ |
||||||
|
#define WRITE 2 /* compress done, writing output -- can read input */ |
||||||
|
|
||||||
|
/* read-only globals (set by main/read thread before others started) */ |
||||||
|
local int ind; /* input file descriptor */ |
||||||
|
local int outd; /* output file descriptor */ |
||||||
|
local int level; /* compression level */ |
||||||
|
local int procs; /* number of compression threads (>= 2) */ |
||||||
|
local size_t size; /* uncompressed input size per thread (>= 32K) */ |
||||||
|
local struct work *jobs; /* work units: jobs[0..procs-1] */ |
||||||
|
|
||||||
|
/* next and previous jobs[] indices */ |
||||||
|
#define NEXT(n) ((n) == procs - 1 ? 0 : (n) + 1) |
||||||
|
#define PREV(n) ((n) == 0 ? procs - 1 : (n) - 1) |
||||||
|
|
||||||
|
/* sliding dictionary size for deflate */ |
||||||
|
#define DICT 32768U |
||||||
|
|
||||||
|
/* largest power of 2 that fits in an unsigned int -- used to limit requests
|
||||||
|
to zlib functions that use unsigned int lengths */ |
||||||
|
#define MAX ((((unsigned)-1) >> 1) + 1) |
||||||
|
|
||||||
|
/* compress thread: compress the input in the provided work unit and compute
|
||||||
|
its crc -- assume that the amount of space at job->out is guaranteed to be |
||||||
|
enough for the compressed output, as determined by the maximum expansion |
||||||
|
of deflate compression -- use the input in the previous work unit (if there |
||||||
|
is one) to set the deflate dictionary for better compression */ |
||||||
|
local void *compress_thread(void *arg) |
||||||
|
{ |
||||||
|
size_t len; /* input length for this work unit */ |
||||||
|
unsigned long crc; /* crc of input data */ |
||||||
|
struct work *prev; /* previous work unit */ |
||||||
|
struct work *job = arg; /* work unit for this thread */ |
||||||
|
z_stream *strm = &(job->strm); /* zlib stream for this work unit */ |
||||||
|
|
||||||
|
/* reset state for a new compressed stream */ |
||||||
|
(void)deflateReset(strm); |
||||||
|
|
||||||
|
/* initialize input, output, and crc */ |
||||||
|
strm->next_in = job->buf; |
||||||
|
strm->next_out = job->out; |
||||||
|
len = job->len; |
||||||
|
crc = crc32(0L, Z_NULL, 0); |
||||||
|
|
||||||
|
/* set dictionary if this isn't the first work unit, and if we will be
|
||||||
|
compressing something (the read thread assures that the dictionary |
||||||
|
data in the previous work unit is still there) */ |
||||||
|
prev = jobs + PREV(job - jobs); |
||||||
|
if (prev->buf != NULL && len != 0) |
||||||
|
deflateSetDictionary(strm, prev->buf + (size - DICT), DICT); |
||||||
|
|
||||||
|
/* run MAX-sized amounts of input through deflate and crc32 -- this loop
|
||||||
|
is needed for those cases where the integer type is smaller than the |
||||||
|
size_t type, or when len is close to the limit of the size_t type */ |
||||||
|
while (len > MAX) { |
||||||
|
strm->avail_in = MAX; |
||||||
|
strm->avail_out = (unsigned)-1; |
||||||
|
crc = crc32(crc, strm->next_in, strm->avail_in); |
||||||
|
(void)deflate(strm, Z_NO_FLUSH); |
||||||
|
len -= MAX; |
||||||
|
} |
||||||
|
|
||||||
|
/* run last piece through deflate and crc32, follow with a sync marker */ |
||||||
|
if (len) { |
||||||
|
strm->avail_in = len; |
||||||
|
strm->avail_out = (unsigned)-1; |
||||||
|
crc = crc32(crc, strm->next_in, strm->avail_in); |
||||||
|
(void)deflate(strm, Z_SYNC_FLUSH); |
||||||
|
} |
||||||
|
|
||||||
|
/* don't need to Z_FINISH, since we'd delete the last two bytes anyway */ |
||||||
|
|
||||||
|
/* return result */ |
||||||
|
job->crc = crc; |
||||||
|
return NULL; |
||||||
|
} |
||||||
|
|
||||||
|
/* put a 4-byte integer into a byte array in LSB order */ |
||||||
|
#define PUT4(a,b) (*(a)=(b),(a)[1]=(b)>>8,(a)[2]=(b)>>16,(a)[3]=(b)>>24) |
||||||
|
|
||||||
|
/* write thread: wait for compression threads to complete, write output in
|
||||||
|
order, also write gzip header and trailer around the compressed data */ |
||||||
|
local void *write_thread(void *arg) |
||||||
|
{ |
||||||
|
int n; /* compress thread index */ |
||||||
|
size_t len; /* length of input processed */ |
||||||
|
unsigned long tot; /* total uncompressed size (overflow ok) */ |
||||||
|
unsigned long crc; /* CRC-32 of uncompressed data */ |
||||||
|
unsigned char wrap[10]; /* gzip header or trailer */ |
||||||
|
|
||||||
|
/* write simple gzip header */ |
||||||
|
memcpy(wrap, "\037\213\10\0\0\0\0\0\0\3", 10); |
||||||
|
wrap[8] = level == 9 ? 2 : (level == 1 ? 4 : 0); |
||||||
|
writen(outd, wrap, 10); |
||||||
|
|
||||||
|
/* process output of compress threads until end of input */
|
||||||
|
tot = 0; |
||||||
|
crc = crc32(0L, Z_NULL, 0); |
||||||
|
n = 0; |
||||||
|
do { |
||||||
|
/* wait for compress thread to start, then wait to complete */ |
||||||
|
flag_wait(&(jobs[n].busy), COMP); |
||||||
|
pthread_join(jobs[n].comp, NULL); |
||||||
|
|
||||||
|
/* now that compress is done, allow read thread to use input buffer */ |
||||||
|
flag_set(&(jobs[n].busy), WRITE); |
||||||
|
|
||||||
|
/* write compressed data and update length and crc */ |
||||||
|
writen(outd, jobs[n].out, jobs[n].strm.next_out - jobs[n].out); |
||||||
|
len = jobs[n].len; |
||||||
|
tot += len; |
||||||
|
crc = crc32_combine(crc, jobs[n].crc, len); |
||||||
|
|
||||||
|
/* release this work unit and go to the next work unit */ |
||||||
|
flag_set(&(jobs[n].busy), IDLE); |
||||||
|
n = NEXT(n); |
||||||
|
|
||||||
|
/* an input buffer less than size in length indicates end of input */ |
||||||
|
} while (len == size); |
||||||
|
|
||||||
|
/* write final static block and gzip trailer (crc and len mod 2^32) */ |
||||||
|
wrap[0] = 3; wrap[1] = 0; |
||||||
|
PUT4(wrap + 2, crc); |
||||||
|
PUT4(wrap + 6, tot); |
||||||
|
writen(outd, wrap, 10); |
||||||
|
return NULL; |
||||||
|
} |
||||||
|
|
||||||
|
/* one-time initialization of a work unit -- this is where we set the deflate
|
||||||
|
compression level and request raw deflate, and also where we set the size |
||||||
|
of the output buffer to guarantee enough space for a worst-case deflate |
||||||
|
ending with a Z_SYNC_FLUSH */ |
||||||
|
local void job_init(struct work *job) |
||||||
|
{ |
||||||
|
int ret; /* deflateInit2() return value */ |
||||||
|
|
||||||
|
job->buf = malloc(size); |
||||||
|
job->out = malloc(size + (size >> 11) + 10); |
||||||
|
job->strm.zfree = Z_NULL; |
||||||
|
job->strm.zalloc = Z_NULL; |
||||||
|
job->strm.opaque = Z_NULL; |
||||||
|
ret = deflateInit2(&(job->strm), level, Z_DEFLATED, -15, 8, |
||||||
|
Z_DEFAULT_STRATEGY); |
||||||
|
if (job->buf == NULL || job->out == NULL || ret != Z_OK) |
||||||
|
bail("not enough memory"); |
||||||
|
} |
||||||
|
|
||||||
|
/* compress ind to outd in the gzip format, using multiple threads for the
|
||||||
|
compression and crc calculation and another thread for writing the output -- |
||||||
|
the read thread is the main thread */ |
||||||
|
local void read_thread(void) |
||||||
|
{ |
||||||
|
int n; /* general index */ |
||||||
|
size_t got; /* amount read */ |
||||||
|
pthread_attr_t attr; /* thread attributes (left at defaults) */ |
||||||
|
pthread_t write; /* write thread */ |
||||||
|
|
||||||
|
/* set defaults (not all pthread implementations default to joinable) */ |
||||||
|
pthread_attr_init(&attr); |
||||||
|
pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); |
||||||
|
|
||||||
|
/* allocate and set up work list (individual work units will be initialized
|
||||||
|
as needed, in case the input is short), assure that allocation size |
||||||
|
arithmetic does not overflow */ |
||||||
|
if (size + (size >> 11) + 10 < (size >> 11) + 10 || |
||||||
|
(ssize_t)(size + (size >> 11) + 10) < 0 || |
||||||
|
((size_t)0 - 1) / procs <= sizeof(struct work) || |
||||||
|
(jobs = malloc(procs * sizeof(struct work))) == NULL) |
||||||
|
bail("not enough memory"); |
||||||
|
for (n = 0; n < procs; n++) { |
||||||
|
jobs[n].buf = NULL; |
||||||
|
flag_init(&(jobs[n].busy), IDLE); |
||||||
|
} |
||||||
|
|
||||||
|
/* start write thread */ |
||||||
|
pthread_create(&write, &attr, write_thread, NULL); |
||||||
|
|
||||||
|
/* read from input and start compress threads (write thread will pick up
|
||||||
|
the output of the compress threads) */ |
||||||
|
n = 0; |
||||||
|
do { |
||||||
|
/* initialize this work unit if it's the first time it's used */ |
||||||
|
if (jobs[n].buf == NULL) |
||||||
|
job_init(jobs + n); |
||||||
|
|
||||||
|
/* read input data, but wait for last compress on this work unit to be
|
||||||
|
done, and wait for the dictionary to be used by the last compress on |
||||||
|
the next work unit */ |
||||||
|
flag_wait_not(&(jobs[n].busy), COMP); |
||||||
|
flag_wait_not(&(jobs[NEXT(n)].busy), COMP); |
||||||
|
got = readn(ind, jobs[n].buf, size); |
||||||
|
|
||||||
|
/* start compress thread, but wait for write to be done first */ |
||||||
|
flag_wait(&(jobs[n].busy), IDLE); |
||||||
|
jobs[n].len = got; |
||||||
|
pthread_create(&(jobs[n].comp), &attr, compress_thread, jobs + n); |
||||||
|
|
||||||
|
/* mark work unit so write thread knows compress was started */ |
||||||
|
flag_set(&(jobs[n].busy), COMP); |
||||||
|
|
||||||
|
/* go to the next work unit */ |
||||||
|
n = NEXT(n); |
||||||
|
|
||||||
|
/* do until end of input, indicated by a read less than size */ |
||||||
|
} while (got == size); |
||||||
|
|
||||||
|
/* wait for the write thread to complete -- the write thread will join with
|
||||||
|
all of the compress threads, so this waits for all of the threads to |
||||||
|
complete */ |
||||||
|
pthread_join(write, NULL); |
||||||
|
|
||||||
|
/* free up all requested resources and return */ |
||||||
|
for (n = procs - 1; n >= 0; n--) { |
||||||
|
flag_done(&(jobs[n].busy)); |
||||||
|
(void)deflateEnd(&(jobs[n].strm)); |
||||||
|
free(jobs[n].out); |
||||||
|
free(jobs[n].buf); |
||||||
|
} |
||||||
|
free(jobs); |
||||||
|
pthread_attr_destroy(&attr); |
||||||
|
} |
||||||
|
|
||||||
|
/* Process arguments for level, size, and procs, compress from stdin to
|
||||||
|
stdout in the gzip format. Note that procs must be at least two in |
||||||
|
order to provide a dictionary in one work unit for the other work |
||||||
|
unit, and that size must be at least 32K to store a full dictionary. */ |
||||||
|
int main(int argc, char **argv) |
||||||
|
{ |
||||||
|
int n; /* general index */ |
||||||
|
int get; /* command line parameters to get */ |
||||||
|
char *arg; /* command line argument */ |
||||||
|
|
||||||
|
/* set defaults -- 32 processes and 128K buffers was found to provide
|
||||||
|
good utilization of four cores (about 97%) and balanced the overall |
||||||
|
execution time impact of more threads against more dictionary |
||||||
|
processing for a fixed amount of memory -- the memory usage for these |
||||||
|
settings and full use of all work units (at least 4 MB of input) is |
||||||
|
16.2 MB |
||||||
|
*/ |
||||||
|
level = Z_DEFAULT_COMPRESSION; |
||||||
|
procs = 32; |
||||||
|
size = 131072UL; |
||||||
|
|
||||||
|
/* process command-line arguments */ |
||||||
|
get = 0; |
||||||
|
for (n = 1; n < argc; n++) { |
||||||
|
arg = argv[n]; |
||||||
|
if (*arg == '-') { |
||||||
|
while (*++arg) |
||||||
|
if (*arg >= '0' && *arg <= '9') /* compression level */ |
||||||
|
level = *arg - '0'; |
||||||
|
else if (*arg == 'b') /* chunk size in K */ |
||||||
|
get |= 1; |
||||||
|
else if (*arg == 'p') /* number of processes */ |
||||||
|
get |= 2; |
||||||
|
else if (*arg == 'h') { /* help */ |
||||||
|
fputs("usage: pigz [-0..9] [-b blocksizeinK]", stderr); |
||||||
|
fputs(" [-p processes] < foo > foo.gz\n", stderr); |
||||||
|
return 0; |
||||||
|
} |
||||||
|
else |
||||||
|
bail("invalid option"); |
||||||
|
} |
||||||
|
else if (get & 1) { |
||||||
|
if (get & 2) |
||||||
|
bail("you need to separate the -b and -p options"); |
||||||
|
size = (size_t)(atol(arg)) << 10; /* chunk size */ |
||||||
|
if (size < DICT) |
||||||
|
bail("invalid option"); |
||||||
|
get = 0; |
||||||
|
} |
||||||
|
else if (get & 2) { |
||||||
|
procs = atoi(arg); /* processes */ |
||||||
|
if (procs < 2) |
||||||
|
bail("invalid option"); |
||||||
|
get = 0; |
||||||
|
} |
||||||
|
else |
||||||
|
bail("invalid option (you need to pipe input and output)"); |
||||||
|
} |
||||||
|
if (get) |
||||||
|
bail("missing option argument"); |
||||||
|
|
||||||
|
/* do parallel compression from stdin to stdout (the read thread starts up
|
||||||
|
the write thread and the compression threads, and they all join before |
||||||
|
the read thread returns) */ |
||||||
|
ind = 0; |
||||||
|
outd = 1; |
||||||
|
read_thread(); |
||||||
|
|
||||||
|
/* done */ |
||||||
|
return 0; |
||||||
|
} |
||||||
@ -0,0 +1,152 @@ |
|||||||
|
#!/usr/bin/perl |
||||||
|
|
||||||
|
# Transform K&R C function definitions into ANSI equivalent. |
||||||
|
# |
||||||
|
# Author: Paul Marquess |
||||||
|
# Version: 1.0 |
||||||
|
# Date: 3 October 2006 |
||||||
|
|
||||||
|
# TODO |
||||||
|
# |
||||||
|
# Asumes no function pointer parameters. unless they are typedefed. |
||||||
|
# Assumes no literal strings that look like function definitions |
||||||
|
# Assumes functions start at the beginning of a line |
||||||
|
|
||||||
|
use strict; |
||||||
|
use warnings; |
||||||
|
|
||||||
|
local $/; |
||||||
|
$_ = <>; |
||||||
|
|
||||||
|
my $sp = qr{ \s* (?: /\* .*? \*/ )? \s* }x; # assume no nested comments |
||||||
|
|
||||||
|
my $d1 = qr{ $sp (?: [\w\*\s]+ $sp)* $sp \w+ $sp [\[\]\s]* $sp }x ; |
||||||
|
my $decl = qr{ $sp (?: \w+ $sp )+ $d1 }xo ; |
||||||
|
my $dList = qr{ $sp $decl (?: $sp , $d1 )* $sp ; $sp }xo ; |
||||||
|
|
||||||
|
|
||||||
|
while (s/^ |
||||||
|
( # Start $1 |
||||||
|
( # Start $2 |
||||||
|
.*? # Minimal eat content |
||||||
|
( ^ \w [\w\s\*]+ ) # $3 -- function name |
||||||
|
\s* # optional whitespace |
||||||
|
) # $2 - Matched up to before parameter list |
||||||
|
|
||||||
|
\( \s* # Literal "(" + optional whitespace |
||||||
|
( [^\)]+ ) # $4 - one or more anythings except ")" |
||||||
|
\s* \) # optional whitespace surrounding a Literal ")" |
||||||
|
|
||||||
|
( (?: $dList )+ ) # $5 |
||||||
|
|
||||||
|
$sp ^ { # literal "{" at start of line |
||||||
|
) # Remember to $1 |
||||||
|
//xsom |
||||||
|
) |
||||||
|
{ |
||||||
|
my $all = $1 ; |
||||||
|
my $prefix = $2; |
||||||
|
my $param_list = $4 ; |
||||||
|
my $params = $5; |
||||||
|
|
||||||
|
StripComments($params); |
||||||
|
StripComments($param_list); |
||||||
|
$param_list =~ s/^\s+//; |
||||||
|
$param_list =~ s/\s+$//; |
||||||
|
|
||||||
|
my $i = 0 ; |
||||||
|
my %pList = map { $_ => $i++ } |
||||||
|
split /\s*,\s*/, $param_list; |
||||||
|
my $pMatch = '(\b' . join('|', keys %pList) . '\b)\W*$' ; |
||||||
|
|
||||||
|
my @params = split /\s*;\s*/, $params; |
||||||
|
my @outParams = (); |
||||||
|
foreach my $p (@params) |
||||||
|
{ |
||||||
|
if ($p =~ /,/) |
||||||
|
{ |
||||||
|
my @bits = split /\s*,\s*/, $p; |
||||||
|
my $first = shift @bits; |
||||||
|
$first =~ s/^\s*//; |
||||||
|
push @outParams, $first; |
||||||
|
$first =~ /^(\w+\s*)/; |
||||||
|
my $type = $1 ; |
||||||
|
push @outParams, map { $type . $_ } @bits; |
||||||
|
} |
||||||
|
else |
||||||
|
{ |
||||||
|
$p =~ s/^\s+//; |
||||||
|
push @outParams, $p; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
my %tmp = map { /$pMatch/; $_ => $pList{$1} } |
||||||
|
@outParams ; |
||||||
|
|
||||||
|
@outParams = map { " $_" } |
||||||
|
sort { $tmp{$a} <=> $tmp{$b} } |
||||||
|
@outParams ; |
||||||
|
|
||||||
|
print $prefix ; |
||||||
|
print "(\n" . join(",\n", @outParams) . ")\n"; |
||||||
|
print "{" ; |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
# Output any trailing code. |
||||||
|
print ; |
||||||
|
exit 0; |
||||||
|
|
||||||
|
|
||||||
|
sub StripComments |
||||||
|
{ |
||||||
|
|
||||||
|
no warnings; |
||||||
|
|
||||||
|
# Strip C & C++ coments |
||||||
|
# From the perlfaq |
||||||
|
$_[0] =~ |
||||||
|
|
||||||
|
s{ |
||||||
|
/\* ## Start of /* ... */ comment |
||||||
|
[^*]*\*+ ## Non-* followed by 1-or-more *'s |
||||||
|
( |
||||||
|
[^/*][^*]*\*+ |
||||||
|
)* ## 0-or-more things which don't start with / |
||||||
|
## but do end with '*' |
||||||
|
/ ## End of /* ... */ comment |
||||||
|
|
||||||
|
| ## OR C++ Comment |
||||||
|
// ## Start of C++ comment // |
||||||
|
[^\n]* ## followed by 0-or-more non end of line characters |
||||||
|
|
||||||
|
| ## OR various things which aren't comments: |
||||||
|
|
||||||
|
( |
||||||
|
" ## Start of " ... " string |
||||||
|
( |
||||||
|
\\. ## Escaped char |
||||||
|
| ## OR |
||||||
|
[^"\\] ## Non "\ |
||||||
|
)* |
||||||
|
" ## End of " ... " string |
||||||
|
|
||||||
|
| ## OR |
||||||
|
|
||||||
|
' ## Start of ' ... ' string |
||||||
|
( |
||||||
|
\\. ## Escaped char |
||||||
|
| ## OR |
||||||
|
[^'\\] ## Non '\ |
||||||
|
)* |
||||||
|
' ## End of ' ... ' string |
||||||
|
|
||||||
|
| ## OR |
||||||
|
|
||||||
|
. ## Anything other char |
||||||
|
[^/"'\\]* ## Chars which doesn't start a comment, string or escape |
||||||
|
) |
||||||
|
}{$2}gxs; |
||||||
|
|
||||||
|
} |
||||||
Loading…
Reference in new issue