Chiebot-Mirror
/
protobuf
mirror of https://github.com/protocolbuffers/protobuf.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity
Protocol Buffers - Google's data interchange format (grpc依赖) https://developers.google.com/protocol-buffers/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
605 Commits
201 Branches
346 Tags
254 MiB
 
 
 
 
 
 
Tag: Branch: Tree: 6d9a9c727f
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '6d9a9c727f'
${ noResults }
protobuf/upb/pb/compile_decoder_x64.dasc

1131 lines
32 KiB
Raw Blame History

|//
|// upb - a minimalist implementation of protocol buffers.
|//
|// Copyright (c) 2011-2013 Google Inc. See LICENSE for details.
|// Author: Josh Haberman <jhaberman@gmail.com>
|//
|// JIT compiler for upb_pbdecoder on x86-64. Generates machine code from the
|// bytecode generated in compile_decoder.c.
|
|.arch x64
|.actionlist upb_jit_actionlist
|.globals UPB_JIT_GLOBAL_
|.globalnames upb_jit_globalnames
|
|// Calling conventions. Note -- this will need to be changed for
|// Windows, which uses a different calling convention!
|.define ARG1_64, rdi
|.define ARG2_8, r6b // DynASM's equivalent to "sil" -- low byte of esi.
|.define ARG2_32, esi
|.define ARG2_64, rsi
|.define ARG3_8, dl
|.define ARG3_32, edx
|.define ARG3_64, rdx
|.define ARG4_64, rcx
|.define ARG5_64, r8
|.define XMMARG1, xmm0
|
|// Register allocation / type map.
|// ALL of the code in this file uses these register allocations.
|// When we "call" within this file, we do not use regular calling
|// conventions, but of course when calling to user callbacks we must.
|.define PTR, rbx // DECODER->ptr (unsynced)
|.define DATAEND, r12 // DECODER->data_end (unsynced)
|.define CLOSURE, r13 // FRAME->closure (unsynced)
|.type FRAME, upb_pbdecoder_frame, r14 // DECODER->top (unsynced)
|.type DECODER, upb_pbdecoder, r15 // DECODER (immutable)
|.define DELIMEND, rbp
|
| // Spills unsynced registers back to memory.
|.macro commit_regs
| mov DECODER->top, FRAME
| mov DECODER->ptr, PTR
| mov DECODER->data_end, DATAEND
| // We don't guarantee that delim_end is NULL when out of range like the
| // interpreter does.
| mov DECODER->delim_end, DELIMEND
| sub DELIMEND, DECODER->buf
| add DELIMEND, DECODER->bufstart_ofs
| mov FRAME->end_ofs, DELIMEND
| mov FRAME->sink.closure, CLOSURE
|.endmacro
|
| // Loads unsynced registers from memory back into registers.
|.macro load_regs
| mov FRAME, DECODER->top
| mov PTR, DECODER->ptr
| mov DATAEND, DECODER->data_end
| mov CLOSURE, FRAME->sink.closure
| mov DELIMEND, FRAME->end_ofs
| sub DELIMEND, DECODER->bufstart_ofs
| add DELIMEND, DECODER->buf
|.endmacro
|
| // Calls an external C function at address "addr".
|.macro callp, addr
| mov64 rax, (uintptr_t)addr
|
| // Stack must be 16-byte aligned (x86-64 ABI requires this).
| //
| // OPT: possibly remove this by statically ensuring correct alignment.
| //
| // OPT: use "call rel32" where possible.
| push r12
| mov r12, rsp
| and rsp, 0xfffffffffffffff0UL // Align stack.
| call rax
| mov rsp, r12
| pop r12
|.endmacro
|
|.macro ld64, val
|| {
|| uintptr_t v = (uintptr_t)val;
|| if (v > 0xffffffff) {
| mov64 ARG2_64, v
|| } else if (v) {
| mov ARG2_32, v
|| } else {
| xor ARG2_32, ARG2_32
|| }
|| }
|.endmacro
|
|.macro load_handler_data, h, arg
| ld64 upb_handlers_gethandlerdata(h, arg)
|.endmacro
|
|.macro chkeob, bytes, target
|| if (bytes == 1) {
| cmp PTR, DATAEND
| je target
|| } else {
| mov rcx, DATAEND
| sub rcx, PTR
| cmp rcx, bytes
| jb target
|| }
|.endmacro
|
|.macro chkneob, bytes, target
|| if (bytes == 1) {
| cmp PTR, DATAEND
| jne target
|| } else {
| mov rcx, DATAEND
| sub rcx, PTR
| cmp rcx, bytes
| jae target
|| }
|.endmacro
|.macro sethas, reg, hasbit
|| if (hasbit >= 0) {
| or byte [reg + ((uint32_t)hasbit / 8)], (1 << ((uint32_t)hasbit % 8))
|| }
|.endmacro
|
| // Decodes 32-bit varint into rdx, inlining 1 byte.
|.macro dv32
| chkeob 1, >7
| movzx edx, byte [PTR]
| test dl, dl
| jns >8
|7:
| call ->decodev32_fallback
|8:
| add PTR, 1
|.endmacro
#define DECODE_EOF -3
static upb_func *gethandler(const upb_handlers *h, upb_selector_t sel) {
return h ? upb_handlers_gethandler(h, sel) : NULL;
}
/* Defines an "assembly label" for the current code generation offset.
* This label exists *purely* for debugging purposes: it is emitted into
* the .so, and printed as part of JIT debugging output when UPB_JIT_LOAD_SO is
* defined.
*
* We would define this in the .c file except that it conditionally defines a
* pclabel. */
static void asmlabel(jitcompiler *jc, const char *fmt, ...) {
#ifndef NDEBUG
int ofs = jc->dynasm->section->ofs;
assert(ofs != jc->lastlabelofs);
jc->lastlabelofs = ofs;
#endif
#ifndef UPB_JIT_LOAD_SO
UPB_UNUSED(jc);
UPB_UNUSED(fmt);
#else
va_list args;
va_start(args, fmt);
char *str = upb_vasprintf(fmt, args);
va_end(args);
int pclabel = alloc_pclabel(jc);
/* Normally we would prefer to allocate this inline with the codegen,
* ie.
* |=>asmlabel(...)
* But since we do this conditionally, only when UPB_JIT_LOAD_SO is defined,
* we do it here instead. */
|=>pclabel:
upb_inttable_insert(&jc->asmlabels, pclabel, upb_value_ptr(str));
#endif
}
/* Should only be called when the associated handler is known to exist. */
static bool alwaysok(const upb_handlers *h, upb_selector_t sel) {
upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
bool ok = upb_handlers_getattr(h, sel, &attr);
bool ret;
UPB_ASSERT_VAR(ok, ok);
ret = upb_handlerattr_alwaysok(&attr);
upb_handlerattr_uninit(&attr);
return ret;
}
/* Emit static assembly routines; code that does not vary based on the message
* schema. Since it's not input-dependent, we only need one single copy of it.
* For the moment we generate a single copy per generated handlers. Eventually
* we should generate this code at compile time and link it into the binary so
* we have one copy total. To do that we'll want to be sure that it is within
* 2GB of our JIT code, so that branches between the two are near (rel32).
*
* We'd put this assembly in a .s file directly, but DynASM's ability to
* calculate structure offsets automatically is too useful to pass up (it's way
* more convenient to write DECODER->sink than [rbx + 0x96], especially since
* the latter would have to be changed whenever the structure is updated). */
static void emit_static_asm(jitcompiler *jc) {
| // Trampolines for entering/exiting the JIT. These are a bit tricky to
| // support full resuming; when we suspend we copy the JIT's portion of
| // the call stack into the upb_pbdecoder and restore it when we resume.
asmlabel(jc, "enterjit");
|->enterjit:
|1:
| push rbp
| push r15
| push r14
| push r13
| push r12
| push rbx
|
| mov rbx, ARG2_64 // Preserve JIT method.
|
| mov DECODER, rdi
| callp upb_pbdecoder_resume // Same args as us; reuse regs.
| test eax, eax
| jns >1
| mov DECODER->saved_rsp, rsp
| mov rax, rbx
| load_regs
|
| // Test whether we have a saved stack to resume.
| mov ARG3_64, DECODER->call_len
| test ARG3_64, ARG3_64
| jnz >2
|
| call rax
|
| mov rax, DECODER->size_param
| mov qword DECODER->call_len, 0
|1:
| pop rbx
| pop r12
| pop r13
| pop r14
| pop r15
| pop rbp
| ret
|
|2:
| // Resume decoder.
| mov ARG2_64, DECODER->callstack
| sub rsp, ARG3_64
| mov ARG1_64, rsp
| callp memcpy // Restore stack.
| ret // Return to resumed function (not ->enterjit caller).
|
| // Other code can call this to suspend the JIT.
| // To the calling code, it will appear that the function returns when
| // the JIT resumes, and more buffer space will be available.
| // Args: eax=the value that decode() should return.
asmlabel(jc, "exitjit");
|->exitjit:
| // Save the stack into DECODER->callstack.
| mov ARG1_64, DECODER->callstack
| mov ARG2_64, rsp
| mov ARG3_64, DECODER->saved_rsp
| sub ARG3_64, rsp
| mov DECODER->call_len, ARG3_64 // Preserve len for next resume.
| mov ebx, eax // Preserve return value across memcpy.
| callp memcpy // Copy stack into decoder.
| mov eax, ebx // This will be our return value.
|
| // Must NOT do this before the memcpy(), otherwise memcpy() will
| // clobber the stack we are trying to save!
| mov rsp, DECODER->saved_rsp
| pop rbx
| pop r12
| pop r13
| pop r14
| pop r15
| pop rbp
| ret
|
| // Like suspend() in the C decoder, except that the function appears
| // (from the caller's perspective) not to return until the decoder is
| // resumed.
asmlabel(jc, "suspend");
|->suspend:
| cmp DECODER->ptr, PTR
| je >1
| mov DECODER->checkpoint, PTR
|1:
| commit_regs
| mov rdi, DECODER
| callp upb_pbdecoder_suspend
| jmp ->exitjit
|
asmlabel(jc, "pushlendelim");
|->pushlendelim:
|1:
| mov FRAME->sink.closure, CLOSURE
| mov DECODER->checkpoint, PTR
| dv32
| mov rcx, DELIMEND
| sub rcx, PTR
| sub rcx, rdx
| jb ->err // Len is greater than enclosing message.
| mov FRAME->end_ofs, rcx
| cmp FRAME, DECODER->limit
| je >3 // Stack overflow
| add FRAME, sizeof(upb_pbdecoder_frame)
| mov DELIMEND, PTR
| add DELIMEND, rdx
| mov dword FRAME->groupnum, 0
| test rcx, rcx
| jz >2
| mov DATAEND, DECODER->end
| cmp PTR, DELIMEND
| ja >2
| cmp DELIMEND, DATAEND
| ja >2
| mov DATAEND, DELIMEND // If DELIMEND >= PTR && DELIMEND < DATAEND
|2:
| ret
|3:
| // Error -- call seterr.
| mov PTR, DECODER->checkpoint // Rollback to before the delim len.
| // Prepare seterr args.
| mov ARG1_64, DECODER
| ld64 kPbDecoderStackOverflow
| callp upb_pbdecoder_seterr
| call ->suspend
| jmp <1
|
| // For getting a value that spans a buffer seam. Falls back to C.
|.macro getvalue_slow, func, bytes
| sub rsp, 8 // Need stack space for func to write value to.
|1:
| mov qword [rsp], 0 // For parsing routines that only parse 32 bits.
| mov ARG1_64, DECODER
| mov ARG2_64, rsp
| mov DECODER->checkpoint, PTR
| commit_regs
| callp func
| load_regs
| test eax, eax
| jns >2
| // Success; return parsed data (in rdx AND xmm0).
| mov rdx, [rsp]
| movsd xmm0, qword [rsp]
| add rsp, 8
| sub PTR, bytes // Bias our buffer pointer to rejoin the fast-path.
| mov DECODER->ptr, PTR
| ret
|2:
| call ->exitjit // Return eax from decode function.
| jmp <1
|.endmacro
|
asmlabel(jc, "parse_unknown");
| // Args: edx=fieldnum, cl=wire type
|->parse_unknown:
| // OPT: handle directly instead of kicking to C.
| // Check for ENDGROUP.
| mov ARG1_64, DECODER
| mov ARG2_32, edx
| movzx ARG3_32, cl
| commit_regs
| callp upb_pbdecoder_skipunknown
| load_regs
| cmp eax, DECODE_ENDGROUP
| jne >1
| ret // Return eax=DECODE_ENDGROUP, not zero
|1:
| cmp eax, DECODE_OK
| je >1
| call ->exitjit // Return eax from decode function.
|1:
| xor eax, eax
| ret
|
| // Fallback functions for parsing single values. These are used when the
| // buffer doesn't contain enough remaining data for the fast path. Each
| // primitive type (v32, v64, f32, f64) has two functions: decode & skip.
| // Decode functions return their value in rsi/esi.
| //
| // These functions leave PTR = value_end - fast_path_bytes, so that we can
| // re-join the fast path which will add fast_path_bytes after the callback
| // completes. We also set DECODER->ptr to this value which is a signal to
| // ->suspend that DECODER->checkpoint is up to date.
asmlabel(jc, "skip_decode_f32_fallback");
|->skipf32_fallback:
|->decodef32_fallback:
| getvalue_slow upb_pbdecoder_decode_f32, 4
|
asmlabel(jc, "skip_decode_f64_fallback");
|->skipf64_fallback:
|->decodef64_fallback:
| getvalue_slow upb_pbdecoder_decode_f64, 8
|
| // Called for varint >= 1 byte.
asmlabel(jc, "skip_decode_v32_fallback");
|->skipv32_fallback:
|->skipv64_fallback:
| chkeob 16, >1
| // With at least 16 bytes left, we can do a branch-less SSE version.
| movdqu xmm0, [PTR]
| pmovmskb eax, xmm0 // bits 0-15 are continuation bits, 16-31 are 0.
| not eax
| bsf eax, eax
| cmp al, 10
| jae ->decode_varint_slow // Error (>10 byte varint).
| add PTR, rax // bsf result is 0-based, so PTR=end-1, as desired.
| ret
|
|1:
| // With fewer than 16 bytes, we have to read byte by byte.
| lea rcx, [PTR + 10]
| mov rax, PTR // Preserve PTR in case of fallback to slow path.
| cmp rcx, DATAEND
| cmova rcx, DATAEND // rcx = MIN(DATAEND, PTR + 10)
|2:
| cmp rax, rcx
| je ->decode_varint_slow
| test byte [rax], 0x80
| jz >3
| add rax, 1
| jmp <2
|3:
| mov PTR, rax // PTR = varint_end - 1, as desired
| ret
|
| // Returns tag in edx
asmlabel(jc, "decode_unknown_tag_fallback");
|->decode_unknown_tag_fallback:
| sub rsp, 16
|1:
| cmp PTR, DELIMEND
| jne >2
| add rsp, 16
| xor eax, eax
| ret
|2:
| // OPT: Have a medium-fast path before falling back to _slow.
| mov ARG1_64, DECODER
| mov ARG2_64, rsp
| commit_regs
| callp upb_pbdecoder_decode_varint_slow
| load_regs
| cmp eax, 0
| jge >3
| mov edx, [rsp] // Success; return parsed data.
| add rsp, 16
| ret
|3:
| call ->exitjit // Return eax from decode function.
| jmp <1
|
| // Called for varint >= 1 byte.
asmlabel(jc, "decode_v32_v64_fallback");
|->decodev32_fallback:
|->decodev64_fallback:
| chkeob 10, ->decode_varint_slow
| // OPT: do something faster than just calling the C version.
| mov rdi, PTR
| callp upb_vdecode_fast
| test rax, rax
| je ->decode_varint_slow // Unterminated varint.
| mov PTR, rax
| sub PTR, 1
| mov DECODER->ptr, PTR
| ret
|
asmlabel(jc, "decode_varint_slow");
|->decode_varint_slow:
| // Slow path: end of buffer or error (varint length >= 10).
| getvalue_slow upb_pbdecoder_decode_varint_slow, 1
|
| // Args: rsi=expected tag, return=rax (DECODE_{OK,MISMATCH})
asmlabel(jc, "checktag_fallback");
|->checktag_fallback:
| sub rsp, 8
| mov [rsp], rsi // Preserve expected tag.
|1:
| mov ARG1_64, DECODER
| commit_regs
| mov DECODER->checkpoint, PTR
| callp upb_pbdecoder_checktag_slow
| load_regs
| cmp eax, 0
| jge >2
| add rsp, 8
| ret
|2:
| call ->exitjit
| mov rsi, [rsp]
| cmp PTR, DELIMEND
| jne <1
| mov eax, DECODE_EOF
| add rsp, 8
| ret
|
| // Args: rsi=upb_inttable, rdx=key, return=rax (-1 if not found).
| // Preserves: rcx, rdx
| // OPT: Could write this in assembly if it's a hotspot.
asmlabel(jc, "hashlookup");
|->hashlookup:
| push rcx
| push rdx
| sub rsp, 16
| mov rdi, rsi
| mov rsi, rdx
| mov rdx, rsp
| callp upb_inttable_lookup
| add rsp, 16
| pop rdx
| pop rcx
| test al, al
| jz >2 // Unknown field.
| mov rax, [rsp-32] // Value from table.
| ret
|2:
| xor rax, rax
| not rax
| ret
}
static void jitprimitive(jitcompiler *jc, opcode op,
const upb_handlers *h, upb_selector_t sel) {
typedef enum { V32, V64, F32, F64, X } valtype_t;
static valtype_t types[] = {
X, F64, F32, V64, V64, V32, F64, F32, V64, X, X, X, X, V32, V32, F32, F64,
V32, V64 };
static char fastpath_bytes[] = { 1, 1, 4, 8 };
const valtype_t vtype = types[op];
const int fastbytes = fastpath_bytes[vtype];
upb_func *handler = gethandler(h, sel);
upb_fieldtype_t ftype;
const upb_shim_data *data;
if (handler) {
|1:
| chkneob fastbytes, >3
|2:
switch (vtype) {
case V32:
| call ->decodev32_fallback
break;
case V64:
| call ->decodev64_fallback
break;
case F32:
| call ->decodef32_fallback
break;
case F64:
| call ->decodef64_fallback
break;
case X: break;
}
| jmp >4
/* Fast path decode; for when check_bytes bytes are available. */
|3:
switch (op) {
case OP_PARSE_SFIXED32:
case OP_PARSE_FIXED32:
| mov edx, dword [PTR]
break;
case OP_PARSE_SFIXED64:
case OP_PARSE_FIXED64:
| mov rdx, qword [PTR]
break;
case OP_PARSE_FLOAT:
| movss xmm0, dword [PTR]
break;
case OP_PARSE_DOUBLE:
| movsd xmm0, qword [PTR]
break;
default:
/* Inline one byte of varint decoding. */
| movzx edx, byte [PTR]
| test dl, dl
| js <2 // Fallback to slow path for >1 byte varint.
break;
}
/* Second-stage decode; used for both fast and slow paths */
/* (only needed for a few types). */
|4:
switch (op) {
case OP_PARSE_SINT32:
/* 32-bit zig-zag decode. */
| mov eax, edx
| shr edx, 1
| and eax, 1
| neg eax
| xor edx, eax
break;
case OP_PARSE_SINT64:
/* 64-bit zig-zag decode. */
| mov rax, rdx
| shr rdx, 1
| and rax, 1
| neg rax
| xor rdx, rax
break;
case OP_PARSE_BOOL:
| test rdx, rdx
| setne dl
break;
default: break;
}
/* Call callback (or specialize if we can). */
data = upb_shim_getdata(h, sel, &ftype);
if (data) {
switch (ftype) {
case UPB_TYPE_INT64:
case UPB_TYPE_UINT64:
| mov [CLOSURE + data->offset], rdx
break;
case UPB_TYPE_INT32:
case UPB_TYPE_UINT32:
case UPB_TYPE_ENUM:
| mov [CLOSURE + data->offset], edx
break;
case UPB_TYPE_DOUBLE:
| movsd qword [CLOSURE + data->offset], XMMARG1
break;
case UPB_TYPE_FLOAT:
| movss dword [CLOSURE + data->offset], XMMARG1
break;
case UPB_TYPE_BOOL:
| mov [CLOSURE + data->offset], dl
break;
case UPB_TYPE_STRING:
case UPB_TYPE_BYTES:
case UPB_TYPE_MESSAGE:
assert(false); break;
}
| sethas CLOSURE, data->hasbit
} else if (handler) {
| mov ARG1_64, CLOSURE
| load_handler_data h, sel
| callp handler
if (!alwaysok(h, sel)) {
| test al, al
| jnz >5
| call ->suspend
| jmp <1
|5:
}
}
/* We do this last so that the checkpoint is not advanced past the user's
* data until the callback has returned success. */
| add PTR, fastbytes
} else {
/* No handler registered for this value, just skip it. */
| chkneob fastbytes, >3
|2:
switch (vtype) {
case V32:
| call ->skipv32_fallback
break;
case V64:
| call ->skipv64_fallback
break;
case F32:
| call ->skipf32_fallback
break;
case F64:
| call ->skipf64_fallback
break;
case X: break;
}
/* Fast-path skip. */
|3:
if (vtype == V32 || vtype == V64) {
| test byte [PTR], 0x80
| jnz <2
}
| add PTR, fastbytes
}
}
static void jitdispatch(jitcompiler *jc,
const upb_pbdecodermethod *method) {
/* Lots of room for tweaking/optimization here. */
const upb_inttable *dispatch = &method->dispatch;
bool has_hash_entries = (dispatch->t.count > 0);
/* Whether any of the fields for this message can have two wire types which
* are both valid (packed & non-packed).
*
* OPT: populate this more precisely; not all messages with hash entries have
* this characteristic. */
bool has_multi_wiretype = has_hash_entries;
|=>define_jmptarget(jc, &method->dispatch):
|1:
/* Decode the field tag. */
| mov aword DECODER->checkpoint, PTR
| chkeob 2, >6
| movzx edx, byte [PTR]
| test dl, dl
| jns >7 // Jump if first byte has no continuation bit.
| movzx ecx, byte [PTR + 1]
| test cl, cl
| js >6 // Jump if second byte has continuation bit.
| // Confirmed two-byte varint.
| shl ecx, 7
| and edx, 0x7f
| or edx, ecx
| add PTR, 2
| jmp >8
|6:
| call ->decode_unknown_tag_fallback
| test eax, eax // Hit DELIMEND?
| jnz >8
| ret
|7:
| add PTR, 1
|8:
| mov ecx, edx
| shr edx, 3
| and cl, 7
/* See comment attached to upb_pbdecodermethod.dispatch for layout of the
* dispatch table. */
|2:
| cmp edx, dispatch->array_size
if (has_hash_entries) {
| jae >7
} else {
| jae >5
}
| // OPT: Compact the lookup arr into 32-bit entries.
if ((uintptr_t)dispatch->array > 0x7fffffff) {
| mov64 rax, (uintptr_t)dispatch->array
| mov rax, qword [rax + rdx * 8]
} else {
| mov rax, qword [rdx * 8 + dispatch->array]
}
|3:
| // We take advantage of the fact that non-present entries are stored
| // as -1, which will result in wire types that will never match.
| cmp al, cl
if (has_multi_wiretype) {
| jne >6
} else {
| jne >5
}
| shr rax, 16
|
| // Load the machine code address from the table entry.
| // The table entry is relative to the dispatch->array jmptarget
| // (patchdispatch() took care of this) which is the same as
| // local label "4". The "lea" is really just trying to do
| // lea rax, [>4 + rax]
| //
| // But we can't write that directly for some reason, so we use
| // rdx as a temporary.
| lea rdx, [>4]
|=>define_jmptarget(jc, dispatch->array):
|4:
| add rax, rdx
| ret
|
|5:
| // Field isn't in our table.
| call ->parse_unknown
| test eax, eax // ENDGROUP?
| jz <1
| lea rax, [>9] // ENDGROUP; Load address of OP_ENDMSG.
| ret
if (has_multi_wiretype) {
|6:
| // Primary wire type didn't match, check secondary wire type.
| cmp ah, cl
| jne <5
| // Secondary wire type is a match, look up fn + UPB_MAX_FIELDNUMBER.
| add rdx, UPB_MAX_FIELDNUMBER
| // This key will never be in the array part, so do a hash lookup.
assert(has_hash_entries);
| ld64 dispatch
| jmp ->hashlookup // Tail call.
}
if (has_hash_entries) {
|7:
| // Hash table lookup.
| ld64 dispatch
| call ->hashlookup
| jmp <3
}
}
static void jittag(jitcompiler *jc, uint64_t tag, int n, int ofs,
const upb_pbdecodermethod *method) {
/* Internally we parse unknown fields; if this runs us into DELIMEND we jump
* to the corresponding DELIMEND target (either msg end or repeated field
* end), which we find from the OP_CHECKDELIM which must have necessarily
* preceded us. */
uint32_t last_instruction = *(jc->pc - 2);
int last_arg = (int32_t)last_instruction >> 8;
uint32_t *delimend = (jc->pc - 1) + last_arg;
const size_t ptr_words = sizeof(void*) / sizeof(uint32_t);
assert((last_instruction & 0xff) == OP_CHECKDELIM);
if (getop(*(jc->pc - 1)) == OP_TAGN) {
jc->pc += ptr_words;
}
| chkneob n, >1
| // OPT: this is way too much fallback code to put here.
| // Reduce and/or move to a separate section to make better icache usage.
| ld64 tag
| call ->checktag_fallback
| cmp eax, DECODE_MISMATCH
| je >3
| cmp eax, DECODE_EOF
| je =>jmptarget(jc, delimend)
| jmp >5
|1:
switch (n) {
case 1:
| cmp byte [PTR], tag
break;
case 2:
| cmp word [PTR], tag
break;
case 3:
| // OPT: Slightly more efficient code, but depends on an extra byte.
| // mov eax, dword [PTR]
| // shl eax, 8
| // cmp eax, tag << 8
| cmp word [PTR], (tag & 0xffff)
| jne >2
| cmp byte [PTR + 2], (tag >> 16)
|2:
break;
case 4:
| cmp dword [PTR], tag
break;
case 5:
| cmp dword [PTR], (tag & 0xffffffff)
| jne >3
| cmp byte [PTR + 4], (tag >> 32)
}
| je >4
|3:
if (ofs == 0) {
| call =>jmptarget(jc, &method->dispatch)
| test rax, rax
| jz =>jmptarget(jc, delimend)
| jmp rax
} else {
| jmp =>jmptarget(jc, jc->pc + ofs)
}
|4:
| add PTR, n
|5:
}
/* Compile the bytecode to x64. */
static void jitbytecode(jitcompiler *jc) {
upb_pbdecodermethod *method = NULL;
const upb_handlers *h = NULL;
for (jc->pc = jc->group->bytecode; jc->pc < jc->group->bytecode_end; ) {
int32_t instr = *jc->pc;
opcode op = instr & 0xff;
uint32_t arg = instr >> 8;
int32_t longofs = arg;
if (op != OP_SETDISPATCH) {
/* Skipped for SETDISPATCH because it defines its own asmlabel for the
* dispatch code it emits. */
asmlabel(jc, "0x%lx.%s", pcofs(jc), upb_pbdecoder_getopname(op));
/* Skipped for SETDISPATCH because it should point at the function
* prologue, not the dispatch function that is emitted first.
* TODO: optimize this to only define pclabels that are actually used. */
|=>define_jmptarget(jc, jc->pc):
}
jc->pc++;
switch (op) {
case OP_STARTMSG: {
upb_func *startmsg = gethandler(h, UPB_STARTMSG_SELECTOR);
if (startmsg) {
/* bool startmsg(void *closure, const void *hd) */
|1:
| mov ARG1_64, CLOSURE
| load_handler_data h, UPB_STARTMSG_SELECTOR
| callp startmsg
if (!alwaysok(h, UPB_STARTMSG_SELECTOR)) {
| test al, al
| jnz >2
| call ->suspend
| jmp <1
|2:
}
} else {
| nop
}
break;
}
case OP_ENDMSG: {
upb_func *endmsg = gethandler(h, UPB_ENDMSG_SELECTOR);
|9:
if (endmsg) {
/* bool endmsg(void *closure, const void *hd, upb_status *status) */
| mov ARG1_64, CLOSURE
| load_handler_data h, UPB_ENDMSG_SELECTOR
| mov ARG3_64, DECODER->status
| callp endmsg
}
break;
}
case OP_SETDISPATCH: {
uint32_t *op_pc = jc->pc - 1;
const char *msgname;
upb_inttable *dispatch;
/* Load info for new method. */
memcpy(&dispatch, jc->pc, sizeof(void*));
jc->pc += sizeof(void*) / sizeof(uint32_t);
/* The OP_SETDISPATCH bytecode contains a pointer that is
* &method->dispatch; we want to go backwards and recover method. */
method =
(void*)((char*)dispatch - offsetof(upb_pbdecodermethod, dispatch));
/* May be NULL, in which case no handlers for this message will be found.
* OPT: we should do better by completely skipping the message in this
* case instead of parsing it field by field. We should also do the skip
* in the containing message's code. */
h = method->dest_handlers_;
msgname = upb_msgdef_fullname(upb_handlers_msgdef(h));
/* Emit dispatch code for new method. */
asmlabel(jc, "0x%lx.dispatch.%s", pcofs(jc), msgname);
jitdispatch(jc, method);
/* Emit function prologue for new method. */
asmlabel(jc, "0x%lx.parse.%s", pcofs(jc), msgname);
|=>define_jmptarget(jc, op_pc):
|=>define_jmptarget(jc, method):
| sub rsp, 8
break;
}
case OP_PARSE_DOUBLE:
case OP_PARSE_FLOAT:
case OP_PARSE_INT64:
case OP_PARSE_UINT64:
case OP_PARSE_INT32:
case OP_PARSE_FIXED64:
case OP_PARSE_FIXED32:
case OP_PARSE_BOOL:
case OP_PARSE_UINT32:
case OP_PARSE_SFIXED32:
case OP_PARSE_SFIXED64:
case OP_PARSE_SINT32:
case OP_PARSE_SINT64:
jitprimitive(jc, op, h, arg);
break;
case OP_STARTSEQ:
case OP_STARTSUBMSG:
case OP_STARTSTR: {
upb_func *start = gethandler(h, arg);
if (start) {
/* void *startseq(void *closure, const void *hd)
* void *startsubmsg(void *closure, const void *hd)
* void *startstr(void *closure, const void *hd, size_t size_hint) */
|1:
| mov ARG1_64, CLOSURE
| load_handler_data h, arg
if (op == OP_STARTSTR) {
| mov ARG3_64, DELIMEND
| sub ARG3_64, PTR
}
| callp start
if (!alwaysok(h, arg)) {
| test rax, rax
| jnz >2
| call ->suspend
| jmp <1
|2:
}
| mov CLOSURE, rax
} else {
/* TODO: nop is only required because of asmlabel(). */
| nop
}
break;
}
case OP_ENDSEQ:
case OP_ENDSUBMSG:
case OP_ENDSTR: {
upb_func *end = gethandler(h, arg);
if (end) {
/* bool endseq(void *closure, const void *hd)
* bool endsubmsg(void *closure, const void *hd)
* bool endstr(void *closure, const void *hd) */
|1:
| mov ARG1_64, CLOSURE
| load_handler_data h, arg
| callp end
if (!alwaysok(h, arg)) {
| test al, al
| jnz >2
| call ->suspend
| jmp <1
|2:
}
} else {
/* TODO: nop is only required because of asmlabel(). */
| nop
}
break;
}
case OP_STRING: {
upb_func *str = gethandler(h, arg);
| cmp PTR, DELIMEND
| je >4
|1:
| cmp PTR, DATAEND
| jne >2
| call ->suspend
| jmp <1
|2:
if (str) {
/* size_t str(void *closure, const void *hd, const char *str,
* size_t n) */
| mov ARG1_64, CLOSURE
| load_handler_data h, arg
| mov ARG3_64, PTR
| mov ARG4_64, DATAEND
| sub ARG4_64, PTR
| mov ARG5_64, qword DECODER->handle
| callp str
| add PTR, rax
if (!alwaysok(h, arg)) {
| cmp PTR, DATAEND
| je >3
| call ->strret_fallback
|3:
}
} else {
| mov PTR, DATAEND
}
| cmp PTR, DELIMEND
| jne <1
|4:
break;
}
case OP_PUSHTAGDELIM:
| mov FRAME->sink.closure, CLOSURE
| // This shouldn't need to be read, because tag-delimited fields
| // shouldn't have an OP_SETDELIM after them. But for the moment
| // non-packed repeated fields do OP_SETDELIM so they can share more
| // code with the packed code-path. If this is changed later, this
| // store can be removed.
| mov qword FRAME->end_ofs, 0
| cmp FRAME, DECODER->limit
| je ->err
| add FRAME, sizeof(upb_pbdecoder_frame)
| mov dword FRAME->groupnum, arg
break;
case OP_PUSHLENDELIM:
| call ->pushlendelim
break;
case OP_POP:
| sub FRAME, sizeof(upb_pbdecoder_frame)
| mov CLOSURE, FRAME->sink.closure
break;
case OP_SETDELIM:
/* OPT: experiment with testing vs old offset to optimize away. */
| mov DATAEND, DECODER->end
| add DELIMEND, FRAME->end_ofs
| cmp DELIMEND, DECODER->buf
| jb >1
| cmp DELIMEND, DATAEND
| ja >1 // OPT: try cmov.
| mov DATAEND, DELIMEND
|1:
break;
case OP_SETBIGGROUPNUM:
| mov dword FRAME->groupnum, *jc->pc++
break;
case OP_CHECKDELIM:
| cmp DELIMEND, PTR
| je =>jmptarget(jc, jc->pc + longofs)
break;
case OP_CALL:
| call =>jmptarget(jc, jc->pc + longofs)
break;
case OP_BRANCH:
| jmp =>jmptarget(jc, jc->pc + longofs);
break;
case OP_RET:
|9:
| add rsp, 8
| ret
break;
case OP_TAG1:
jittag(jc, (arg >> 8) & 0xff, 1, (int8_t)arg, method);
break;
case OP_TAG2:
jittag(jc, (arg >> 8) & 0xffff, 2, (int8_t)arg, method);
break;
case OP_TAGN: {
uint64_t tag;
memcpy(&tag, jc->pc, 8);
jittag(jc, tag, arg >> 8, (int8_t)arg, method);
break;
}
case OP_DISPATCH:
| call =>jmptarget(jc, &method->dispatch)
break;
case OP_HALT:
assert(false);
}
}
asmlabel(jc, "eof");
| nop
}