Chiebot-Mirror
/
protobuf
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Protocol Buffers - Google's data interchange format (grpc依赖) https://developers.google.com/protocol-buffers/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
495 Commits
204 Branches
369 Tags
184 MiB
 
 
 
 
 
 
Tag: Branch: Tree: 86bad61b76
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '86bad61b76'
${ noResults }
protobuf/upb/pb/decoder_x64.dasc

808 lines
25 KiB
Raw Blame History

|//
|// upb - a minimalist implementation of protocol buffers.
|//
|// Copyright (c) 2011 Google Inc. See LICENSE for details.
|// Author: Josh Haberman <jhaberman@gmail.com>
|//
|// JIT compiler for upb_decoder on x86. Given a upb_decoderplan object (which
|// contains an embedded set of upb_handlers), generates code specialized to
|// parsing the specific message and calling specific handlers.
|//
|// Since the JIT can call other functions (the JIT'ted code is not a leaf
|// function) we must respect alignment rules. All x86-64 systems require
|// 16-byte stack alignment.
#include <sys/mman.h>
#include "dynasm/dasm_x86.h"
#ifndef MAP_ANONYMOUS
# define MAP_ANONYMOUS MAP_ANON
#endif
// We map into the low 32 bits when we can, but if this is not available
// (like on OS X) we take what we can get. It's not required for correctness,
// it's just a performance thing that makes it more likely that our jumps
// can be rel32 (i.e. within 32-bits of our pc) instead of the longer
// sequence required for other jumps (see callp).
#ifndef MAP_32BIT
#define MAP_32BIT 0
#endif
// To debug JIT-ted code with GDB we need to tell GDB about the JIT-ted code
// at runtime. GDB 7.x+ has defined an interface for doing this, and these
// structure/function defintions are copied out of gdb/jit.h
//
// We need to give GDB an ELF file at runtime describing the symbols we have
// generated. To avoid implementing the ELF format, we generate an ELF file
// at compile-time and compile it in as a character string. We can replace
// a few key constants (address of JIT-ted function and its size) by looking
// for a few magic numbers and doing a dumb string replacement.
#ifndef __APPLE__
const unsigned char upb_jit_debug_elf_file[] = {
#include "upb/pb/jit_debug_elf_file.h"
};
typedef enum
{
GDB_JIT_NOACTION = 0,
GDB_JIT_REGISTER,
GDB_JIT_UNREGISTER
} jit_actions_t;
typedef struct gdb_jit_entry {
struct gdb_jit_entry *next_entry;
struct gdb_jit_entry *prev_entry;
const char *symfile_addr;
uint64_t symfile_size;
} gdb_jit_entry;
typedef struct {
uint32_t version;
uint32_t action_flag;
gdb_jit_entry *relevant_entry;
gdb_jit_entry *first_entry;
} gdb_jit_descriptor;
gdb_jit_descriptor __jit_debug_descriptor = {1, GDB_JIT_NOACTION, NULL, NULL};
void __attribute__((noinline)) __jit_debug_register_code() { __asm__ __volatile__(""); }
void upb_reg_jit_gdb(upb_decoderplan *plan) {
// Create debug info.
size_t elf_len = sizeof(upb_jit_debug_elf_file);
plan->debug_info = malloc(elf_len);
memcpy(plan->debug_info, upb_jit_debug_elf_file, elf_len);
uint64_t *p = (void*)plan->debug_info;
for (; (void*)(p+1) <= (void*)plan->debug_info + elf_len; ++p) {
if (*p == 0x12345678) { *p = (uintptr_t)plan->jit_code; }
if (*p == 0x321) { *p = plan->jit_size; }
}
// Register the JIT-ted code with GDB.
gdb_jit_entry *e = malloc(sizeof(gdb_jit_entry));
e->next_entry = __jit_debug_descriptor.first_entry;
e->prev_entry = NULL;
if (e->next_entry) e->next_entry->prev_entry = e;
e->symfile_addr = plan->debug_info;
e->symfile_size = elf_len;
__jit_debug_descriptor.first_entry = e;
__jit_debug_descriptor.relevant_entry = e;
__jit_debug_descriptor.action_flag = GDB_JIT_REGISTER;
__jit_debug_register_code();
}
#else
void upb_reg_jit_gdb(upb_decoderplan *plan) {
(void)plan;
}
#endif
// Has to be a separate function, otherwise GCC will complain about
// expressions like (&foo != NULL) because they will never evaluate
// to false.
static void upb_assert_notnull(void *addr) { assert(addr != NULL); (void)addr; }
|.arch x64
|.actionlist upb_jit_actionlist
|.globals UPB_JIT_GLOBAL_
|.globalnames upb_jit_globalnames
|
|// Calling conventions. Note -- this will need to be changed for
|// Windows, which uses a different calling convention!
|.define ARG1_64, rdi
|.define ARG2_8, sil
|.define ARG2_32, esi
|.define ARG2_64, rsi
|.define ARG3_8, dl
|.define ARG3_32, edx
|.define ARG3_64, rdx
|.define ARG4_64, rcx
|.define ARG5_32, r8d
|
|// Register allocation / type map.
|// ALL of the code in this file uses these register allocations.
|// When we "call" within this file, we do not use regular calling
|// conventions, but of course when calling to user callbacks we must.
|.define PTR, rbx // Writing this to DECODER->ptr commits our progress.
|.define CLOSURE, r12
|.type FRAME, upb_dispatcher_frame, r13
|.type BYTEREGION,upb_byteregion, r14
|.type DECODER, upb_decoder, r15
|.type STDARRAY, upb_stdarray
|
|.macro callp, addr
|| upb_assert_notnull(addr);
|| if ((uintptr_t)addr < 0xffffffff) {
| call &addr
|| } else {
| mov64 rax, (uintptr_t)addr
| call rax
|| }
|.endmacro
|
|// Checks PTR for end-of-buffer.
|.macro check_eob, m
| cmp PTR, DECODER->effective_end
|| if (m->is_group) {
| jae ->exit_jit
|| } else {
| jae =>m->jit_endofbuf_pclabel
|| }
|.endmacro
|
|// Decodes varint from [PTR + offset] -> ARG3.
|// Saves new pointer as rax.
|.macro decode_loaded_varint, offset
| // Check for <=2 bytes inline, otherwise jump to 2-10 byte decoder.
| lea rax, [PTR + offset + 1]
| mov ARG3_32, ecx
| and ARG3_32, 0x7f
| test cl, cl
| jns >9
| lea rax, [PTR + offset + 2]
| movzx esi, ch
| and esi, 0x7f
| shl esi, 7
| or ARG3_32, esi
| test cx, cx
| jns >9
| mov ARG1_64, rax
| mov ARG2_32, ARG3_32
| callp upb_vdecode_max8_fast
| test rax, rax
| jz ->exit_jit // >10-byte varint.
|9:
|.endmacro
|
|.macro decode_varint, offset
| mov ecx, dword [PTR + offset]
| decode_loaded_varint offset
| mov PTR, rax
|.endmacro
|
|// Decode the tag -> edx.
|// Could specialize this by avoiding the value masking: could just key the
|// table on the raw (length-masked) varint to save 3-4 cycles of latency.
|// Currently only support tables where all entries are in the array part.
|.macro dyndispatch_, m
|=>m->jit_dyndispatch_pclabel:
| decode_loaded_varint, 0
| mov ecx, edx
| shr ecx, 3
| and edx, 0x7 // For the type check that will happen later.
| cmp ecx, m->max_field_number // Bounds-check the field.
| ja ->exit_jit // In the future; could be unknown label
|| if ((uintptr_t)m->tablearray < 0xffffffff) {
| // TODO: support hybrid array/hash tables.
| mov rax, qword [rcx*8 + m->tablearray]
|| } else {
| mov64 rax, (uintptr_t)m->tablearray
| mov rax, qword [rax + rcx*8]
|| }
| jmp rax // Dispatch: unpredictable jump.
|.endmacro
|
|.if 1
| // Replicated dispatch: larger code, but better branch prediction.
| .define dyndispatch, dyndispatch_
|.else
| .macro dyndispatch, m
| jmp =>m->jit_dyndispatch_pclabel
| .endmacro
|.endif
|
|// Push a stack frame (not the CPU stack, the upb_decoder stack).
|.macro pushframe, f, end_offset_, is_sequence_
| lea rax, [FRAME + sizeof(upb_dispatcher_frame)] // rax for shorter addressing.
| cmp rax, qword DECODER->dispatcher.limit
| jae ->exit_jit // Frame stack overflow.
| mov64 r8, (uintptr_t)f
| mov qword FRAME:rax->f, r8
| mov qword FRAME:rax->end_ofs, end_offset_
| mov byte FRAME:rax->is_sequence, is_sequence_
| mov DECODER->dispatcher.top, rax
| mov FRAME, rax
|.endmacro
|
|.macro popframe, m
| sub FRAME, sizeof(upb_dispatcher_frame)
| mov DECODER->dispatcher.top, FRAME
| setmsgend m
| mov CLOSURE, FRAME->closure
|.endmacro
|
|.macro setmsgend, m
| mov rsi, DECODER->jit_end
|| if (m->is_group) {
| mov64 rax, 0xffffffffffffffff
| mov qword DECODER->delim_end, rax
| mov DECODER->effective_end, rsi
|| } else {
| // Could store a correctly-biased version in the frame, at the cost of
| // a larger stack.
| mov eax, dword FRAME->end_ofs
| add rax, qword DECODER->buf
| mov DECODER->delim_end, rax // delim_end = d->buf + f->end_ofs
| cmp rax, rsi
| jb >8
| mov rax, rsi // effective_end = min(d->delim_end, d->jit_end)
|8:
| mov DECODER->effective_end, rax
|| }
|.endmacro
|
|// rax contains the tag, compare it against "tag", but since it is a varint
|// we must only compare as many bytes as actually have data.
|.macro checktag, tag
|| switch (upb_value_size(tag)) {
|| case 1:
| cmp cl, tag
|| break;
|| case 2:
| cmp cx, tag
|| break;
|| case 3:
| and ecx, 0xffffff // 3 bytes
| cmp rcx, tag
|| case 4:
| cmp ecx, tag
|| break;
|| case 5:
| mov64 rdx, 0xffffffffff // 5 bytes
| and rcx, rdx
| cmp rcx, tag
|| break;
|| default: abort();
|| }
|.endmacro
|
|// TODO: optimize for 0 (xor) and 32-bits.
|.macro loadfval, f
||#ifndef NDEBUG
||// Since upb_value carries type information in debug mode
||// only, we need to pass the arguments slightly differently.
| mov ARG3_32, f->fval.type
||#endif
|| if (f->fval.val.uint64 == 0) {
| xor ARG2_32, ARG2_32
|| } else if (f->fval.val.uint64 < 0xffffffff) {
| mov ARG2_32, f->fval.val.uint64
|| } else {
| mov64 ARG2_64, f->fval.val.uint64
|| }
|.endmacro
|
|.macro sethas, reg, hasbit
|| if (hasbit >= 0) {
| or byte [reg + ((uint32_t)hasbit / 8)], (1 << ((uint32_t)hasbit % 8))
|| }
|.endmacro
#include <stdlib.h>
#include "upb/pb/varint.h"
#include "upb/msg.h"
// Decodes the next val into ARG3, advances PTR.
static void upb_decoderplan_jit_decodefield(upb_decoderplan *plan,
uint8_t type, size_t tag_size) {
// Decode the value into arg 3 for the callback.
switch (type) {
case UPB_TYPE(DOUBLE):
case UPB_TYPE(FIXED64):
case UPB_TYPE(SFIXED64):
| mov ARG3_64, qword [PTR + tag_size]
| add PTR, 8 + tag_size
break;
case UPB_TYPE(FLOAT):
case UPB_TYPE(FIXED32):
case UPB_TYPE(SFIXED32):
| mov ARG3_32, dword [PTR + tag_size]
| add PTR, 4 + tag_size
break;
case UPB_TYPE(BOOL):
// Can't assume it's one byte long, because bool must be wire-compatible
// with all of the varint integer types.
| decode_varint tag_size
| test ARG3_64, ARG3_64
| setne ARG3_8 // Other bytes left with val, should be ok.
break;
case UPB_TYPE(INT64):
case UPB_TYPE(UINT64):
case UPB_TYPE(INT32):
case UPB_TYPE(UINT32):
case UPB_TYPE(ENUM):
| decode_varint tag_size
break;
case UPB_TYPE(SINT64):
// 64-bit zig-zag decoding.
| decode_varint tag_size
| mov rax, ARG3_64
| shr ARG3_64, 1
| and rax, 1
| neg rax
| xor ARG3_64, rax
break;
case UPB_TYPE(SINT32):
// 32-bit zig-zag decoding.
| decode_varint tag_size
| mov eax, ARG3_32
| shr ARG3_32, 1
| and eax, 1
| neg eax
| xor ARG3_32, eax
break;
case UPB_TYPE(STRING):
case UPB_TYPE(BYTES):
// We only handle the case where the entire string is in our current
// buf, which sidesteps any security problems. The C path has more
// robust checks.
| mov ecx, dword [PTR + tag_size]
| decode_loaded_varint tag_size
| mov rdi, DECODER->end
| sub rdi, rax
| cmp ARG3_64, rdi // if (len > d->end - str)
| ja ->exit_jit // Can't deliver, whole string not in buf.
// Update PTR to point past end of string.
| mov rdi, rax
| add rdi, ARG3_64
| mov PTR, rdi
// Populate BYTEREGION appropriately.
| sub rax, DECODER->buf
| add rax, DECODER->bufstart_ofs // = d->ptr - d->buf + d->bufstart_ofs
| mov BYTEREGION->start, rax
| mov BYTEREGION->discard, rax
| add rax, ARG3_64
| mov BYTEREGION->end, rax
| mov BYTEREGION->fetch, rax // Fast path ensures whole string is loaded
| mov ARG3_64, BYTEREGION
break;
// Will dispatch callbacks and call submessage in a second.
case UPB_TYPE(MESSAGE):
| decode_varint tag_size
break;
case UPB_TYPE(GROUP):
| add PTR, tag_size
break;
default: abort();
}
}
static void upb_decoderplan_jit_callcb(upb_decoderplan *plan,
upb_fhandlers *f) {
// Call callbacks. Specializing the append accessors didn't yield a speed
// increase in benchmarks.
if (upb_issubmsgtype(f->type)) {
if (f->type == UPB_TYPE(MESSAGE)) {
| mov rsi, PTR
| sub rsi, DECODER->buf
| add rsi, ARG3_64 // = (d->ptr - d->buf) + delim_len
} else {
assert(f->type == UPB_TYPE(GROUP));
| mov rsi, UPB_NONDELIMITED
}
| pushframe f, rsi, false
// Call startsubmsg handler (if any).
if (f->startsubmsg) {
// upb_sflow_t startsubmsg(void *closure, upb_value fval)
| mov ARG1_64, CLOSURE
| loadfval f
| callp f->startsubmsg
| sethas CLOSURE, f->hasbit
| mov CLOSURE, rdx
} else {
| sethas CLOSURE, f->hasbit
}
| mov qword FRAME->closure, CLOSURE
// TODO: Handle UPB_SKIPSUBMSG, UPB_BREAK
| mov DECODER->ptr, PTR
const upb_mhandlers *sub_m = upb_fhandlers_getsubmsg(f);
| call =>sub_m->jit_startmsg_pclabel;
| popframe upb_fhandlers_getmsg(f)
// Call endsubmsg handler (if any).
if (f->endsubmsg) {
// upb_flow_t endsubmsg(void *closure, upb_value fval);
| mov ARG1_64, CLOSURE
| loadfval f
| callp f->endsubmsg
}
// TODO: Handle UPB_SKIPSUBMSG, UPB_BREAK
| mov DECODER->ptr, PTR
} else {
| mov ARG1_64, CLOSURE
// Test for callbacks we can specialize.
// Can't switch() on function pointers.
if (f->value == &upb_stdmsg_setint64 ||
f->value == &upb_stdmsg_setuint64 ||
f->value == &upb_stdmsg_setptr ||
f->value == &upb_stdmsg_setdouble) {
const upb_fielddef *fd = upb_value_getfielddef(f->fval);
| mov [ARG1_64 + fd->offset], ARG3_64
} else if (f->value == &upb_stdmsg_setint32 ||
f->value == &upb_stdmsg_setuint32 ||
f->value == &upb_stdmsg_setfloat) {
const upb_fielddef *fd = upb_value_getfielddef(f->fval);
| mov [ARG1_64 + fd->offset], ARG3_32
} else if (f->value == &upb_stdmsg_setbool) {
const upb_fielddef *fd = upb_value_getfielddef(f->fval);
| mov [ARG1_64 + fd->offset], ARG3_8
} else if (f->value) {
// Load closure and fval into arg registers.
||#ifndef NDEBUG
||// Since upb_value carries type information in debug mode
||// only, we need to pass the arguments slightly differently.
| mov ARG4_64, ARG3_64
| mov ARG5_32, upb_types[f->type].inmemory_type
||#endif
| loadfval f
| callp f->value
}
| sethas CLOSURE, f->hasbit
// TODO: Handle UPB_SKIPSUBMSG, UPB_BREAK
| mov DECODER->ptr, PTR
}
}
static uint64_t upb_get_encoded_tag(upb_fhandlers *f) {
uint32_t tag = (f->number << 3) | upb_decoder_types[f->type].native_wire_type;
uint64_t encoded_tag = upb_vencode32(tag);
// No tag should be greater than 5 bytes.
assert(encoded_tag <= 0xffffffffff);
return encoded_tag;
}
// PTR should point to the beginning of the tag.
static void upb_decoderplan_jit_field(upb_decoderplan *plan, upb_mhandlers *m,
upb_fhandlers *f, upb_fhandlers *next_f) {
uint64_t tag = upb_get_encoded_tag(f);
uint64_t next_tag = next_f ? upb_get_encoded_tag(next_f) : 0;
// PC-label for the dispatch table.
// We check the wire type (which must be loaded in edx) because the
// table is keyed on field number, not type.
|=>f->jit_pclabel:
| cmp edx, (tag & 0x7)
| jne ->exit_jit // In the future: could be an unknown field or packed.
|=>f->jit_pclabel_notypecheck:
if (f->repeated) {
| mov rsi, FRAME->end_ofs
| pushframe f, rsi, true
if (f->startseq) {
| mov ARG1_64, CLOSURE
| loadfval f
| callp f->startseq
| sethas CLOSURE, f->hasbit
| mov CLOSURE, rdx
} else {
| sethas CLOSURE, f->hasbit
}
| mov qword FRAME->closure, CLOSURE
}
|1: // Label for repeating this field.
int tag_size = upb_value_size(tag);
if (f->type == UPB_TYPE_ENDGROUP) {
| add PTR, tag_size
| jmp =>m->jit_endofmsg_pclabel
return;
}
upb_decoderplan_jit_decodefield(plan, f->type, tag_size);
upb_decoderplan_jit_callcb(plan, f);
// Epilogue: load next tag, check for repeated field.
| check_eob m
| mov rcx, qword [PTR]
if (f->repeated) {
| checktag tag
| je <1
if (f->endseq) {
| mov ARG1_64, CLOSURE
| loadfval f
| callp f->endseq
}
| popframe m
}
if (next_tag != 0) {
| checktag next_tag
| je =>next_f->jit_pclabel_notypecheck
}
// Fall back to dynamic dispatch.
| dyndispatch m
|1:
}
static int upb_compare_uint32(const void *a, const void *b) {
// TODO: always put ENDGROUP at the end.
return *(uint32_t*)a - *(uint32_t*)b;
}
static void upb_decoderplan_jit_msg(upb_decoderplan *plan, upb_mhandlers *m) {
|=>m->jit_afterstartmsg_pclabel:
// There was a call to get here, so we need to align the stack.
| sub rsp, 8
| jmp >1
|=>m->jit_startmsg_pclabel:
// There was a call to get here, so we need to align the stack.
| sub rsp, 8
// Call startmsg handler (if any):
if (m->startmsg) {
// upb_flow_t startmsg(void *closure);
| mov ARG1_64, FRAME->closure
| callp m->startmsg
// TODO: Handle UPB_SKIPSUBMSG, UPB_BREAK
}
|1:
| setmsgend m
| check_eob m
| mov ecx, dword [PTR]
| dyndispatch_ m
// --------- New code section (does not fall through) ------------------------
// Emit code for parsing each field (dynamic dispatch contains pointers to
// all of these).
// Create an ordering over the fields (inttable ordering is undefined).
int num_keys = upb_inttable_count(&m->fieldtab);
uint32_t *keys = malloc(num_keys * sizeof(*keys));
int idx = 0;
upb_inttable_iter i;
upb_inttable_begin(&i, &m->fieldtab);
for(; !upb_inttable_done(&i); upb_inttable_next(&i)) {
keys[idx++] = upb_inttable_iter_key(&i);
}
qsort(keys, num_keys, sizeof(uint32_t), &upb_compare_uint32);
for(int i = 0; i < num_keys; i++) {
upb_fhandlers *f = upb_mhandlers_lookup(m, keys[i]);
upb_fhandlers *next_f =
(i + 1 < num_keys) ? upb_mhandlers_lookup(m, keys[i + 1]) : NULL;
upb_decoderplan_jit_field(plan, m, f, next_f);
}
free(keys);
// --------- New code section (does not fall through) ------------------------
// End-of-buf / end-of-message.
if (!m->is_group) {
// This case doesn't exist for groups, because there eob really means
// eob, so that case just exits the jit directly.
|=>m->jit_endofbuf_pclabel:
| cmp PTR, DECODER->delim_end
| jb ->exit_jit // We are at eob, but not end-of-submsg.
}
|=>m->jit_endofmsg_pclabel:
// We are at end-of-submsg: call endmsg handler (if any):
if (m->endmsg) {
// void endmsg(void *closure, upb_status *status) {
| mov ARG1_64, FRAME->closure
| lea ARG2_64, DECODER->dispatcher.status
| callp m->endmsg
}
if (m->is_group) {
// Advance past the "end group" tag.
// TODO: Handle UPB_BREAK
| mov DECODER->ptr, PTR
}
// Counter previous alignment.
| add rsp, 8
| ret
}
static void upb_decoderplan_jit(upb_decoderplan *plan) {
// The JIT prologue/epilogue trampoline that is generated in this function
// does not depend on the handlers, so it will never vary. Ideally we would
// put it in an object file and just link it into upb so we could have only a
// single copy of it instead of one copy for each decoderplan. But our
// options for doing that are undesirable: GCC inline assembly is
// complicated, not portable to other compilers, and comes with subtle
// caveats about incorrect things what the optimizer might do if you eg.
// execute non-local jumps. Putting this code in a .s file would force us to
// calculate the structure offsets ourself instead of symbolically
// (ie. [r15 + 0xcd] instead of DECODER->ptr). So we tolerate a bit of
// unnecessary duplication/redundancy.
| push rbp
| mov rbp, rsp
| push r15
| push r14
| push r13
| push r12
| push rbx
// Align stack.
| sub rsp, 8
| mov DECODER, ARG1_64
| mov FRAME, DECODER:ARG1_64->dispatcher.top
| lea BYTEREGION, DECODER:ARG1_64->str_byteregion
| mov CLOSURE, FRAME->closure
| mov PTR, DECODER->ptr
// TODO: push return addresses for re-entry (will be necessary for multiple
// buffer support).
| call ARG2_64
|->exit_jit:
// Restore stack pointer to where it was before any "call" instructions
// inside our generated code.
| lea rsp, [rbp - 48]
// Counter previous alignment.
| add rsp, 8
| pop rbx
| pop r12
| pop r13
| pop r14
| pop r15
| leave
| ret
upb_handlers *h = plan->handlers;
for (int i = 0; i < h->msgs_len; i++)
upb_decoderplan_jit_msg(plan, h->msgs[i]);
}
static void upb_decoderplan_jit_assignfieldlabs(upb_fhandlers *f,
uint32_t *pclabel_count) {
f->jit_pclabel = (*pclabel_count)++;
f->jit_pclabel_notypecheck = (*pclabel_count)++;
}
static void upb_decoderplan_jit_assignmsglabs(upb_mhandlers *m,
uint32_t *pclabel_count) {
m->jit_startmsg_pclabel = (*pclabel_count)++;
m->jit_afterstartmsg_pclabel = (*pclabel_count)++;
m->jit_endofbuf_pclabel = (*pclabel_count)++;
m->jit_endofmsg_pclabel = (*pclabel_count)++;
m->jit_dyndispatch_pclabel = (*pclabel_count)++;
m->jit_unknownfield_pclabel = (*pclabel_count)++;
m->max_field_number = 0;
upb_inttable_iter i;
upb_inttable_begin(&i, &m->fieldtab);
for(; !upb_inttable_done(&i); upb_inttable_next(&i)) {
uint32_t key = upb_inttable_iter_key(&i);
m->max_field_number = UPB_MAX(m->max_field_number, key);
upb_fhandlers *f = upb_value_getptr(upb_inttable_iter_value(&i));
upb_decoderplan_jit_assignfieldlabs(f, pclabel_count);
}
// TODO: support large field numbers by either using a hash table or
// generating code for a binary search. For now large field numbers
// will just fall back to the table decoder.
m->max_field_number = UPB_MIN(m->max_field_number, 16000);
m->tablearray = malloc((m->max_field_number + 1) * sizeof(void*));
}
static void upb_decoderplan_makejit(upb_decoderplan *plan) {
plan->debug_info = NULL;
// Assign pclabels.
uint32_t pclabel_count = 0;
upb_handlers *h = plan->handlers;
for (int i = 0; i < h->msgs_len; i++)
upb_decoderplan_jit_assignmsglabs(h->msgs[i], &pclabel_count);
void **globals = malloc(UPB_JIT_GLOBAL__MAX * sizeof(*globals));
dasm_init(plan, 1);
dasm_setupglobal(plan, globals, UPB_JIT_GLOBAL__MAX);
dasm_growpc(plan, pclabel_count);
dasm_setup(plan, upb_jit_actionlist);
upb_decoderplan_jit(plan);
int dasm_status = dasm_link(plan, &plan->jit_size);
(void)dasm_status;
assert(dasm_status == DASM_S_OK);
plan->jit_code = mmap(NULL, plan->jit_size, PROT_READ | PROT_WRITE,
MAP_32BIT | MAP_ANONYMOUS | MAP_PRIVATE, 0, 0);
upb_reg_jit_gdb(plan);
dasm_encode(plan, plan->jit_code);
// Create dispatch tables.
for (int i = 0; i < h->msgs_len; i++) {
upb_mhandlers *m = h->msgs[i];
// We jump to after the startmsg handler since it is called before entering
// the JIT (either by upb_decoder or by a previous call to the JIT).
m->jit_func =
plan->jit_code + dasm_getpclabel(plan, m->jit_afterstartmsg_pclabel);
for (uint32_t j = 0; j <= m->max_field_number; j++) {
upb_fhandlers *f = upb_mhandlers_lookup(m, j);
if (f) {
m->tablearray[j] =
plan->jit_code + dasm_getpclabel(plan, f->jit_pclabel);
} else {
// TODO: extend the JIT to handle unknown fields.
// For the moment we exit the JIT for any unknown field.
m->tablearray[j] = globals[UPB_JIT_GLOBAL_exit_jit];
}
}
}
dasm_free(plan);
free(globals);
mprotect(plan->jit_code, plan->jit_size, PROT_EXEC | PROT_READ);
// View with: objdump -M intel -D -b binary -mi386 -Mx86-64 /tmp/machine-code
// Or: ndisasm -b 64 /tmp/machine-code
FILE *f = fopen("/tmp/machine-code", "wb");
fwrite(plan->jit_code, plan->jit_size, 1, f);
fclose(f);
}
static void upb_decoderplan_freejit(upb_decoderplan *plan) {
munmap(plan->jit_code, plan->jit_size);
free(plan->debug_info);
// TODO: unregister
}
static void upb_decoder_enterjit(upb_decoder *d) {
if (d->plan->jit_code &&
d->dispatcher.top == d->dispatcher.stack &&
d->ptr && d->ptr < d->jit_end) {
#ifndef NDEBUG
register uint64_t rbx asm ("rbx") = 11;
register uint64_t r12 asm ("r12") = 12;
register uint64_t r13 asm ("r13") = 13;
register uint64_t r14 asm ("r14") = 14;
register uint64_t r15 asm ("r15") = 15;
#endif
// Decodes as many fields as possible, updating d->ptr appropriately,
// before falling through to the slow(er) path.
void (*upb_jit_decode)(upb_decoder *d, void*) = (void*)d->plan->jit_code;
upb_jit_decode(d, d->plan->handlers->msgs[d->msg_offset]->jit_func);
assert(d->ptr <= d->end);
// Test that callee-save registers were properly restored.
assert(rbx == 11);
assert(r12 == 12);
assert(r13 == 13);
assert(r14 == 14);
assert(r15 == 15);
}
}