Chiebot-Mirror
/
protobuf
mirror of https://github.com/protocolbuffers/protobuf.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity
Protocol Buffers - Google's data interchange format (grpc依赖) https://developers.google.com/protocol-buffers/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
702 Commits
197 Branches
346 Tags
251 MiB
 
 
 
 
 
 
Tag: Branch: Tree: 512130adf1
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '512130adf1'
${ noResults }
protobuf/upb/pb/compile_decoder_x64.h

1726 lines
64 KiB
Raw Blame History

/*
** This file has been pre-processed with DynASM.
** http://luajit.org/dynasm.html
** DynASM version 1.3.0, DynASM x64 version 1.3.0
** DO NOT EDIT! The original file is in "upb/pb/compile_decoder_x64.dasc".
*/
#if DASM_VERSION != 10300
#error "Version mismatch between DynASM and included encoding engine"
#endif
# 1 "upb/pb/compile_decoder_x64.dasc"
/*|// */
/*|// upb - a minimalist implementation of protocol buffers. */
/*|// */
/*|// Copyright (c) 2011-2013 Google Inc. See LICENSE for details. */
/*|// Author: Josh Haberman <jhaberman@gmail.com> */
/*|// */
/*|// JIT compiler for upb_pbdecoder on x86-64. Generates machine code from the */
/*|// bytecode generated in compile_decoder.c. */
/*| */
/*|.arch x64 */
/*|.actionlist upb_jit_actionlist */
static const unsigned char upb_jit_actionlist[2459] = {
249,255,248,10,248,1,85,65,87,65,86,65,85,65,84,83,72,137,252,243,73,137,
252,255,72,184,237,237,65,84,73,137,228,72,129,228,239,252,255,208,76,137,
228,65,92,133,192,15,137,244,247,73,137,167,233,72,137,216,77,139,183,233,
73,139,159,233,77,139,167,233,77,139,174,233,73,139,174,233,73,43,175,233,
73,3,175,233,73,139,151,233,72,133,210,15,133,244,248,252,255,208,73,139,
135,233,73,199,135,233,0,0,0,0,248,1,255,91,65,92,65,93,65,94,65,95,93,195,
248,2,73,139,183,233,72,41,212,72,137,231,72,184,237,237,65,84,73,137,228,
72,129,228,239,252,255,208,76,137,228,65,92,195,255,248,11,73,139,191,233,
72,137,230,73,139,151,233,72,41,226,73,137,151,233,137,195,72,184,237,237,
65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,92,137,216,73,139,
167,233,91,65,92,65,93,65,94,65,95,93,195,255,248,12,73,57,159,233,15,132,
244,247,73,137,159,233,248,1,77,137,183,233,73,137,159,233,77,137,167,233,
73,137,175,233,73,43,175,233,73,3,175,233,73,137,174,233,77,137,174,233,76,
137,252,255,72,184,237,237,65,84,73,137,228,72,129,228,239,252,255,208,76,
137,228,65,92,252,233,244,11,255,248,13,248,1,77,137,174,233,73,137,159,233,
255,76,57,227,15,132,244,253,255,76,137,225,72,41,217,72,131,252,249,1,15,
130,244,253,255,15,182,19,132,210,15,137,244,254,248,7,232,244,14,248,8,72,
131,195,1,72,137,252,233,72,41,217,72,41,209,15,130,244,250,73,137,142,233,
77,59,183,233,15,132,244,249,73,129,198,239,72,137,221,72,1,213,65,199,134,
233,0,0,0,0,72,133,201,15,132,244,248,77,139,167,233,72,57,252,235,15,135,
244,248,76,57,229,15,135,244,248,255,73,137,252,236,248,2,195,248,3,73,139,
159,233,76,137,252,255,255,72,190,237,237,255,190,237,255,49,252,246,255,
72,184,237,237,65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,
92,232,244,12,252,233,244,1,248,4,73,139,159,233,76,137,252,255,255,72,184,
237,237,65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,92,232,
244,12,252,233,244,1,255,248,15,76,137,252,255,137,214,15,182,209,77,137,
183,233,73,137,159,233,77,137,167,233,73,137,175,233,73,43,175,233,73,3,175,
233,73,137,174,233,77,137,174,233,72,184,237,237,65,84,73,137,228,72,129,
228,239,252,255,208,76,137,228,65,92,77,139,183,233,73,139,159,233,77,139,
167,233,77,139,174,233,73,139,174,233,73,43,175,233,73,3,175,233,129,252,
248,239,255,15,133,244,247,195,248,1,129,252,248,239,15,132,244,247,232,244,
11,248,1,49,192,195,255,248,16,248,17,72,131,252,236,8,248,1,72,199,4,36,
0,0,0,0,76,137,252,255,72,137,230,73,137,159,233,77,137,183,233,73,137,159,
233,77,137,167,233,73,137,175,233,73,43,175,233,73,3,175,233,73,137,174,233,
77,137,174,233,72,184,237,237,65,84,73,137,228,72,129,228,239,252,255,208,
76,137,228,65,92,77,139,183,233,73,139,159,233,77,139,167,233,77,139,174,
233,73,139,174,233,255,73,43,175,233,73,3,175,233,133,192,15,137,244,248,
72,139,20,36,252,242,15,16,4,36,72,131,196,8,72,131,252,235,4,73,137,159,
233,195,248,2,232,244,11,252,233,244,1,255,248,18,248,19,72,131,252,236,8,
248,1,72,199,4,36,0,0,0,0,76,137,252,255,72,137,230,73,137,159,233,77,137,
183,233,73,137,159,233,77,137,167,233,73,137,175,233,73,43,175,233,73,3,175,
233,73,137,174,233,77,137,174,233,72,184,237,237,65,84,73,137,228,72,129,
228,239,252,255,208,76,137,228,65,92,77,139,183,233,73,139,159,233,77,139,
167,233,77,139,174,233,73,139,174,233,255,73,43,175,233,73,3,175,233,133,
192,15,137,244,248,72,139,20,36,252,242,15,16,4,36,72,131,196,8,72,131,252,
235,8,73,137,159,233,195,248,2,232,244,11,252,233,244,1,255,248,20,248,21,
255,76,57,227,15,132,244,247,255,76,137,225,72,41,217,72,131,252,249,16,15,
130,244,247,255,252,243,15,111,3,102,15,215,192,252,247,208,15,188,192,60,
10,15,131,244,22,72,1,195,195,248,1,72,141,139,233,72,137,216,76,57,225,73,
15,71,204,248,2,72,57,200,15,132,244,22,252,246,0,128,15,132,244,249,72,131,
192,1,252,233,244,2,248,3,72,137,195,195,255,248,23,72,131,252,236,16,248,
1,72,57,252,235,15,133,244,248,72,131,196,16,49,192,195,248,2,76,137,252,
255,72,137,230,77,137,183,233,73,137,159,233,77,137,167,233,73,137,175,233,
73,43,175,233,73,3,175,233,73,137,174,233,77,137,174,233,72,184,237,237,65,
84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,92,77,139,183,233,
73,139,159,233,77,139,167,233,77,139,174,233,255,73,139,174,233,73,43,175,
233,73,3,175,233,131,252,248,0,15,141,244,249,139,20,36,72,131,196,16,195,
248,3,232,244,11,252,233,244,1,255,248,14,248,24,255,76,57,227,15,132,244,
22,255,76,137,225,72,41,217,72,131,252,249,10,15,130,244,22,255,72,137,223,
72,184,237,237,65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,
92,72,133,192,15,132,244,22,72,137,195,72,131,252,235,1,73,137,159,233,195,
255,248,22,72,131,252,236,8,248,1,72,199,4,36,0,0,0,0,76,137,252,255,72,137,
230,73,137,159,233,77,137,183,233,73,137,159,233,77,137,167,233,73,137,175,
233,73,43,175,233,73,3,175,233,73,137,174,233,77,137,174,233,72,184,237,237,
65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,92,77,139,183,233,
73,139,159,233,77,139,167,233,77,139,174,233,73,139,174,233,73,43,175,233,
255,73,3,175,233,133,192,15,137,244,248,72,139,20,36,252,242,15,16,4,36,72,
131,196,8,72,131,252,235,1,73,137,159,233,195,248,2,232,244,11,252,233,244,
1,255,248,25,72,131,252,236,8,72,137,52,36,248,1,76,137,252,255,77,137,183,
233,73,137,159,233,77,137,167,233,73,137,175,233,73,43,175,233,73,3,175,233,
73,137,174,233,77,137,174,233,73,137,159,233,72,184,237,237,65,84,73,137,
228,72,129,228,239,252,255,208,76,137,228,65,92,77,139,183,233,73,139,159,
233,77,139,167,233,77,139,174,233,73,139,174,233,73,43,175,233,255,73,3,175,
233,131,252,248,0,15,141,244,248,72,131,196,8,195,248,2,232,244,11,72,139,
52,36,72,57,252,235,15,133,244,1,184,237,72,131,196,8,195,255,248,26,81,82,
72,131,252,236,16,72,137,252,247,72,137,214,72,137,226,72,184,237,237,65,
84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,92,72,131,196,16,90,
89,132,192,15,132,244,248,72,139,68,36,224,195,248,2,72,49,192,72,252,247,
208,195,255,76,57,227,15,133,244,249,255,76,137,225,72,41,217,72,129,252,
249,239,15,131,244,249,255,248,2,255,232,244,14,255,232,244,24,255,232,244,
17,255,232,244,19,255,252,233,244,250,255,248,3,255,139,19,255,72,139,19,
255,252,243,15,16,3,255,252,242,15,16,3,255,15,182,19,132,210,15,136,244,
2,255,248,4,255,137,208,209,252,234,131,224,1,252,247,216,49,194,255,72,137,
208,72,209,252,234,72,131,224,1,72,252,247,216,72,49,194,255,72,133,210,15,
149,210,255,73,137,149,233,255,65,137,149,233,255,252,242,65,15,17,133,233,
255,252,243,65,15,17,133,233,255,65,136,149,233,255,65,128,141,233,235,255,
76,137,252,239,255,72,184,237,237,65,84,73,137,228,72,129,228,239,252,255,
208,76,137,228,65,92,255,132,192,15,133,244,251,232,244,12,252,233,244,1,
248,5,255,72,129,195,239,255,232,244,20,255,232,244,21,255,232,244,16,255,
232,244,18,255,252,246,3,128,15,133,244,2,255,249,248,1,255,76,57,227,15,
132,244,252,255,76,137,225,72,41,217,72,131,252,249,2,15,130,244,252,255,
15,182,19,132,210,15,137,244,253,15,182,139,233,132,201,15,136,244,252,193,
225,7,131,226,127,9,202,72,131,195,2,252,233,244,254,248,6,232,244,23,133,
192,15,133,244,254,195,248,7,72,131,195,1,248,8,137,209,193,252,234,3,128,
225,7,255,248,2,129,252,250,239,255,15,131,244,253,255,15,131,244,251,255,
72,184,237,237,72,139,4,208,255,72,139,4,213,237,255,248,3,56,200,255,15,
133,244,252,255,15,133,244,251,255,72,193,232,16,72,141,21,244,250,249,248,
4,72,1,208,195,248,5,232,244,15,133,192,15,132,244,1,72,141,5,244,255,195,
255,248,6,56,204,15,133,244,5,72,129,194,239,255,252,233,244,26,255,248,7,
255,232,244,26,252,233,244,3,255,76,57,227,15,133,244,247,255,76,137,225,
72,41,217,72,129,252,249,239,15,131,244,247,255,232,244,25,129,252,248,239,
15,132,244,249,129,252,248,239,15,132,245,252,233,244,251,255,128,59,235,
255,102,129,59,238,255,102,129,59,238,15,133,244,248,128,187,233,235,248,
2,255,129,59,239,255,129,59,239,15,133,244,249,128,187,233,235,255,15,132,
244,250,248,3,255,232,245,72,133,192,15,132,245,252,255,224,255,252,233,245,
255,248,4,72,129,195,239,248,5,255,248,1,76,137,252,239,255,132,192,15,133,
244,248,232,244,12,252,233,244,1,248,2,255,144,255,248,9,255,73,139,151,233,
72,184,237,237,65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,
92,255,249,249,72,131,252,236,8,255,72,137,252,234,72,41,218,255,72,133,192,
15,133,244,248,232,244,12,252,233,244,1,248,2,255,73,137,197,255,72,57,252,
235,15,132,244,250,248,1,76,57,227,15,133,244,248,232,244,12,252,233,244,
1,248,2,255,72,137,218,76,137,225,72,41,217,77,139,135,233,72,184,237,237,
65,84,73,137,228,72,129,228,239,252,255,208,76,137,228,65,92,72,1,195,255,
76,57,227,15,132,244,249,232,244,27,248,3,255,76,137,227,255,72,57,252,235,
15,133,244,1,248,4,255,77,137,174,233,73,199,134,233,0,0,0,0,77,59,183,233,
15,132,244,28,73,129,198,239,65,199,134,233,237,255,232,244,13,255,73,129,
252,238,239,77,139,174,233,255,77,139,167,233,73,3,174,233,73,59,175,233,
15,130,244,247,76,57,229,15,135,244,247,73,137,252,236,248,1,255,72,57,221,
15,132,245,255,232,245,255,248,9,72,131,196,8,195,255
};
# 12 "upb/pb/compile_decoder_x64.dasc"
/*|.globals UPB_JIT_GLOBAL_ */
enum {
UPB_JIT_GLOBAL_enterjit,
UPB_JIT_GLOBAL_exitjit,
UPB_JIT_GLOBAL_suspend,
UPB_JIT_GLOBAL_pushlendelim,
UPB_JIT_GLOBAL_decodev32_fallback,
UPB_JIT_GLOBAL_parse_unknown,
UPB_JIT_GLOBAL_skipf32_fallback,
UPB_JIT_GLOBAL_decodef32_fallback,
UPB_JIT_GLOBAL_skipf64_fallback,
UPB_JIT_GLOBAL_decodef64_fallback,
UPB_JIT_GLOBAL_skipv32_fallback,
UPB_JIT_GLOBAL_skipv64_fallback,
UPB_JIT_GLOBAL_decode_varint_slow,
UPB_JIT_GLOBAL_decode_unknown_tag_fallback,
UPB_JIT_GLOBAL_decodev64_fallback,
UPB_JIT_GLOBAL_checktag_fallback,
UPB_JIT_GLOBAL_hashlookup,
UPB_JIT_GLOBAL_strret_fallback,
UPB_JIT_GLOBAL_err,
UPB_JIT_GLOBAL__MAX
};
# 13 "upb/pb/compile_decoder_x64.dasc"
/*|.globalnames upb_jit_globalnames */
static const char *const upb_jit_globalnames[] = {
"enterjit",
"exitjit",
"suspend",
"pushlendelim",
"decodev32_fallback",
"parse_unknown",
"skipf32_fallback",
"decodef32_fallback",
"skipf64_fallback",
"decodef64_fallback",
"skipv32_fallback",
"skipv64_fallback",
"decode_varint_slow",
"decode_unknown_tag_fallback",
"decodev64_fallback",
"checktag_fallback",
"hashlookup",
"strret_fallback",
"err",
(const char *)0
};
# 14 "upb/pb/compile_decoder_x64.dasc"
/*| */
/*|// Calling conventions. Note -- this will need to be changed for */
/*|// Windows, which uses a different calling convention! */
/*|.define ARG1_64, rdi */
/*|.define ARG2_8, r6b // DynASM's equivalent to "sil" -- low byte of esi. */
/*|.define ARG2_32, esi */
/*|.define ARG2_64, rsi */
/*|.define ARG3_8, dl */
/*|.define ARG3_32, edx */
/*|.define ARG3_64, rdx */
/*|.define ARG4_64, rcx */
/*|.define ARG5_64, r8 */
/*|.define XMMARG1, xmm0 */
/*| */
/*|// Register allocation / type map. */
/*|// ALL of the code in this file uses these register allocations. */
/*|// When we "call" within this file, we do not use regular calling */
/*|// conventions, but of course when calling to user callbacks we must. */
/*|.define PTR, rbx // DECODER->ptr (unsynced) */
/*|.define DATAEND, r12 // DECODER->data_end (unsynced) */
/*|.define CLOSURE, r13 // FRAME->closure (unsynced) */
/*|.type FRAME, upb_pbdecoder_frame, r14 // DECODER->top (unsynced) */
#define Dt1(_V) (int)(ptrdiff_t)&(((upb_pbdecoder_frame *)0)_V)
# 36 "upb/pb/compile_decoder_x64.dasc"
/*|.type DECODER, upb_pbdecoder, r15 // DECODER (immutable) */
#define Dt2(_V) (int)(ptrdiff_t)&(((upb_pbdecoder *)0)_V)
# 37 "upb/pb/compile_decoder_x64.dasc"
/*|.define DELIMEND, rbp */
/*| */
/*| // Spills unsynced registers back to memory. */
/*|.macro commit_regs */
/*| mov DECODER->top, FRAME */
/*| mov DECODER->ptr, PTR */
/*| mov DECODER->data_end, DATAEND */
/*| // We don't guarantee that delim_end is NULL when out of range like the */
/*| // interpreter does. */
/*| mov DECODER->delim_end, DELIMEND */
/*| sub DELIMEND, DECODER->buf */
/*| add DELIMEND, DECODER->bufstart_ofs */
/*| mov FRAME->end_ofs, DELIMEND */
/*| mov FRAME->sink.closure, CLOSURE */
/*|.endmacro */
/*| */
/*| // Loads unsynced registers from memory back into registers. */
/*|.macro load_regs */
/*| mov FRAME, DECODER->top */
/*| mov PTR, DECODER->ptr */
/*| mov DATAEND, DECODER->data_end */
/*| mov CLOSURE, FRAME->sink.closure */
/*| mov DELIMEND, FRAME->end_ofs */
/*| sub DELIMEND, DECODER->bufstart_ofs */
/*| add DELIMEND, DECODER->buf */
/*|.endmacro */
/*| */
/*| // Calls an external C function at address "addr". */
/*|.macro callp, addr */
/*| mov64 rax, (uintptr_t)addr */
/*| */
/*| // Stack must be 16-byte aligned (x86-64 ABI requires this). */
/*| // */
/*| // OPT: possibly remove this by statically ensuring correct alignment. */
/*| // */
/*| // OPT: use "call rel32" where possible. */
/*| push r12 */
/*| mov r12, rsp */
/*| and rsp, 0xfffffffffffffff0UL // Align stack. */
/*| call rax */
/*| mov rsp, r12 */
/*| pop r12 */
/*|.endmacro */
/*| */
/*|.macro ld64, val */
/*|| { */
/*|| uintptr_t v = (uintptr_t)val; */
/*|| if (v > 0xffffffff) { */
/*| mov64 ARG2_64, v */
/*|| } else if (v) { */
/*| mov ARG2_32, v */
/*|| } else { */
/*| xor ARG2_32, ARG2_32 */
/*|| } */
/*|| } */
/*|.endmacro */
/*| */
/*|.macro load_handler_data, h, arg */
/*| ld64 upb_handlers_gethandlerdata(h, arg) */
/*|.endmacro */
/*| */
/*|.macro chkeob, bytes, target */
/*|| if (bytes == 1) { */
/*| cmp PTR, DATAEND */
/*| je target */
/*|| } else { */
/*| mov rcx, DATAEND */
/*| sub rcx, PTR */
/*| cmp rcx, bytes */
/*| jb target */
/*|| } */
/*|.endmacro */
/*| */
/*|.macro chkneob, bytes, target */
/*|| if (bytes == 1) { */
/*| cmp PTR, DATAEND */
/*| jne target */
/*|| } else { */
/*| mov rcx, DATAEND */
/*| sub rcx, PTR */
/*| cmp rcx, bytes */
/*| jae target */
/*|| } */
/*|.endmacro */
/*|.macro sethas, reg, hasbit */
/*|| if (hasbit >= 0) { */
/*| or byte [reg + ((uint32_t)hasbit / 8)], (1 << ((uint32_t)hasbit % 8)) */
/*|| } */
/*|.endmacro */
/*| */
/*| // Decodes 32-bit varint into rdx, inlining 1 byte. */
/*|.macro dv32 */
/*| chkeob 1, >7 */
/*| movzx edx, byte [PTR] */
/*| test dl, dl */
/*| jns >8 */
/*|7: */
/*| call ->decodev32_fallback */
/*|8: */
/*| add PTR, 1 */
/*|.endmacro */
#define DECODE_EOF -3
static upb_func *gethandler(const upb_handlers *h, upb_selector_t sel) {
return h ? upb_handlers_gethandler(h, sel) : NULL;
}
/* Defines an "assembly label" for the current code generation offset.
* This label exists *purely* for debugging purposes: it is emitted into
* the .so, and printed as part of JIT debugging output when UPB_JIT_LOAD_SO is
* defined.
*
* We would define this in the .c file except that it conditionally defines a
* pclabel. */
static void asmlabel(jitcompiler *jc, const char *fmt, ...) {
#ifndef NDEBUG
int ofs = jc->dynasm->section->ofs;
UPB_ASSERT(ofs != jc->lastlabelofs);
jc->lastlabelofs = ofs;
#endif
#ifndef UPB_JIT_LOAD_SO
UPB_UNUSED(jc);
UPB_UNUSED(fmt);
#else
va_list args;
va_start(args, fmt);
char *str = upb_vasprintf(fmt, args);
va_end(args);
int pclabel = alloc_pclabel(jc);
/* Normally we would prefer to allocate this inline with the codegen,
* ie.
* |=>asmlabel(...)
* But since we do this conditionally, only when UPB_JIT_LOAD_SO is defined,
* we do it here instead. */
/*|=>pclabel: */
dasm_put(Dst, 0, pclabel);
# 176 "upb/pb/compile_decoder_x64.dasc"
upb_inttable_insert(&jc->asmlabels, pclabel, upb_value_ptr(str));
#endif
}
/* Should only be called when the associated handler is known to exist. */
static bool alwaysok(const upb_handlers *h, upb_selector_t sel) {
upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
bool ok = upb_handlers_getattr(h, sel, &attr);
bool ret;
UPB_ASSERT(ok);
ret = upb_handlerattr_alwaysok(&attr);
upb_handlerattr_uninit(&attr);
return ret;
}
/* Emit static assembly routines; code that does not vary based on the message
* schema. Since it's not input-dependent, we only need one single copy of it.
* For the moment we generate a single copy per generated handlers. Eventually
* we should generate this code at compile time and link it into the binary so
* we have one copy total. To do that we'll want to be sure that it is within
* 2GB of our JIT code, so that branches between the two are near (rel32).
*
* We'd put this assembly in a .s file directly, but DynASM's ability to
* calculate structure offsets automatically is too useful to pass up (it's way
* more convenient to write DECODER->sink than [rbx + 0x96], especially since
* the latter would have to be changed whenever the structure is updated). */
static void emit_static_asm(jitcompiler *jc) {
/*| // Trampolines for entering/exiting the JIT. These are a bit tricky to */
/*| // support full resuming; when we suspend we copy the JIT's portion of */
/*| // the call stack into the upb_pbdecoder and restore it when we resume. */
asmlabel(jc, "enterjit");
/*|->enterjit: */
/*|1: */
/*| push rbp */
/*| push r15 */
/*| push r14 */
/*| push r13 */
/*| push r12 */
/*| push rbx */
/*| */
/*| mov rbx, ARG2_64 // Preserve JIT method. */
/*| */
/*| mov DECODER, rdi */
/*| callp upb_pbdecoder_resume // Same args as us; reuse regs. */
/*| test eax, eax */
/*| jns >1 */
/*| mov DECODER->saved_rsp, rsp */
/*| mov rax, rbx */
/*| load_regs */
/*| */
/*| // Test whether we have a saved stack to resume. */
/*| mov ARG3_64, DECODER->call_len */
/*| test ARG3_64, ARG3_64 */
/*| jnz >2 */
/*| */
/*| call rax */
/*| */
/*| mov rax, DECODER->size_param */
/*| mov qword DECODER->call_len, 0 */
/*|1: */
/*| pop rbx */
dasm_put(Dst, 2, (unsigned int)((uintptr_t)upb_pbdecoder_resume), (unsigned int)(((uintptr_t)upb_pbdecoder_resume)>>32), 0xfffffffffffffff0UL, Dt2(->saved_rsp), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf), Dt2(->call_len), Dt2(->size_param), Dt2(->call_len));
# 238 "upb/pb/compile_decoder_x64.dasc"
/*| pop r12 */
/*| pop r13 */
/*| pop r14 */
/*| pop r15 */
/*| pop rbp */
/*| ret */
/*| */
/*|2: */
/*| // Resume decoder. */
/*| mov ARG2_64, DECODER->callstack */
/*| sub rsp, ARG3_64 */
/*| mov ARG1_64, rsp */
/*| callp memcpy // Restore stack. */
/*| ret // Return to resumed function (not ->enterjit caller). */
/*| */
/*| // Other code can call this to suspend the JIT. */
/*| // To the calling code, it will appear that the function returns when */
/*| // the JIT resumes, and more buffer space will be available. */
/*| // Args: eax=the value that decode() should return. */
dasm_put(Dst, 115, Dt2(->callstack), (unsigned int)((uintptr_t)memcpy), (unsigned int)(((uintptr_t)memcpy)>>32), 0xfffffffffffffff0UL);
# 257 "upb/pb/compile_decoder_x64.dasc"
asmlabel(jc, "exitjit");
/*|->exitjit: */
/*| // Save the stack into DECODER->callstack. */
/*| mov ARG1_64, DECODER->callstack */
/*| mov ARG2_64, rsp */
/*| mov ARG3_64, DECODER->saved_rsp */
/*| sub ARG3_64, rsp */
/*| mov DECODER->call_len, ARG3_64 // Preserve len for next resume. */
/*| mov ebx, eax // Preserve return value across memcpy. */
/*| callp memcpy // Copy stack into decoder. */
/*| mov eax, ebx // This will be our return value. */
/*| */
/*| // Must NOT do this before the memcpy(), otherwise memcpy() will */
/*| // clobber the stack we are trying to save! */
/*| mov rsp, DECODER->saved_rsp */
/*| pop rbx */
/*| pop r12 */
/*| pop r13 */
/*| pop r14 */
/*| pop r15 */
/*| pop rbp */
/*| ret */
/*| */
/*| // Like suspend() in the C decoder, except that the function appears */
/*| // (from the caller's perspective) not to return until the decoder is */
/*| // resumed. */
dasm_put(Dst, 161, Dt2(->callstack), Dt2(->saved_rsp), Dt2(->call_len), (unsigned int)((uintptr_t)memcpy), (unsigned int)(((uintptr_t)memcpy)>>32), 0xfffffffffffffff0UL, Dt2(->saved_rsp));
# 283 "upb/pb/compile_decoder_x64.dasc"
asmlabel(jc, "suspend");
/*|->suspend: */
/*| cmp DECODER->ptr, PTR */
/*| je >1 */
/*| mov DECODER->checkpoint, PTR */
/*|1: */
/*| commit_regs */
/*| mov rdi, DECODER */
/*| callp upb_pbdecoder_suspend */
/*| jmp ->exitjit */
/*| */
dasm_put(Dst, 222, Dt2(->ptr), Dt2(->checkpoint), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_suspend), (unsigned int)(((uintptr_t)upb_pbdecoder_suspend)>>32), 0xfffffffffffffff0UL);
# 294 "upb/pb/compile_decoder_x64.dasc"
asmlabel(jc, "pushlendelim");
/*|->pushlendelim: */
/*|1: */
/*| mov FRAME->sink.closure, CLOSURE */
/*| mov DECODER->checkpoint, PTR */
/*| dv32 */
dasm_put(Dst, 300, Dt1(->sink.closure), Dt2(->checkpoint));
if (1 == 1) {
dasm_put(Dst, 313);
} else {
dasm_put(Dst, 321);
}
# 300 "upb/pb/compile_decoder_x64.dasc"
/*| mov rcx, DELIMEND */
/*| sub rcx, PTR */
/*| sub rcx, rdx */
/*| jb >4 // Len is greater than enclosing message. */
/*| mov FRAME->end_ofs, rcx */
/*| cmp FRAME, DECODER->limit */
/*| je >3 // Stack overflow */
/*| add FRAME, sizeof(upb_pbdecoder_frame) */
/*| mov DELIMEND, PTR */
/*| add DELIMEND, rdx */
/*| mov dword FRAME->groupnum, 0 */
/*| test rcx, rcx */
/*| jz >2 */
/*| mov DATAEND, DECODER->end */
/*| cmp PTR, DELIMEND */
/*| ja >2 */
/*| cmp DELIMEND, DATAEND */
/*| ja >2 */
/*| mov DATAEND, DELIMEND // If DELIMEND >= PTR && DELIMEND < DATAEND */
dasm_put(Dst, 337, Dt1(->end_ofs), Dt2(->limit), sizeof(upb_pbdecoder_frame), Dt1(->groupnum), Dt2(->end));
# 319 "upb/pb/compile_decoder_x64.dasc"
/*|2: */
/*| ret */
/*|3: */
/*| // Stack overflow error. */
/*| mov PTR, DECODER->checkpoint // Rollback to before the delim len. */
/*| // Prepare seterr args. */
/*| mov ARG1_64, DECODER */
/*| ld64 kPbDecoderStackOverflow */
dasm_put(Dst, 428, Dt2(->checkpoint));
{
uintptr_t v = (uintptr_t)kPbDecoderStackOverflow;
if (v > 0xffffffff) {
dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32));
} else if (v) {
dasm_put(Dst, 451, v);
} else {
dasm_put(Dst, 454);
}
}
# 327 "upb/pb/compile_decoder_x64.dasc"
/*| callp upb_pbdecoder_seterr */
/*| call ->suspend */
/*| jmp <1 */
/*|4: */
/*| // Overextended len. */
/*| mov PTR, DECODER->checkpoint // Rollback to before the delim len. */
/*| // Prepare seterr args. */
/*| mov ARG1_64, DECODER */
/*| ld64 kPbDecoderSubmessageTooLong */
dasm_put(Dst, 458, (unsigned int)((uintptr_t)upb_pbdecoder_seterr), (unsigned int)(((uintptr_t)upb_pbdecoder_seterr)>>32), 0xfffffffffffffff0UL, Dt2(->checkpoint));
{
uintptr_t v = (uintptr_t)kPbDecoderSubmessageTooLong;
if (v > 0xffffffff) {
dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32));
} else if (v) {
dasm_put(Dst, 451, v);
} else {
dasm_put(Dst, 454);
}
}
# 336 "upb/pb/compile_decoder_x64.dasc"
/*| callp upb_pbdecoder_seterr */
/*| call ->suspend */
/*| jmp <1 */
/*| */
/*| // For getting a value that spans a buffer seam. Falls back to C. */
/*|.macro getvalue_slow, func, bytes */
/*| sub rsp, 8 // Need stack space for func to write value to. */
/*|1: */
/*| mov qword [rsp], 0 // For parsing routines that only parse 32 bits. */
/*| mov ARG1_64, DECODER */
/*| mov ARG2_64, rsp */
/*| mov DECODER->checkpoint, PTR */
/*| commit_regs */
/*| callp func */
/*| load_regs */
/*| test eax, eax */
/*| jns >2 */
/*| // Success; return parsed data (in rdx AND xmm0). */
/*| mov rdx, [rsp] */
/*| movsd xmm0, qword [rsp] */
/*| add rsp, 8 */
/*| sub PTR, bytes // Bias our buffer pointer to rejoin the fast-path. */
/*| mov DECODER->ptr, PTR */
/*| ret */
/*|2: */
/*| call ->exitjit // Return eax from decode function. */
/*| jmp <1 */
/*|.endmacro */
/*| */
dasm_put(Dst, 497, (unsigned int)((uintptr_t)upb_pbdecoder_seterr), (unsigned int)(((uintptr_t)upb_pbdecoder_seterr)>>32), 0xfffffffffffffff0UL);
# 365 "upb/pb/compile_decoder_x64.dasc"
asmlabel(jc, "parse_unknown");
/*| // Args: edx=fieldnum, cl=wire type */
/*|->parse_unknown: */
/*| // OPT: handle directly instead of kicking to C. */
/*| // Check for ENDGROUP. */
/*| mov ARG1_64, DECODER */
/*| mov ARG2_32, edx */
/*| movzx ARG3_32, cl */
/*| commit_regs */
/*| callp upb_pbdecoder_skipunknown */
/*| load_regs */
/*| cmp eax, DECODE_ENDGROUP */
/*| jne >1 */
dasm_put(Dst, 526, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_skipunknown), (unsigned int)(((uintptr_t)upb_pbdecoder_skipunknown)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf), DECODE_ENDGROUP);
# 378 "upb/pb/compile_decoder_x64.dasc"
/*| ret // Return eax=DECODE_ENDGROUP, not zero */
/*|1: */
/*| cmp eax, DECODE_OK */
/*| je >1 */
/*| call ->exitjit // Return eax from decode function. */
/*|1: */
/*| xor eax, eax */
/*| ret */
/*| */
/*| // Fallback functions for parsing single values. These are used when the */
/*| // buffer doesn't contain enough remaining data for the fast path. Each */
/*| // primitive type (v32, v64, f32, f64) has two functions: decode & skip. */
/*| // Decode functions return their value in rsi/esi. */
/*| // */
/*| // These functions leave PTR = value_end - fast_path_bytes, so that we can */
/*| // re-join the fast path which will add fast_path_bytes after the callback */
/*| // completes. We also set DECODER->ptr to this value which is a signal to */
/*| // ->suspend that DECODER->checkpoint is up to date. */
dasm_put(Dst, 623, DECODE_OK);
# 396 "upb/pb/compile_decoder_x64.dasc"
asmlabel(jc, "skip_decode_f32_fallback");
/*|->skipf32_fallback: */
/*|->decodef32_fallback: */
/*| getvalue_slow upb_pbdecoder_decode_f32, 4 */
dasm_put(Dst, 647, Dt2(->checkpoint), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_decode_f32), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_f32)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs));
# 400 "upb/pb/compile_decoder_x64.dasc"
/*| */
dasm_put(Dst, 751, Dt2(->bufstart_ofs), Dt2(->buf), Dt2(->ptr));
# 401 "upb/pb/compile_decoder_x64.dasc"
asmlabel(jc, "skip_decode_f64_fallback");
/*|->skipf64_fallback: */
/*|->decodef64_fallback: */
/*| getvalue_slow upb_pbdecoder_decode_f64, 8 */
dasm_put(Dst, 799, Dt2(->checkpoint), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_decode_f64), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_f64)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs));
# 405 "upb/pb/compile_decoder_x64.dasc"
/*| */
/*| // Called for varint >= 1 byte. */
dasm_put(Dst, 903, Dt2(->bufstart_ofs), Dt2(->buf), Dt2(->ptr));
# 407 "upb/pb/compile_decoder_x64.dasc"
asmlabel(jc, "skip_decode_v32_fallback");
/*|->skipv32_fallback: */
/*|->skipv64_fallback: */
/*| chkeob 16, >1 */
dasm_put(Dst, 951);
if (16 == 1) {
dasm_put(Dst, 956);
} else {
dasm_put(Dst, 964);
}
# 411 "upb/pb/compile_decoder_x64.dasc"
/*| // With at least 16 bytes left, we can do a branch-less SSE version. */
/*| movdqu xmm0, [PTR] */
/*| pmovmskb eax, xmm0 // bits 0-15 are continuation bits, 16-31 are 0. */
/*| not eax */
/*| bsf eax, eax */
/*| cmp al, 10 */
/*| jae ->decode_varint_slow // Error (>10 byte varint). */
/*| add PTR, rax // bsf result is 0-based, so PTR=end-1, as desired. */
/*| ret */
/*| */
/*|1: */
/*| // With fewer than 16 bytes, we have to read byte by byte. */
/*| lea rcx, [PTR + 10] */
/*| mov rax, PTR // Preserve PTR in case of fallback to slow path. */
/*| cmp rcx, DATAEND */
/*| cmova rcx, DATAEND // rcx = MIN(DATAEND, PTR + 10) */
/*|2: */
/*| cmp rax, rcx */
/*| je ->decode_varint_slow */
/*| test byte [rax], 0x80 */
/*| jz >3 */
/*| add rax, 1 */
/*| jmp <2 */
/*|3: */
/*| mov PTR, rax // PTR = varint_end - 1, as desired */
/*| ret */
/*| */
/*| // Returns tag in edx */
dasm_put(Dst, 980, 10);
# 439 "upb/pb/compile_decoder_x64.dasc"
asmlabel(jc, "decode_unknown_tag_fallback");
/*|->decode_unknown_tag_fallback: */
/*| sub rsp, 16 */
/*|1: */
/*| cmp PTR, DELIMEND */
/*| jne >2 */
/*| add rsp, 16 */
/*| xor eax, eax */
/*| ret */
/*|2: */
/*| // OPT: Have a medium-fast path before falling back to _slow. */
/*| mov ARG1_64, DECODER */
/*| mov ARG2_64, rsp */
/*| commit_regs */
/*| callp upb_pbdecoder_decode_varint_slow */
/*| load_regs */
dasm_put(Dst, 1053, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_decode_varint_slow), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_varint_slow)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure));
# 455 "upb/pb/compile_decoder_x64.dasc"
/*| cmp eax, 0 */
/*| jge >3 */
/*| mov edx, [rsp] // Success; return parsed data. */
/*| add rsp, 16 */
/*| ret */
/*|3: */
/*| call ->exitjit // Return eax from decode function. */
/*| jmp <1 */
/*| */
/*| // Called for varint >= 1 byte. */
dasm_put(Dst, 1156, Dt1(->end_ofs), Dt2(->bufstart_ofs), Dt2(->buf));
# 465 "upb/pb/compile_decoder_x64.dasc"
asmlabel(jc, "decode_v32_v64_fallback");
/*|->decodev32_fallback: */
/*|->decodev64_fallback: */
/*| chkeob 10, ->decode_varint_slow */
dasm_put(Dst, 1194);
if (10 == 1) {
dasm_put(Dst, 1199);
} else {
dasm_put(Dst, 1207);
}
# 469 "upb/pb/compile_decoder_x64.dasc"
/*| // OPT: do something faster than just calling the C version. */
/*| mov rdi, PTR */
/*| callp upb_vdecode_fast */
/*| test rax, rax */
/*| je ->decode_varint_slow // Unterminated varint. */
/*| mov PTR, rax */
/*| sub PTR, 1 */
/*| mov DECODER->ptr, PTR */
/*| ret */
/*| */
dasm_put(Dst, 1223, (unsigned int)((uintptr_t)upb_vdecode_fast), (unsigned int)(((uintptr_t)upb_vdecode_fast)>>32), 0xfffffffffffffff0UL, Dt2(->ptr));
# 479 "upb/pb/compile_decoder_x64.dasc"
asmlabel(jc, "decode_varint_slow");
/*|->decode_varint_slow: */
/*| // Slow path: end of buffer or error (varint length >= 10). */
/*| getvalue_slow upb_pbdecoder_decode_varint_slow, 1 */
dasm_put(Dst, 1268, Dt2(->checkpoint), Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), (unsigned int)((uintptr_t)upb_pbdecoder_decode_varint_slow), (unsigned int)(((uintptr_t)upb_pbdecoder_decode_varint_slow)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs));
# 483 "upb/pb/compile_decoder_x64.dasc"
/*| */
/*| // Args: rsi=expected tag, return=rax (DECODE_{OK,MISMATCH}) */
dasm_put(Dst, 1374, Dt2(->buf), Dt2(->ptr));
# 485 "upb/pb/compile_decoder_x64.dasc"
asmlabel(jc, "checktag_fallback");
/*|->checktag_fallback: */
/*| sub rsp, 8 */
/*| mov [rsp], rsi // Preserve expected tag. */
/*|1: */
/*| mov ARG1_64, DECODER */
/*| commit_regs */
/*| mov DECODER->checkpoint, PTR */
/*| callp upb_pbdecoder_checktag_slow */
/*| load_regs */
dasm_put(Dst, 1418, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt2(->delim_end), Dt2(->buf), Dt2(->bufstart_ofs), Dt1(->end_ofs), Dt1(->sink.closure), Dt2(->checkpoint), (unsigned int)((uintptr_t)upb_pbdecoder_checktag_slow), (unsigned int)(((uintptr_t)upb_pbdecoder_checktag_slow)>>32), 0xfffffffffffffff0UL, Dt2(->top), Dt2(->ptr), Dt2(->data_end), Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->bufstart_ofs));
# 495 "upb/pb/compile_decoder_x64.dasc"
/*| cmp eax, 0 */
/*| jge >2 */
/*| add rsp, 8 */
/*| ret */
/*|2: */
/*| call ->exitjit */
/*| mov rsi, [rsp] */
/*| cmp PTR, DELIMEND */
/*| jne <1 */
/*| mov eax, DECODE_EOF */
/*| add rsp, 8 */
/*| ret */
/*| */
/*| // Args: rsi=upb_inttable, rdx=key, return=rax (-1 if not found). */
/*| // Preserves: rcx, rdx */
/*| // OPT: Could write this in assembly if it's a hotspot. */
dasm_put(Dst, 1517, Dt2(->buf), DECODE_EOF);
# 511 "upb/pb/compile_decoder_x64.dasc"
asmlabel(jc, "hashlookup");
/*|->hashlookup: */
/*| push rcx */
/*| push rdx */
/*| sub rsp, 16 */
/*| mov rdi, rsi */
/*| mov rsi, rdx */
/*| mov rdx, rsp */
/*| callp upb_inttable_lookup */
/*| add rsp, 16 */
/*| pop rdx */
/*| pop rcx */
/*| test al, al */
/*| jz >2 // Unknown field. */
/*| mov rax, [rsp-32] // Value from table. */
/*| ret */
/*|2: */
/*| xor rax, rax */
/*| not rax */
/*| ret */
dasm_put(Dst, 1559, (unsigned int)((uintptr_t)upb_inttable_lookup), (unsigned int)(((uintptr_t)upb_inttable_lookup)>>32), 0xfffffffffffffff0UL);
# 531 "upb/pb/compile_decoder_x64.dasc"
}
static void jitprimitive(jitcompiler *jc, opcode op,
const upb_handlers *h, upb_selector_t sel) {
typedef enum { V32, V64, F32, F64, X } valtype_t;
static valtype_t types[] = {
X, F64, F32, V64, V64, V32, F64, F32, V64, X, X, X, X, V32, V32, F32, F64,
V32, V64 };
static char fastpath_bytes[] = { 1, 1, 4, 8 };
const valtype_t vtype = types[op];
const int fastbytes = fastpath_bytes[vtype];
upb_func *handler = gethandler(h, sel);
upb_fieldtype_t ftype;
size_t offset;
int32_t hasbit;
if (handler) {
/*|1: */
/*| chkneob fastbytes, >3 */
dasm_put(Dst, 112);
if (fastbytes == 1) {
dasm_put(Dst, 1628);
} else {
dasm_put(Dst, 1636, fastbytes);
}
# 550 "upb/pb/compile_decoder_x64.dasc"
/*|2: */
dasm_put(Dst, 1652);
# 551 "upb/pb/compile_decoder_x64.dasc"
switch (vtype) {
case V32:
/*| call ->decodev32_fallback */
dasm_put(Dst, 1655);
# 554 "upb/pb/compile_decoder_x64.dasc"
break;
case V64:
/*| call ->decodev64_fallback */
dasm_put(Dst, 1659);
# 557 "upb/pb/compile_decoder_x64.dasc"
break;
case F32:
/*| call ->decodef32_fallback */
dasm_put(Dst, 1663);
# 560 "upb/pb/compile_decoder_x64.dasc"
break;
case F64:
/*| call ->decodef64_fallback */
dasm_put(Dst, 1667);
# 563 "upb/pb/compile_decoder_x64.dasc"
break;
case X: break;
}
/*| jmp >4 */
dasm_put(Dst, 1671);
# 567 "upb/pb/compile_decoder_x64.dasc"
/* Fast path decode; for when check_bytes bytes are available. */
/*|3: */
dasm_put(Dst, 1676);
# 570 "upb/pb/compile_decoder_x64.dasc"
switch (op) {
case OP_PARSE_SFIXED32:
case OP_PARSE_FIXED32:
/*| mov edx, dword [PTR] */
dasm_put(Dst, 1679);
# 574 "upb/pb/compile_decoder_x64.dasc"
break;
case OP_PARSE_SFIXED64:
case OP_PARSE_FIXED64:
/*| mov rdx, qword [PTR] */
dasm_put(Dst, 1682);
# 578 "upb/pb/compile_decoder_x64.dasc"
break;
case OP_PARSE_FLOAT:
/*| movss xmm0, dword [PTR] */
dasm_put(Dst, 1686);
# 581 "upb/pb/compile_decoder_x64.dasc"
break;
case OP_PARSE_DOUBLE:
/*| movsd xmm0, qword [PTR] */
dasm_put(Dst, 1692);
# 584 "upb/pb/compile_decoder_x64.dasc"
break;
default:
/* Inline one byte of varint decoding. */
/*| movzx edx, byte [PTR] */
/*| test dl, dl */
/*| js <2 // Fallback to slow path for >1 byte varint. */
dasm_put(Dst, 1698);
# 590 "upb/pb/compile_decoder_x64.dasc"
break;
}
/* Second-stage decode; used for both fast and slow paths */
/* (only needed for a few types). */
/*|4: */
dasm_put(Dst, 1708);
# 596 "upb/pb/compile_decoder_x64.dasc"
switch (op) {
case OP_PARSE_SINT32:
/* 32-bit zig-zag decode. */
/*| mov eax, edx */
/*| shr edx, 1 */
/*| and eax, 1 */
/*| neg eax */
/*| xor edx, eax */
dasm_put(Dst, 1711);
# 604 "upb/pb/compile_decoder_x64.dasc"
break;
case OP_PARSE_SINT64:
/* 64-bit zig-zag decode. */
/*| mov rax, rdx */
/*| shr rdx, 1 */
/*| and rax, 1 */
/*| neg rax */
/*| xor rdx, rax */
dasm_put(Dst, 1725);
# 612 "upb/pb/compile_decoder_x64.dasc"
break;
case OP_PARSE_BOOL:
/*| test rdx, rdx */
/*| setne dl */
dasm_put(Dst, 1744);
# 616 "upb/pb/compile_decoder_x64.dasc"
break;
default: break;
}
/* Call callback (or specialize if we can). */
if (upb_msg_getscalarhandlerdata(h, sel, &ftype, &offset, &hasbit)) {
switch (ftype) {
case UPB_TYPE_INT64:
case UPB_TYPE_UINT64:
/*| mov [CLOSURE + offset], rdx */
dasm_put(Dst, 1751, offset);
# 626 "upb/pb/compile_decoder_x64.dasc"
break;
case UPB_TYPE_INT32:
case UPB_TYPE_UINT32:
case UPB_TYPE_ENUM:
/*| mov [CLOSURE + offset], edx */
dasm_put(Dst, 1756, offset);
# 631 "upb/pb/compile_decoder_x64.dasc"
break;
case UPB_TYPE_DOUBLE:
/*| movsd qword [CLOSURE + offset], XMMARG1 */
dasm_put(Dst, 1761, offset);
# 634 "upb/pb/compile_decoder_x64.dasc"
break;
case UPB_TYPE_FLOAT:
/*| movss dword [CLOSURE + offset], XMMARG1 */
dasm_put(Dst, 1769, offset);
# 637 "upb/pb/compile_decoder_x64.dasc"
break;
case UPB_TYPE_BOOL:
/*| mov [CLOSURE + offset], dl */
dasm_put(Dst, 1777, offset);
# 640 "upb/pb/compile_decoder_x64.dasc"
break;
case UPB_TYPE_STRING:
case UPB_TYPE_BYTES:
case UPB_TYPE_MESSAGE:
UPB_ASSERT(false); break;
}
/*| sethas CLOSURE, hasbit */
if (hasbit >= 0) {
dasm_put(Dst, 1782, ((uint32_t)hasbit / 8), (1 << ((uint32_t)hasbit % 8)));
}
# 647 "upb/pb/compile_decoder_x64.dasc"
} else if (handler) {
/*| mov ARG1_64, CLOSURE */
/*| load_handler_data h, sel */
dasm_put(Dst, 1788);
{
uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, sel);
if (v > 0xffffffff) {
dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32));
} else if (v) {
dasm_put(Dst, 451, v);
} else {
dasm_put(Dst, 454);
}
}
# 650 "upb/pb/compile_decoder_x64.dasc"
/*| callp handler */
dasm_put(Dst, 1793, (unsigned int)((uintptr_t)handler), (unsigned int)(((uintptr_t)handler)>>32), 0xfffffffffffffff0UL);
# 651 "upb/pb/compile_decoder_x64.dasc"
if (!alwaysok(h, sel)) {
/*| test al, al */
/*| jnz >5 */
/*| call ->suspend */
/*| jmp <1 */
/*|5: */
dasm_put(Dst, 1815);
# 657 "upb/pb/compile_decoder_x64.dasc"
}
}
/* We do this last so that the checkpoint is not advanced past the user's
* data until the callback has returned success. */
/*| add PTR, fastbytes */
dasm_put(Dst, 1831, fastbytes);
# 663 "upb/pb/compile_decoder_x64.dasc"
} else {
/* No handler registered for this value, just skip it. */
/*| chkneob fastbytes, >3 */
if (fastbytes == 1) {
dasm_put(Dst, 1628);
} else {
dasm_put(Dst, 1636, fastbytes);
}
# 666 "upb/pb/compile_decoder_x64.dasc"
/*|2: */
dasm_put(Dst, 1652);
# 667 "upb/pb/compile_decoder_x64.dasc"
switch (vtype) {
case V32:
/*| call ->skipv32_fallback */
dasm_put(Dst, 1836);
# 670 "upb/pb/compile_decoder_x64.dasc"
break;
case V64:
/*| call ->skipv64_fallback */
dasm_put(Dst, 1840);
# 673 "upb/pb/compile_decoder_x64.dasc"
break;
case F32:
/*| call ->skipf32_fallback */
dasm_put(Dst, 1844);
# 676 "upb/pb/compile_decoder_x64.dasc"
break;
case F64:
/*| call ->skipf64_fallback */
dasm_put(Dst, 1848);
# 679 "upb/pb/compile_decoder_x64.dasc"
break;
case X: break;
}
/* Fast-path skip. */
/*|3: */
dasm_put(Dst, 1676);
# 685 "upb/pb/compile_decoder_x64.dasc"
if (vtype == V32 || vtype == V64) {
/*| test byte [PTR], 0x80 */
/*| jnz <2 */
dasm_put(Dst, 1852);
# 688 "upb/pb/compile_decoder_x64.dasc"
}
/*| add PTR, fastbytes */
dasm_put(Dst, 1831, fastbytes);
# 690 "upb/pb/compile_decoder_x64.dasc"
}
}
static void jitdispatch(jitcompiler *jc,
const upb_pbdecodermethod *method) {
/* Lots of room for tweaking/optimization here. */
const upb_inttable *dispatch = &method->dispatch;
bool has_hash_entries = (dispatch->t.count > 0);
/* Whether any of the fields for this message can have two wire types which
* are both valid (packed & non-packed).
*
* OPT: populate this more precisely; not all messages with hash entries have
* this characteristic. */
bool has_multi_wiretype = has_hash_entries;
/*|=>define_jmptarget(jc, &method->dispatch): */
/*|1: */
dasm_put(Dst, 1861, define_jmptarget(jc, &method->dispatch));
# 709 "upb/pb/compile_decoder_x64.dasc"
/* Decode the field tag. */
/*| mov aword DECODER->checkpoint, PTR */
/*| chkeob 2, >6 */
dasm_put(Dst, 308, Dt2(->checkpoint));
if (2 == 1) {
dasm_put(Dst, 1865);
} else {
dasm_put(Dst, 1873);
}
# 712 "upb/pb/compile_decoder_x64.dasc"
/*| movzx edx, byte [PTR] */
/*| test dl, dl */
/*| jns >7 // Jump if first byte has no continuation bit. */
/*| movzx ecx, byte [PTR + 1] */
/*| test cl, cl */
/*| js >6 // Jump if second byte has continuation bit. */
/*| // Confirmed two-byte varint. */
/*| shl ecx, 7 */
/*| and edx, 0x7f */
/*| or edx, ecx */
/*| add PTR, 2 */
/*| jmp >8 */
/*|6: */
/*| call ->decode_unknown_tag_fallback */
/*| test eax, eax // Hit DELIMEND? */
/*| jnz >8 */
/*| ret */
/*|7: */
/*| add PTR, 1 */
/*|8: */
/*| mov ecx, edx */
/*| shr edx, 3 */
/*| and cl, 7 */
dasm_put(Dst, 1889, 1);
# 735 "upb/pb/compile_decoder_x64.dasc"
/* See comment attached to upb_pbdecodermethod.dispatch for layout of the
* dispatch table. */
/*|2: */
/*| cmp edx, dispatch->array_size */
dasm_put(Dst, 1954, dispatch->array_size);
# 740 "upb/pb/compile_decoder_x64.dasc"
if (has_hash_entries) {
/*| jae >7 */
dasm_put(Dst, 1961);
# 742 "upb/pb/compile_decoder_x64.dasc"
} else {
/*| jae >5 */
dasm_put(Dst, 1966);
# 744 "upb/pb/compile_decoder_x64.dasc"
}
/*| // OPT: Compact the lookup arr into 32-bit entries. */
if ((uintptr_t)dispatch->array > 0x7fffffff) {
/*| mov64 rax, (uintptr_t)dispatch->array */
/*| mov rax, qword [rax + rdx * 8] */
dasm_put(Dst, 1971, (unsigned int)((uintptr_t)dispatch->array), (unsigned int)(((uintptr_t)dispatch->array)>>32));
# 749 "upb/pb/compile_decoder_x64.dasc"
} else {
/*| mov rax, qword [rdx * 8 + dispatch->array] */
dasm_put(Dst, 1980, dispatch->array);
# 751 "upb/pb/compile_decoder_x64.dasc"
}
/*|3: */
/*| // We take advantage of the fact that non-present entries are stored */
/*| // as -1, which will result in wire types that will never match. */
/*| cmp al, cl */
dasm_put(Dst, 1986);
# 756 "upb/pb/compile_decoder_x64.dasc"
if (has_multi_wiretype) {
/*| jne >6 */
dasm_put(Dst, 1991);
# 758 "upb/pb/compile_decoder_x64.dasc"
} else {
/*| jne >5 */
dasm_put(Dst, 1996);
# 760 "upb/pb/compile_decoder_x64.dasc"
}
/*| shr rax, 16 */
/*| */
/*| // Load the machine code address from the table entry. */
/*| // The table entry is relative to the dispatch->array jmptarget */
/*| // (patchdispatch() took care of this) which is the same as */
/*| // local label "4". The "lea" is really just trying to do */
/*| // lea rax, [>4 + rax] */
/*| // */
/*| // But we can't write that directly for some reason, so we use */
/*| // rdx as a temporary. */
/*| lea rdx, [>4] */
/*|=>define_jmptarget(jc, dispatch->array): */
/*|4: */
/*| add rax, rdx */
/*| ret */
/*| */
/*|5: */
/*| // Field isn't in our table. */
/*| call ->parse_unknown */
/*| test eax, eax // ENDGROUP? */
/*| jz <1 */
/*| lea rax, [>9] // ENDGROUP; Load address of OP_ENDMSG. */
/*| ret */
dasm_put(Dst, 2001, define_jmptarget(jc, dispatch->array));
# 784 "upb/pb/compile_decoder_x64.dasc"
if (has_multi_wiretype) {
/*|6: */
/*| // Primary wire type didn't match, check secondary wire type. */
/*| cmp ah, cl */
/*| jne <5 */
/*| // Secondary wire type is a match, look up fn + UPB_MAX_FIELDNUMBER. */
/*| add rdx, UPB_MAX_FIELDNUMBER */
/*| // This key will never be in the array part, so do a hash lookup. */
dasm_put(Dst, 2035, UPB_MAX_FIELDNUMBER);
# 793 "upb/pb/compile_decoder_x64.dasc"
UPB_ASSERT(has_hash_entries);
/*| ld64 dispatch */
{
uintptr_t v = (uintptr_t)dispatch;
if (v > 0xffffffff) {
dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32));
} else if (v) {
dasm_put(Dst, 451, v);
} else {
dasm_put(Dst, 454);
}
}
# 795 "upb/pb/compile_decoder_x64.dasc"
/*| jmp ->hashlookup // Tail call. */
dasm_put(Dst, 2048);
# 796 "upb/pb/compile_decoder_x64.dasc"
}
if (has_hash_entries) {
/*|7: */
/*| // Hash table lookup. */
/*| ld64 dispatch */
dasm_put(Dst, 2053);
{
uintptr_t v = (uintptr_t)dispatch;
if (v > 0xffffffff) {
dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32));
} else if (v) {
dasm_put(Dst, 451, v);
} else {
dasm_put(Dst, 454);
}
}
# 802 "upb/pb/compile_decoder_x64.dasc"
/*| call ->hashlookup */
/*| jmp <3 */
dasm_put(Dst, 2056);
# 804 "upb/pb/compile_decoder_x64.dasc"
}
}
static void jittag(jitcompiler *jc, uint64_t tag, int n, int ofs,
const upb_pbdecodermethod *method) {
/* Internally we parse unknown fields; if this runs us into DELIMEND we jump
* to the corresponding DELIMEND target (either msg end or repeated field
* end), which we find from the OP_CHECKDELIM which must have necessarily
* preceded us. */
uint32_t last_instruction = *(jc->pc - 2);
int last_arg = (int32_t)last_instruction >> 8;
uint32_t *delimend = (jc->pc - 1) + last_arg;
const size_t ptr_words = sizeof(void*) / sizeof(uint32_t);
UPB_ASSERT((last_instruction & 0xff) == OP_CHECKDELIM);
if (getop(*(jc->pc - 1)) == OP_TAGN) {
jc->pc += ptr_words;
}
/*| chkneob n, >1 */
if (n == 1) {
dasm_put(Dst, 2064);
} else {
dasm_put(Dst, 2072, n);
}
# 825 "upb/pb/compile_decoder_x64.dasc"
/*| // OPT: this is way too much fallback code to put here. */
/*| // Reduce and/or move to a separate section to make better icache usage. */
/*| ld64 tag */
{
uintptr_t v = (uintptr_t)tag;
if (v > 0xffffffff) {
dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32));
} else if (v) {
dasm_put(Dst, 451, v);
} else {
dasm_put(Dst, 454);
}
}
# 829 "upb/pb/compile_decoder_x64.dasc"
/*| call ->checktag_fallback */
/*| cmp eax, DECODE_MISMATCH */
/*| je >3 */
/*| cmp eax, DECODE_EOF */
/*| je =>jmptarget(jc, delimend) */
/*| jmp >5 */
dasm_put(Dst, 2088, DECODE_MISMATCH, DECODE_EOF, jmptarget(jc, delimend));
# 835 "upb/pb/compile_decoder_x64.dasc"
/*|1: */
dasm_put(Dst, 112);
# 837 "upb/pb/compile_decoder_x64.dasc"
switch (n) {
case 1:
/*| cmp byte [PTR], tag */
dasm_put(Dst, 2111, tag);
# 840 "upb/pb/compile_decoder_x64.dasc"
break;
case 2:
/*| cmp word [PTR], tag */
dasm_put(Dst, 2115, tag);
# 843 "upb/pb/compile_decoder_x64.dasc"
break;
case 3:
/*| // OPT: Slightly more efficient code, but depends on an extra byte. */
/*| // mov eax, dword [PTR] */
/*| // shl eax, 8 */
/*| // cmp eax, tag << 8 */
/*| cmp word [PTR], (tag & 0xffff) */
/*| jne >2 */
/*| cmp byte [PTR + 2], (tag >> 16) */
/*|2: */
dasm_put(Dst, 2120, (tag & 0xffff), 2, (tag >> 16));
# 853 "upb/pb/compile_decoder_x64.dasc"
break;
case 4:
/*| cmp dword [PTR], tag */
dasm_put(Dst, 2135, tag);
# 856 "upb/pb/compile_decoder_x64.dasc"
break;
case 5:
/*| cmp dword [PTR], (tag & 0xffffffff) */
/*| jne >3 */
/*| cmp byte [PTR + 4], (tag >> 32) */
dasm_put(Dst, 2139, (tag & 0xffffffff), 4, (tag >> 32));
# 861 "upb/pb/compile_decoder_x64.dasc"
}
/*| je >4 */
/*|3: */
dasm_put(Dst, 2151);
# 864 "upb/pb/compile_decoder_x64.dasc"
if (ofs == 0) {
/*| call =>jmptarget(jc, &method->dispatch) */
/*| test rax, rax */
/*| jz =>jmptarget(jc, delimend) */
/*| jmp rax */
dasm_put(Dst, 2158, jmptarget(jc, &method->dispatch), jmptarget(jc, delimend));
# 869 "upb/pb/compile_decoder_x64.dasc"
} else {
/*| jmp =>jmptarget(jc, jc->pc + ofs) */
dasm_put(Dst, 2170, jmptarget(jc, jc->pc + ofs));
# 871 "upb/pb/compile_decoder_x64.dasc"
}
/*|4: */
/*| add PTR, n */
/*|5: */
dasm_put(Dst, 2174, n);
# 875 "upb/pb/compile_decoder_x64.dasc"
}
/* Compile the bytecode to x64. */
static void jitbytecode(jitcompiler *jc) {
upb_pbdecodermethod *method = NULL;
const upb_handlers *h = NULL;
for (jc->pc = jc->group->bytecode; jc->pc < jc->group->bytecode_end; ) {
int32_t instr = *jc->pc;
opcode op = instr & 0xff;
uint32_t arg = instr >> 8;
int32_t longofs = arg;
if (op != OP_SETDISPATCH) {
/* Skipped for SETDISPATCH because it defines its own asmlabel for the
* dispatch code it emits. */
asmlabel(jc, "0x%lx.%s", pcofs(jc), upb_pbdecoder_getopname(op));
/* Skipped for SETDISPATCH because it should point at the function
* prologue, not the dispatch function that is emitted first.
* TODO: optimize this to only define pclabels that are actually used. */
/*|=>define_jmptarget(jc, jc->pc): */
dasm_put(Dst, 0, define_jmptarget(jc, jc->pc));
# 896 "upb/pb/compile_decoder_x64.dasc"
}
jc->pc++;
switch (op) {
case OP_STARTMSG: {
upb_func *startmsg = gethandler(h, UPB_STARTMSG_SELECTOR);
if (startmsg) {
/* bool startmsg(void *closure, const void *hd) */
/*|1: */
/*| mov ARG1_64, CLOSURE */
/*| load_handler_data h, UPB_STARTMSG_SELECTOR */
dasm_put(Dst, 2183);
{
uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, UPB_STARTMSG_SELECTOR);
if (v > 0xffffffff) {
dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32));
} else if (v) {
dasm_put(Dst, 451, v);
} else {
dasm_put(Dst, 454);
}
}
# 908 "upb/pb/compile_decoder_x64.dasc"
/*| callp startmsg */
dasm_put(Dst, 1793, (unsigned int)((uintptr_t)startmsg), (unsigned int)(((uintptr_t)startmsg)>>32), 0xfffffffffffffff0UL);
# 909 "upb/pb/compile_decoder_x64.dasc"
if (!alwaysok(h, UPB_STARTMSG_SELECTOR)) {
/*| test al, al */
/*| jnz >2 */
/*| call ->suspend */
/*| jmp <1 */
/*|2: */
dasm_put(Dst, 2190);
# 915 "upb/pb/compile_decoder_x64.dasc"
}
} else {
/*| nop */
dasm_put(Dst, 2206);
# 918 "upb/pb/compile_decoder_x64.dasc"
}
break;
}
case OP_ENDMSG: {
upb_func *endmsg = gethandler(h, UPB_ENDMSG_SELECTOR);
/*|9: */
dasm_put(Dst, 2208);
# 924 "upb/pb/compile_decoder_x64.dasc"
if (endmsg) {
/* bool endmsg(void *closure, const void *hd, upb_status *status) */
/*| mov ARG1_64, CLOSURE */
/*| load_handler_data h, UPB_ENDMSG_SELECTOR */
dasm_put(Dst, 1788);
{
uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, UPB_ENDMSG_SELECTOR);
if (v > 0xffffffff) {
dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32));
} else if (v) {
dasm_put(Dst, 451, v);
} else {
dasm_put(Dst, 454);
}
}
# 928 "upb/pb/compile_decoder_x64.dasc"
/*| mov ARG3_64, DECODER->status */
/*| callp endmsg */
dasm_put(Dst, 2211, Dt2(->status), (unsigned int)((uintptr_t)endmsg), (unsigned int)(((uintptr_t)endmsg)>>32), 0xfffffffffffffff0UL);
# 930 "upb/pb/compile_decoder_x64.dasc"
}
break;
}
case OP_SETDISPATCH: {
uint32_t *op_pc = jc->pc - 1;
const char *msgname;
upb_inttable *dispatch;
/* Load info for new method. */
memcpy(&dispatch, jc->pc, sizeof(void*));
jc->pc += sizeof(void*) / sizeof(uint32_t);
/* The OP_SETDISPATCH bytecode contains a pointer that is
* &method->dispatch; we want to go backwards and recover method. */
method =
(void*)((char*)dispatch - offsetof(upb_pbdecodermethod, dispatch));
/* May be NULL, in which case no handlers for this message will be found.
* OPT: we should do better by completely skipping the message in this
* case instead of parsing it field by field. We should also do the skip
* in the containing message's code. */
h = method->dest_handlers_;
msgname = upb_msgdef_fullname(upb_handlers_msgdef(h));
/* Emit dispatch code for new method. */
asmlabel(jc, "0x%lx.dispatch.%s", pcofs(jc), msgname);
jitdispatch(jc, method);
/* Emit function prologue for new method. */
asmlabel(jc, "0x%lx.parse.%s", pcofs(jc), msgname);
/*|=>define_jmptarget(jc, op_pc): */
/*|=>define_jmptarget(jc, method): */
/*| sub rsp, 8 */
dasm_put(Dst, 2237, define_jmptarget(jc, op_pc), define_jmptarget(jc, method));
# 961 "upb/pb/compile_decoder_x64.dasc"
break;
}
case OP_PARSE_DOUBLE:
case OP_PARSE_FLOAT:
case OP_PARSE_INT64:
case OP_PARSE_UINT64:
case OP_PARSE_INT32:
case OP_PARSE_FIXED64:
case OP_PARSE_FIXED32:
case OP_PARSE_BOOL:
case OP_PARSE_UINT32:
case OP_PARSE_SFIXED32:
case OP_PARSE_SFIXED64:
case OP_PARSE_SINT32:
case OP_PARSE_SINT64:
jitprimitive(jc, op, h, arg);
break;
case OP_STARTSEQ:
case OP_STARTSUBMSG:
case OP_STARTSTR: {
upb_func *start = gethandler(h, arg);
if (start) {
/* void *startseq(void *closure, const void *hd)
* void *startsubmsg(void *closure, const void *hd)
* void *startstr(void *closure, const void *hd, size_t size_hint) */
/*|1: */
/*| mov ARG1_64, CLOSURE */
/*| load_handler_data h, arg */
dasm_put(Dst, 2183);
{
uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, arg);
if (v > 0xffffffff) {
dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32));
} else if (v) {
dasm_put(Dst, 451, v);
} else {
dasm_put(Dst, 454);
}
}
# 990 "upb/pb/compile_decoder_x64.dasc"
if (op == OP_STARTSTR) {
/*| mov ARG3_64, DELIMEND */
/*| sub ARG3_64, PTR */
dasm_put(Dst, 2245);
# 993 "upb/pb/compile_decoder_x64.dasc"
}
/*| callp start */
dasm_put(Dst, 1793, (unsigned int)((uintptr_t)start), (unsigned int)(((uintptr_t)start)>>32), 0xfffffffffffffff0UL);
# 995 "upb/pb/compile_decoder_x64.dasc"
if (!alwaysok(h, arg)) {
/*| test rax, rax */
/*| jnz >2 */
/*| call ->suspend */
/*| jmp <1 */
/*|2: */
dasm_put(Dst, 2253);
# 1001 "upb/pb/compile_decoder_x64.dasc"
}
/*| mov CLOSURE, rax */
dasm_put(Dst, 2270);
# 1003 "upb/pb/compile_decoder_x64.dasc"
} else {
/* TODO: nop is only required because of asmlabel(). */
/*| nop */
dasm_put(Dst, 2206);
# 1006 "upb/pb/compile_decoder_x64.dasc"
}
break;
}
case OP_ENDSEQ:
case OP_ENDSUBMSG:
case OP_ENDSTR: {
upb_func *end = gethandler(h, arg);
if (end) {
/* bool endseq(void *closure, const void *hd)
* bool endsubmsg(void *closure, const void *hd)
* bool endstr(void *closure, const void *hd) */
/*|1: */
/*| mov ARG1_64, CLOSURE */
/*| load_handler_data h, arg */
dasm_put(Dst, 2183);
{
uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, arg);
if (v > 0xffffffff) {
dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32));
} else if (v) {
dasm_put(Dst, 451, v);
} else {
dasm_put(Dst, 454);
}
}
# 1020 "upb/pb/compile_decoder_x64.dasc"
/*| callp end */
dasm_put(Dst, 1793, (unsigned int)((uintptr_t)end), (unsigned int)(((uintptr_t)end)>>32), 0xfffffffffffffff0UL);
# 1021 "upb/pb/compile_decoder_x64.dasc"
if (!alwaysok(h, arg)) {
/*| test al, al */
/*| jnz >2 */
/*| call ->suspend */
/*| jmp <1 */
/*|2: */
dasm_put(Dst, 2190);
# 1027 "upb/pb/compile_decoder_x64.dasc"
}
} else {
/* TODO: nop is only required because of asmlabel(). */
/*| nop */
dasm_put(Dst, 2206);
# 1031 "upb/pb/compile_decoder_x64.dasc"
}
break;
}
case OP_STRING: {
upb_func *str = gethandler(h, arg);
/*| cmp PTR, DELIMEND */
/*| je >4 */
/*|1: */
/*| cmp PTR, DATAEND */
/*| jne >2 */
/*| call ->suspend */
/*| jmp <1 */
/*|2: */
dasm_put(Dst, 2274);
# 1044 "upb/pb/compile_decoder_x64.dasc"
if (str) {
/* size_t str(void *closure, const void *hd, const char *str,
* size_t n) */
/*| mov ARG1_64, CLOSURE */
/*| load_handler_data h, arg */
dasm_put(Dst, 1788);
{
uintptr_t v = (uintptr_t)upb_handlers_gethandlerdata(h, arg);
if (v > 0xffffffff) {
dasm_put(Dst, 446, (unsigned int)(v), (unsigned int)((v)>>32));
} else if (v) {
dasm_put(Dst, 451, v);
} else {
dasm_put(Dst, 454);
}
}
# 1049 "upb/pb/compile_decoder_x64.dasc"
/*| mov ARG3_64, PTR */
/*| mov ARG4_64, DATAEND */
/*| sub ARG4_64, PTR */
/*| mov ARG5_64, qword DECODER->handle */
/*| callp str */
/*| add PTR, rax */
dasm_put(Dst, 2301, Dt2(->handle), (unsigned int)((uintptr_t)str), (unsigned int)(((uintptr_t)str)>>32), 0xfffffffffffffff0UL);
# 1055 "upb/pb/compile_decoder_x64.dasc"
if (!alwaysok(h, arg)) {
/*| cmp PTR, DATAEND */
/*| je >3 */
/*| call ->strret_fallback */
/*|3: */
dasm_put(Dst, 2339);
# 1060 "upb/pb/compile_decoder_x64.dasc"
}
} else {
/*| mov PTR, DATAEND */
dasm_put(Dst, 2352);
# 1063 "upb/pb/compile_decoder_x64.dasc"
}
/*| cmp PTR, DELIMEND */
/*| jne <1 */
/*|4: */
dasm_put(Dst, 2356);
# 1067 "upb/pb/compile_decoder_x64.dasc"
break;
}
case OP_PUSHTAGDELIM:
/*| mov FRAME->sink.closure, CLOSURE */
/*| // This shouldn't need to be read, because tag-delimited fields */
/*| // shouldn't have an OP_SETDELIM after them. But for the moment */
/*| // non-packed repeated fields do OP_SETDELIM so they can share more */
/*| // code with the packed code-path. If this is changed later, this */
/*| // store can be removed. */
/*| mov qword FRAME->end_ofs, 0 */
/*| cmp FRAME, DECODER->limit */
/*| je ->err */
/*| add FRAME, sizeof(upb_pbdecoder_frame) */
/*| mov dword FRAME->groupnum, arg */
dasm_put(Dst, 2367, Dt1(->sink.closure), Dt1(->end_ofs), Dt2(->limit), sizeof(upb_pbdecoder_frame), Dt1(->groupnum), arg);
# 1081 "upb/pb/compile_decoder_x64.dasc"
break;
case OP_PUSHLENDELIM:
/*| call ->pushlendelim */
dasm_put(Dst, 2397);
# 1084 "upb/pb/compile_decoder_x64.dasc"
break;
case OP_POP:
/*| sub FRAME, sizeof(upb_pbdecoder_frame) */
/*| mov CLOSURE, FRAME->sink.closure */
dasm_put(Dst, 2401, sizeof(upb_pbdecoder_frame), Dt1(->sink.closure));
# 1088 "upb/pb/compile_decoder_x64.dasc"
break;
case OP_SETDELIM:
/* OPT: experiment with testing vs old offset to optimize away. */
/*| mov DATAEND, DECODER->end */
/*| add DELIMEND, FRAME->end_ofs */
/*| cmp DELIMEND, DECODER->buf */
/*| jb >1 */
/*| cmp DELIMEND, DATAEND */
/*| ja >1 // OPT: try cmov. */
/*| mov DATAEND, DELIMEND */
/*|1: */
dasm_put(Dst, 2411, Dt2(->end), Dt1(->end_ofs), Dt2(->buf));
# 1099 "upb/pb/compile_decoder_x64.dasc"
break;
case OP_SETBIGGROUPNUM:
/*| mov dword FRAME->groupnum, *jc->pc++ */
dasm_put(Dst, 2391, Dt1(->groupnum), *jc->pc++);
# 1102 "upb/pb/compile_decoder_x64.dasc"
break;
case OP_CHECKDELIM:
/*| cmp DELIMEND, PTR */
/*| je =>jmptarget(jc, jc->pc + longofs) */
dasm_put(Dst, 2441, jmptarget(jc, jc->pc + longofs));
# 1106 "upb/pb/compile_decoder_x64.dasc"
break;
case OP_CALL:
/*| call =>jmptarget(jc, jc->pc + longofs) */
dasm_put(Dst, 2448, jmptarget(jc, jc->pc + longofs));
# 1109 "upb/pb/compile_decoder_x64.dasc"
break;
case OP_BRANCH:
/*| jmp =>jmptarget(jc, jc->pc + longofs); */
dasm_put(Dst, 2170, jmptarget(jc, jc->pc + longofs));
# 1112 "upb/pb/compile_decoder_x64.dasc"
break;
case OP_RET:
/*|9: */
/*| add rsp, 8 */
/*| ret */
dasm_put(Dst, 2451);
# 1117 "upb/pb/compile_decoder_x64.dasc"
break;
case OP_TAG1:
jittag(jc, (arg >> 8) & 0xff, 1, (int8_t)arg, method);
break;
case OP_TAG2:
jittag(jc, (arg >> 8) & 0xffff, 2, (int8_t)arg, method);
break;
case OP_TAGN: {
uint64_t tag;
memcpy(&tag, jc->pc, 8);
jittag(jc, tag, arg >> 8, (int8_t)arg, method);
break;
}
case OP_DISPATCH:
/*| call =>jmptarget(jc, &method->dispatch) */
dasm_put(Dst, 2448, jmptarget(jc, &method->dispatch));
# 1132 "upb/pb/compile_decoder_x64.dasc"
break;
case OP_HALT:
UPB_ASSERT(false);
}
}
asmlabel(jc, "eof");
/*| nop */
dasm_put(Dst, 2206);
# 1140 "upb/pb/compile_decoder_x64.dasc"
}