Josh Haberman
10 years ago
parent
43f2455cbd
commit
8405e5e389
4 changed files with 335 additions and 7 deletions
@ -0,0 +1,2 @@ |
|||||||
|
This is PROTOTYPE code -- all interfaces are experimental |
||||||
|
and will almost certainly change. |
||||||
@ -0,0 +1,7 @@ |
|||||||
|
#!/usr/bin/ruby |
||||||
|
|
||||||
|
require 'mkmf' |
||||||
|
find_header("upb/upb.h", "../../..") or raise "Can't find upb headers" |
||||||
|
find_library("upb_pic", "upb_msgdef_new", "../..") or raise "Can't find upb lib" |
||||||
|
$CFLAGS += " -Wall" |
||||||
|
create_makefile("upb") |
||||||
@ -0,0 +1,300 @@ |
|||||||
|
|
||||||
|
#include "ruby.h" |
||||||
|
#include "upb/def.h" |
||||||
|
#include "upb/handlers.h" |
||||||
|
#include "upb/pb/decoder.h" |
||||||
|
#include "upb/pb/glue.h" |
||||||
|
#include "upb/shim/shim.h" |
||||||
|
#include "upb/symtab.h" |
||||||
|
|
||||||
|
static VALUE cMessageDef; |
||||||
|
static VALUE cMessage; |
||||||
|
|
||||||
|
// Wrapper around a upb_msgdef.
|
||||||
|
typedef struct { |
||||||
|
// The msgdef for this message, and a DecoderMethod to parse protobufs and
|
||||||
|
// fill a message.
|
||||||
|
//
|
||||||
|
// We own refs on both of these.
|
||||||
|
const upb_msgdef *md; |
||||||
|
const upb_pbdecodermethod *fill_method; |
||||||
|
|
||||||
|
size_t size; |
||||||
|
uint32_t *field_offsets; |
||||||
|
} rb_msgdef; |
||||||
|
|
||||||
|
// Ruby message object.
|
||||||
|
// This will be sized according to what fields are actually present.
|
||||||
|
typedef struct { |
||||||
|
union u { |
||||||
|
VALUE rbmsgdef; |
||||||
|
char data[1]; |
||||||
|
} data; |
||||||
|
} rb_msg; |
||||||
|
|
||||||
|
#define DEREF(msg, ofs, type) *(type*)(&msg->data.data[ofs]) |
||||||
|
|
||||||
|
static void symtab_free(void *md) { |
||||||
|
upb_symtab_unref(md, UPB_UNTRACKED_REF); |
||||||
|
} |
||||||
|
|
||||||
|
void rupb_checkstatus(upb_status *s) { |
||||||
|
if (!upb_ok(s)) { |
||||||
|
fprintf(stderr, "YO, error! %s", upb_status_errmsg(s)); |
||||||
|
rb_raise(rb_eRuntimeError, "%s", upb_status_errmsg(s)); |
||||||
|
} else { |
||||||
|
fprintf(stderr, "A-OK!"); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/* handlers *******************************************************************/ |
||||||
|
|
||||||
|
// These are handlers for populating a Ruby protobuf message when parsing.
|
||||||
|
|
||||||
|
static size_t strhandler(void *closure, const void *hd, const char *str, |
||||||
|
size_t len, const upb_bufhandle *handle) { |
||||||
|
rb_msg *msg = closure; |
||||||
|
const size_t *ofs = hd; |
||||||
|
DEREF(msg, *ofs, VALUE) = rb_str_new(str, len); |
||||||
|
return len; |
||||||
|
} |
||||||
|
|
||||||
|
static const void *newhandlerdata(upb_handlers *h, uint32_t ofs) { |
||||||
|
size_t *hd_ofs = ALLOC(size_t); |
||||||
|
*hd_ofs = ofs; |
||||||
|
upb_handlers_addcleanup(h, hd_ofs, free); |
||||||
|
return hd_ofs; |
||||||
|
} |
||||||
|
|
||||||
|
static void add_handlers_for_message(const void *closure, upb_handlers *h) { |
||||||
|
// XXX: Doesn't support submessages properly yet.
|
||||||
|
const rb_msgdef *rmd = closure; |
||||||
|
upb_msg_iter i; |
||||||
|
for (upb_msg_begin(&i, rmd->md); !upb_msg_done(&i); upb_msg_next(&i)) { |
||||||
|
upb_fielddef *f = upb_msg_iter_field(&i); |
||||||
|
|
||||||
|
if (upb_fielddef_isseq(f)) { |
||||||
|
rb_raise(rb_eRuntimeError, "Doesn't support repeated fields yet."); |
||||||
|
} |
||||||
|
|
||||||
|
size_t ofs = rmd->field_offsets[upb_fielddef_index(f)]; |
||||||
|
|
||||||
|
switch (upb_fielddef_type(f)) { |
||||||
|
case UPB_TYPE_BOOL: |
||||||
|
case UPB_TYPE_INT32: |
||||||
|
case UPB_TYPE_UINT32: |
||||||
|
case UPB_TYPE_ENUM: |
||||||
|
case UPB_TYPE_FLOAT: |
||||||
|
case UPB_TYPE_INT64: |
||||||
|
case UPB_TYPE_UINT64: |
||||||
|
case UPB_TYPE_DOUBLE: |
||||||
|
upb_shim_set(h, f, ofs, -1); |
||||||
|
break; |
||||||
|
case UPB_TYPE_STRING: |
||||||
|
case UPB_TYPE_BYTES: { |
||||||
|
upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER; |
||||||
|
upb_handlerattr_sethandlerdata(&attr, newhandlerdata(h, ofs)); |
||||||
|
// XXX: does't currently handle split buffers.
|
||||||
|
upb_handlers_setstring(h, f, strhandler, &attr); |
||||||
|
upb_handlerattr_uninit(&attr); |
||||||
|
break; |
||||||
|
} |
||||||
|
case UPB_TYPE_MESSAGE: |
||||||
|
rb_raise(rb_eRuntimeError, "Doesn't support submessages yet."); |
||||||
|
break; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// Creates upb handlers for populating a message.
|
||||||
|
static const upb_handlers *new_fill_handlers(const rb_msgdef *rmd, |
||||||
|
const void *owner) { |
||||||
|
return upb_handlers_newfrozen(rmd->md, owner, add_handlers_for_message, rmd); |
||||||
|
} |
||||||
|
|
||||||
|
// General alignment rules are that each type needs to be stored at an address
|
||||||
|
// that is a multiple of its size.
|
||||||
|
static size_t align_up(size_t val, size_t align) { |
||||||
|
return val % align == 0 ? val : val + align - (val % align); |
||||||
|
} |
||||||
|
|
||||||
|
// Byte size to store each upb type.
|
||||||
|
static size_t rupb_sizeof(upb_fieldtype_t type) { |
||||||
|
switch (type) { |
||||||
|
case UPB_TYPE_BOOL: |
||||||
|
return 1; |
||||||
|
case UPB_TYPE_INT32: |
||||||
|
case UPB_TYPE_UINT32: |
||||||
|
case UPB_TYPE_ENUM: |
||||||
|
case UPB_TYPE_FLOAT: |
||||||
|
return 4; |
||||||
|
case UPB_TYPE_INT64: |
||||||
|
case UPB_TYPE_UINT64: |
||||||
|
case UPB_TYPE_DOUBLE: |
||||||
|
return 8; |
||||||
|
case UPB_TYPE_STRING: |
||||||
|
case UPB_TYPE_BYTES: |
||||||
|
case UPB_TYPE_MESSAGE: |
||||||
|
return sizeof(VALUE); |
||||||
|
} |
||||||
|
assert(false); |
||||||
|
} |
||||||
|
|
||||||
|
/* msg ************************************************************************/ |
||||||
|
|
||||||
|
static void msg_free(void *msg) { |
||||||
|
free(msg); |
||||||
|
} |
||||||
|
|
||||||
|
// Invoked by the Ruby GC whenever it is doing a mark-and-sweep.
|
||||||
|
static void msg_mark(void *p) { |
||||||
|
rb_msg *msg = p; |
||||||
|
rb_msgdef *rmd; |
||||||
|
Data_Get_Struct(msg->data.rbmsgdef, rb_msgdef, rmd); |
||||||
|
|
||||||
|
// We need to mark all references to other Ruby values: strings, arrays, and
|
||||||
|
// submessages that we point to. Only strings are implemented so far.
|
||||||
|
upb_msg_iter i; |
||||||
|
for (upb_msg_begin(&i, rmd->md); !upb_msg_done(&i); upb_msg_next(&i)) { |
||||||
|
upb_fielddef *f = upb_msg_iter_field(&i); |
||||||
|
if (upb_fielddef_isstring(f)) { |
||||||
|
size_t ofs = rmd->field_offsets[upb_fielddef_index(f)]; |
||||||
|
rb_gc_mark(DEREF(msg, ofs, VALUE)); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
VALUE msg_new(VALUE msgdef) { |
||||||
|
const rb_msgdef *rmd; |
||||||
|
Data_Get_Struct(msgdef, rb_msgdef, rmd); |
||||||
|
|
||||||
|
rb_msg *msg = (rb_msg*)ALLOC_N(char, rmd->size); |
||||||
|
memset(msg, 0, rmd->size); |
||||||
|
msg->data.rbmsgdef = msgdef; |
||||||
|
|
||||||
|
VALUE ret = Data_Wrap_Struct(cMessage, msg_mark, msg_free, msg); |
||||||
|
return ret; |
||||||
|
} |
||||||
|
|
||||||
|
/* msgdef *********************************************************************/ |
||||||
|
|
||||||
|
static void msgdef_free(void *_rmd) { |
||||||
|
rb_msgdef *rmd = _rmd; |
||||||
|
upb_msgdef_unref(rmd->md, &rmd->md); |
||||||
|
upb_pbdecodermethod_unref(rmd->fill_method, &rmd->fill_method); |
||||||
|
free(rmd->field_offsets); |
||||||
|
} |
||||||
|
|
||||||
|
const upb_pbdecodermethod *new_fillmsg_decodermethod(const rb_msgdef *rmd, |
||||||
|
const void *owner) { |
||||||
|
const upb_handlers *fill_handlers = new_fill_handlers(rmd, &fill_handlers); |
||||||
|
upb_pbdecodermethodopts opts; |
||||||
|
upb_pbdecodermethodopts_init(&opts, fill_handlers); |
||||||
|
|
||||||
|
const upb_pbdecodermethod *ret = upb_pbdecodermethod_new(&opts, owner); |
||||||
|
upb_handlers_unref(fill_handlers, &fill_handlers); |
||||||
|
return ret; |
||||||
|
} |
||||||
|
|
||||||
|
// Calculates offsets for each field.
|
||||||
|
//
|
||||||
|
// This lets us pack protos like structs instead of storing them like
|
||||||
|
// dictionaries. This speeds up a parsing a lot and also saves memory
|
||||||
|
// (unless messages are very sparse).
|
||||||
|
static void assign_offsets(rb_msgdef *rmd) { |
||||||
|
size_t ofs = sizeof(rb_msgdef); // Msg starts with msgdef pointer.
|
||||||
|
upb_msg_iter i; |
||||||
|
for (upb_msg_begin(&i, rmd->md); !upb_msg_done(&i); upb_msg_next(&i)) { |
||||||
|
upb_fielddef *f = upb_msg_iter_field(&i); |
||||||
|
size_t field_size = rupb_sizeof(upb_fielddef_type(f)); |
||||||
|
ofs = align_up(ofs, field_size); // Align field properly.
|
||||||
|
rmd->field_offsets[upb_fielddef_index(f)] = ofs; |
||||||
|
ofs += field_size; |
||||||
|
} |
||||||
|
rmd->size = ofs; |
||||||
|
} |
||||||
|
|
||||||
|
// Constructs a new Ruby wrapper object around the given msgdef.
|
||||||
|
static VALUE make_msgdef(const upb_msgdef *md) { |
||||||
|
rb_msgdef *rmd; |
||||||
|
VALUE ret = Data_Make_Struct(cMessageDef, rb_msgdef, NULL, msgdef_free, rmd); |
||||||
|
|
||||||
|
upb_msgdef_ref(md, &rmd->md); |
||||||
|
|
||||||
|
rmd->md = md; |
||||||
|
rmd->field_offsets = ALLOC_N(uint32_t, upb_msgdef_numfields(md)); |
||||||
|
rmd->fill_method = new_fillmsg_decodermethod(rmd, &rmd->fill_method); |
||||||
|
|
||||||
|
assign_offsets(rmd); |
||||||
|
|
||||||
|
return ret; |
||||||
|
} |
||||||
|
|
||||||
|
// Loads a descriptor and constructs a MessageDef to the named message.
|
||||||
|
static VALUE msgdef_load(VALUE klass, VALUE descriptor, VALUE message_name) { |
||||||
|
upb_symtab *symtab = upb_symtab_new(UPB_UNTRACKED_REF); |
||||||
|
|
||||||
|
// Wrap the symtab in a Ruby object so it gets GC'd.
|
||||||
|
// In a real wrapper we would wrap this object more fully (ie. expose its
|
||||||
|
// methods to Ruby callers).
|
||||||
|
Data_Wrap_Struct(rb_cObject, NULL, symtab_free, symtab); |
||||||
|
|
||||||
|
upb_status status = UPB_STATUS_INIT; |
||||||
|
upb_load_descriptor_into_symtab( |
||||||
|
symtab, RSTRING_PTR(descriptor), RSTRING_LEN(descriptor), &status); |
||||||
|
|
||||||
|
if (!upb_ok(&status)) { |
||||||
|
rb_raise(rb_eRuntimeError, |
||||||
|
"Error loading descriptor: %s", upb_status_errmsg(&status)); |
||||||
|
} |
||||||
|
|
||||||
|
const char *name = RSTRING_PTR(message_name); |
||||||
|
const upb_msgdef *m = upb_symtab_lookupmsg(symtab, name); |
||||||
|
|
||||||
|
if (!m) { |
||||||
|
rb_raise(rb_eRuntimeError, "Message name '%s' not found", name); |
||||||
|
} |
||||||
|
|
||||||
|
return make_msgdef(m); |
||||||
|
} |
||||||
|
|
||||||
|
static VALUE msgdef_parse(VALUE self, VALUE binary_protobuf) { |
||||||
|
const rb_msgdef *rmd; |
||||||
|
Data_Get_Struct(self, rb_msgdef, rmd); |
||||||
|
|
||||||
|
VALUE msg = msg_new(self); |
||||||
|
rb_msg *msgp; |
||||||
|
Data_Get_Struct(msg, rb_msg, msgp); |
||||||
|
|
||||||
|
const upb_handlers *h = upb_pbdecodermethod_desthandlers(rmd->fill_method); |
||||||
|
upb_pbdecoder decoder; |
||||||
|
upb_sink sink; |
||||||
|
upb_status status = UPB_STATUS_INIT; |
||||||
|
|
||||||
|
upb_pbdecoder_init(&decoder, rmd->fill_method, &status); |
||||||
|
upb_sink_reset(&sink, h, msgp); |
||||||
|
upb_pbdecoder_resetoutput(&decoder, &sink); |
||||||
|
fprintf(stderr, "STR: %s\n", RSTRING_PTR(binary_protobuf)); |
||||||
|
fprintf(stderr, "LEN: %d\n", (int)RSTRING_LEN(binary_protobuf)); |
||||||
|
size_t n = upb_bufsrc_putbuf(RSTRING_PTR(binary_protobuf), |
||||||
|
RSTRING_LEN(binary_protobuf), |
||||||
|
upb_pbdecoder_input(&decoder)); |
||||||
|
fprintf(stderr, "n: %d\n", (int)n); |
||||||
|
// TODO(haberman): make uninit optional if custom allocator for parsing
|
||||||
|
// returns GC-rooted memory. That will make decoding longjmp-safe (required
|
||||||
|
// if parsing triggers any VM errors like OOM or errors in user handlers).
|
||||||
|
upb_pbdecoder_uninit(&decoder); |
||||||
|
rupb_checkstatus(&status); |
||||||
|
|
||||||
|
return msg; |
||||||
|
} |
||||||
|
|
||||||
|
void Init_upb() { |
||||||
|
VALUE upb = rb_define_module("Upb"); |
||||||
|
|
||||||
|
cMessageDef = rb_define_class_under(upb, "MessageDef", rb_cObject); |
||||||
|
rb_define_singleton_method(cMessageDef, "load", msgdef_load, 2); |
||||||
|
rb_define_method(cMessageDef, "parse", msgdef_parse, 1); |
||||||
|
|
||||||
|
cMessage = rb_define_class_under(upb, "Message", rb_cObject); |
||||||
|
} |
||||||
Loading…
Reference in new issue