Sync to Google-internal development.

pull/13171/head
Josh Haberman 10 years ago
parent 648afe3da6
commit 3d0c7c45da
  1. 59
      Makefile
  2. 14
      tests/bindings/googlepb/test_vs_proto2.cc
  3. 62
      tests/bindings/ruby/upb.rb
  4. 18
      tests/pb/test_decoder.cc
  5. 10
      tools/dump_cinit.lua
  6. 1
      upb/bindings/googlepb/bridge.cc
  7. 27
      upb/bindings/googlepb/proto1.cc
  8. 220
      upb/bindings/googlepb/proto2.cc
  9. 21
      upb/bindings/lua/upb.c
  10. 2
      upb/bindings/ruby/README
  11. 30
      upb/bindings/ruby/README.md
  12. 8
      upb/bindings/ruby/extconf.rb
  13. 1209
      upb/bindings/ruby/upb.c
  14. 25
      upb/def.c
  15. 85
      upb/def.h
  16. 162
      upb/descriptor/descriptor.upb.c
  17. 11
      upb/descriptor/reader.c
  18. 308
      upb/pb/compile_decoder.c
  19. 2
      upb/pb/compile_decoder_x64.c
  20. 28
      upb/pb/compile_decoder_x64.dasc
  21. 769
      upb/pb/encoder.c
  22. 163
      upb/pb/encoder.h
  23. 23
      upb/pb/varint.c
  24. 9
      upb/pb/varint.int.h
  25. 106
      upb/table.c
  26. 29
      upb/table.int.h

@ -44,8 +44,8 @@ CXX=c++
CFLAGS=-std=c99
CXXFLAGS=-Wno-unused-private-field
INCLUDE=-I.
CPPFLAGS=$(INCLUDE) -DNDEBUG -Wall -Wextra -Wno-sign-compare $(USER_CPPFLAGS)
LDLIBS=-lpthread upb/libupb.a
WARNFLAGS=-Wall -Wextra -Wno-sign-compare
CPPFLAGS=$(INCLUDE) -DNDEBUG $(USER_CPPFLAGS)
LUA=lua # 5.1 and 5.2 should both be supported
ifneq ($(WITH_JIT), no)
@ -91,6 +91,7 @@ clean_leave_profile:
@rm -rf tools/upbc deps
@rm -rf upb/bindings/python/build
@rm -f upb/bindings/ruby/Makefile
@rm -f upb/bindings/ruby/upb.o
@rm -f upb/bindings/ruby/upb.so
@rm -f upb/bindings/ruby/mkmf.log
@find . | grep dSYM | xargs rm -rf
@ -109,7 +110,7 @@ clean: clean_leave_profile
# lib/lib%.a: $(call make_objs,o)
# gcc -c -o $@ $^
#
# SECONDEXPANSION: flips on a bit essentially that allows this "seconary
# SECONDEXPANSION: flips on a bit essentially that allows this "secondary
# expansion": it must appear before anything that uses make_objs.
.SECONDEXPANSION:
to_srcs = $(subst .,_,$(1)_SRCS)
@ -130,15 +131,16 @@ upb_SRCS = \
upb/upb.c \
upb_descriptor_SRCS = \
upb/descriptor/reader.c \
upb/descriptor/descriptor.upb.c \
upb/descriptor/reader.c \
upb_pb_SRCS = \
upb/pb/decoder.c \
upb/pb/compile_decoder.c \
upb/pb/decoder.c \
upb/pb/encoder.c \
upb/pb/glue.c \
upb/pb/varint.c \
upb/pb/textprinter.c \
upb/pb/varint.c \
# If the JIT is enabled we include its source.
# If Lua is present we can use DynASM to regenerate the .h file.
@ -146,7 +148,6 @@ ifdef USE_JIT
upb_pb_SRCS += upb/pb/compile_decoder_x64.c
obj/pb/compile_decoder_x64.o obj/pb/compile_decoder_x64.lo: upb/pb/compile_decoder_x64.h
obj/pb/compile_decoder_x64.o: CFLAGS=-std=gnu99
obj/pb/compile_decoder_x64.o: OPT=-Os
upb/pb/compile_decoder_x64.h: upb/pb/compile_decoder_x64.dasc
$(E) DYNASM $<
@ -164,6 +165,11 @@ lib/libupb.a : OPT = -Os
lib/libupb.descriptor.a : OPT = -Os
obj/pb/compile_decoder.o : OPT = -Os
obj/pb/compile_decoder_64.o : OPT = -Os
ifdef USE_JIT
obj/pb/compile_decoder_x64.o: OPT=-Os
endif
endif
$(UPB_PICLIBS): lib/lib%_pic.a: $(call make_objs,lo)
@ -177,32 +183,32 @@ $(UPB_LIBS): lib/lib%.a: $(call make_objs,o)
obj/%.o: upb/%.c | $$(@D)/.
$(E) CC $<
$(Q) $(CC) $(OPT) $(CPPFLAGS) $(CFLAGS) -c -o $@ $<
$(Q) $(CC) $(OPT) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -c -o $@ $<
obj/%.o: upb/%.cc | $$(@D)/.
$(E) CXX $<
$(Q) $(CXX) $(OPT) $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $<
$(Q) $(CXX) $(OPT) $(WARNFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $<
obj/%.lo: upb/%.c | $$(@D)/.
$(E) 'CC -fPIC' $<
$(Q) $(CC) $(OPT) $(CPPFLAGS) $(CFLAGS) -c -o $@ $< -fPIC
$(Q) $(CC) $(OPT) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -c -o $@ $< -fPIC
obj/%.lo: upb/%.cc | $$(@D)/.
$(E) CXX $<
$(Q) $(CXX) $(OPT) $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $< -fPIC
$(Q) $(CXX) $(OPT) $(WARNFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $< -fPIC
# Note: mkdir -p is technically susceptible to races when used with make -j.
%/.:
$(Q) mkdir -p $@
# Regenerating the auto-generated files in upb/.
upb/descriptor.pb: upb/descriptor.proto
upb/descriptor/descriptor.pb: upb/descriptor/descriptor.proto
@# TODO: replace with upbc
protoc upb/descriptor.proto -oupb/descriptor.pb
protoc upb/descriptor/descriptor.proto -oupb/descriptor/descriptor.pb
descriptorgen: upb/descriptor.pb tools/upbc
descriptorgen: upb/descriptor/descriptor.pb tools/upbc
@# Regenerate descriptor_const.h
./tools/upbc -o upb/descriptor upb/descriptor.pb
./tools/upbc -o upb/descriptor/descriptor upb/descriptor/descriptor.pb
tools/upbc: tools/upbc.c $(LIBUPB)
$(E) CC $<
@ -232,15 +238,15 @@ tests: $(TESTS)
tests/testmain.o: tests/testmain.cc
$(E) CXX $<
$(Q) $(CXX) $(CXXFLAGS) $(CPPFLAGS) -c -o $@ $<
$(Q) $(CXX) $(OPT) $(WARNFLAGS) $(CXXFLAGS) $(CPPFLAGS) -c -o $@ $<
$(C_TESTS): % : %.c tests/testmain.o $$(LIBS)
$(E) CC $<
$(Q) $(CC) $(CPPFLAGS) $(CFLAGS) -o $@ tests/testmain.o $< $(LIBS)
$(Q) $(CC) $(OPT) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -o $@ tests/testmain.o $< $(LIBS)
$(CC_TESTS): % : %.cc tests/testmain.o $$(LIBS)
$(E) CXX $<
$(Q) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -Wno-deprecated -o $@ tests/testmain.o $< $(LIBS)
$(Q) $(CXX) $(OPT) $(WARNFLAGS) $(CPPFLAGS) $(CXXFLAGS) -Wno-deprecated -o $@ tests/testmain.o $< $(LIBS)
# Several of these tests don't actually test these libs, but use them
# incidentally to load a descriptor
@ -337,7 +343,7 @@ GOOGLEPB_TEST_DEPS = \
tests/bindings/googlepb/test_vs_proto2.googlemessage1: $(GOOGLEPB_TEST_DEPS) \
benchmarks/googlemessage1.h
$(E) CXX $< '(benchmarks::SpeedMessage1)'
$(Q) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< \
$(Q) $(CXX) $(OPT) $(WARNFLAGS) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< \
-DMESSAGE_CIDENT="benchmarks::SpeedMessage1" \
-DMESSAGE_DATA_HFILE=\"benchmarks/googlemessage1.h\" \
benchmarks/google_messages.pb.cc tests/testmain.o -lprotobuf -lpthread \
@ -346,7 +352,7 @@ tests/bindings/googlepb/test_vs_proto2.googlemessage1: $(GOOGLEPB_TEST_DEPS) \
tests/bindings/googlepb/test_vs_proto2.googlemessage2: $(GOOGLEPB_TEST_DEPS) \
benchmarks/googlemessage2.h
$(E) CXX $< '(benchmarks::SpeedMessage2)'
$(Q) $(CXX) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< \
$(Q) $(CXX) $(OPT) $(WARNFLAGS) $(CPPFLAGS) $(CXXFLAGS) -o $@ $< \
-DMESSAGE_CIDENT="benchmarks::SpeedMessage2" \
-DMESSAGE_DATA_HFILE=\"benchmarks/googlemessage2.h\" \
benchmarks/google_messages.pb.cc tests/testmain.o -lprotobuf -lpthread \
@ -401,7 +407,7 @@ LUA_LIB_DEPS = \
upb/bindings/lua/upb.so: upb/bindings/lua/upb.c upb/bindings/lua/upb.lua.h $(LUA_LIB_DEPS)
$(E) CC upb/bindings/lua/upb.c
$(Q) $(CC) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $< $(LUA_LDFLAGS) $(LUA_LIB_DEPS)
$(Q) $(CC) $(OPT) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $< $(LUA_LDFLAGS) $(LUA_LIB_DEPS)
# TODO: the dependency between upb/pb.so and upb.so is expressed at the
# .so level, which means that the OS will try to load upb.so when upb/pb.so
@ -414,7 +420,7 @@ upb/bindings/lua/upb.so: upb/bindings/lua/upb.c upb/bindings/lua/upb.lua.h $(LUA
# be expressed at the .so level.
upb/bindings/lua/upb/pb.so: upb/bindings/lua/upb/pb.c upb/bindings/lua/upb.so
$(E) CC upb/bindings/lua/upb.pb.c
$(Q) $(CC) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $< upb/bindings/lua/upb.so $(LUA_LDFLAGS)
$(Q) $(CC) $(OPT) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $< upb/bindings/lua/upb.so $(LUA_LDFLAGS)
# Python extension #############################################################
@ -434,9 +440,16 @@ pythontest: $(PYTHONEXT)
RUBY=ruby
RUBYEXT=upb/bindings/ruby/upb.so
ruby: $(RUBYEXT)
# We pass our important flags to Ruby, but leave the warning flags out.
# Some uses of the Ruby/C API trigger the warnings we normally use, so
# we let Ruby decide the set of warning options to use.
upb/bindings/ruby/Makefile: upb/bindings/ruby/extconf.rb lib/libupb_pic.a lib/libupb.pb_pic.a lib/libupb.descriptor_pic.a
$(E) RUBY upb/bindings/ruby/extconf.rb
$(Q) cd upb/bindings/ruby && ruby extconf.rb
$(Q) cd upb/bindings/ruby && ruby extconf.rb "$(OPT) $(CPPFLAGS) $(CFLAGS)"
$(RUBYEXT): upb/bindings/ruby/upb.c upb/bindings/ruby/Makefile
$(E) CC upb/bindings/ruby/upb.c
$(Q) cd upb/bindings/ruby && make
rubytest: $(RUBYEXT) upb/descriptor/descriptor.pb
RUBYLIB="upb/bindings/ruby" ruby tests/bindings/ruby/upb.rb

@ -27,8 +27,14 @@
#include "upb/pb/glue.h"
#include "upb/pb/varint.int.h"
static const char message_data[] = {
#include MESSAGE_DATA_HFILE
// Pull in string data from benchmarks/google_message{1,2}.dat
// (the .h files are generated with xxd).
const char message1_data[] = {
#include "benchmarks/google_message1.h"
};
const char message2_data[] = {
#include "benchmarks/google_message2.h"
};
void compare_metadata(const google::protobuf::Descriptor* d,
@ -117,8 +123,8 @@ extern "C" {
int run_tests(int argc, char *argv[]) {
UPB_UNUSED(argc);
UPB_UNUSED(argv);
size_t len = sizeof(message_data);
const char *str = message_data;
size_t len = sizeof(MESSAGE_DATA_IDENT);
const char *str = MESSAGE_DATA_IDENT;
MESSAGE_CIDENT msg1;
MESSAGE_CIDENT msg2;

@ -0,0 +1,62 @@
#!/usr/bin/ruby
#
# Tests for Ruby upb extension.
require 'test/unit'
require 'set'
require 'upb'
def get_descriptor
File.open("upb/descriptor/descriptor.pb").read
end
def load_descriptor
symtab = Upb::SymbolTable.new
symtab.load_descriptor(get_descriptor())
return symtab
end
def get_message_class(name)
return Upb.get_message_class(load_descriptor().lookup(name))
end
class TestRubyExtension < Test::Unit::TestCase
def test_parsedescriptor
msgdef = load_descriptor.lookup("google.protobuf.FileDescriptorSet")
assert_instance_of(Upb::MessageDef, msgdef)
file_descriptor_set = Upb.get_message_class(msgdef)
msg = file_descriptor_set.parse(get_descriptor())
# A couple message types we know should exist.
names = Set.new(["DescriptorProto", "FieldDescriptorProto"])
msg.file.each { |file|
file.message_type.each { |message_type|
names.delete(message_type.name)
}
}
assert_equal(0, names.size)
end
def test_parseserialize
field_descriptor_proto = get_message_class("google.protobuf.FieldDescriptorProto")
field_options = get_message_class("google.protobuf.FieldOptions")
field = field_descriptor_proto.new
field.name = "MyName"
field.number = 5
field.options = field_options.new
field.options.packed = true
serialized = Upb::Message.serialize(field)
field2 = field_descriptor_proto.parse(serialized)
assert_equal("MyName", field2.name)
assert_equal(5, field2.number)
assert_equal(true, field2.options.packed)
end
end

@ -207,8 +207,16 @@ void indentbuf(string *buf, int depth) {
buf->append(2 * depth, ' ');
}
void check_stack_alignment() {
#ifdef UPB_USE_JIT_X64
void *rsp = __builtin_frame_address(0);
ASSERT(((uintptr_t)rsp % 16) == 0);
#endif
}
#define NUMERIC_VALUE_HANDLER(member, ctype, fmt) \
bool value_##member(int* depth, const uint32_t* num, ctype val) { \
check_stack_alignment(); \
indentbuf(&output, *depth); \
appendf(&output, "%" PRIu32 ":%" fmt "\n", *num, val); \
return true; \
@ -222,12 +230,14 @@ NUMERIC_VALUE_HANDLER(float, float, "g")
NUMERIC_VALUE_HANDLER(double, double, "g")
bool value_bool(int* depth, const uint32_t* num, bool val) {
check_stack_alignment();
indentbuf(&output, *depth);
appendf(&output, "%" PRIu32 ":%s\n", *num, val ? "true" : "false");
return true;
}
int* startstr(int* depth, const uint32_t* num, size_t size_hint) {
check_stack_alignment();
indentbuf(&output, *depth);
appendf(&output, "%" PRIu32 ":(%zu)\"", *num, size_hint);
return depth + 1;
@ -237,6 +247,7 @@ size_t value_string(int* depth, const uint32_t* num, const char* buf,
size_t n, const upb::BufferHandle* handle) {
UPB_UNUSED(num);
UPB_UNUSED(depth);
check_stack_alignment();
output.append(buf, n);
ASSERT(handle == &global_handle);
return n;
@ -245,11 +256,13 @@ size_t value_string(int* depth, const uint32_t* num, const char* buf,
bool endstr(int* depth, const uint32_t* num) {
UPB_UNUSED(depth);
UPB_UNUSED(num);
check_stack_alignment();
output.append("\"\n");
return true;
}
int* startsubmsg(int* depth, const uint32_t* num) {
check_stack_alignment();
indentbuf(&output, *depth);
appendf(&output, "%" PRIu32 ":{\n", *num);
return depth + 1;
@ -257,12 +270,14 @@ int* startsubmsg(int* depth, const uint32_t* num) {
bool endsubmsg(int* depth, const uint32_t* num) {
UPB_UNUSED(num);
check_stack_alignment();
indentbuf(&output, *depth);
output.append("}\n");
return true;
}
int* startseq(int* depth, const uint32_t* num) {
check_stack_alignment();
indentbuf(&output, *depth);
appendf(&output, "%" PRIu32 ":[\n", *num);
return depth + 1;
@ -270,12 +285,14 @@ int* startseq(int* depth, const uint32_t* num) {
bool endseq(int* depth, const uint32_t* num) {
UPB_UNUSED(num);
check_stack_alignment();
indentbuf(&output, *depth);
output.append("]\n");
return true;
}
bool startmsg(int* depth) {
check_stack_alignment();
indentbuf(&output, *depth);
output.append("<\n");
return true;
@ -283,6 +300,7 @@ bool startmsg(int* depth) {
bool endmsg(int* depth, upb_status* status) {
UPB_UNUSED(status);
check_stack_alignment();
indentbuf(&output, *depth);
output.append(">\n");
return true;

@ -453,14 +453,14 @@ local function dump_defs_c(symtab, basename, namespace, append)
intfmt = "0"
end
-- UPB_FIELDDEF_INIT(label, type, intfmt, tagdelim, is_extension, lazy,
-- name, num, msgdef, subdef, selector_base, index,
-- default_value)
append(' UPB_FIELDDEF_INIT(%s, %s, %s, %s, %s, %s, "%s", %d, %s, %s, ' ..
'%d, %d, {0},' .. -- TODO: support default value
-- packed, name, num, msgdef, subdef, selector_base,
-- index, -- default_value)
append(' UPB_FIELDDEF_INIT(%s, %s, %s, %s, %s, %s, %s, "%s", %d, %s, ' ..
'%s, %d, %d, {0},' .. -- TODO: support default value
'&reftables[%d], &reftables[%d]),\n',
const(f, "label"), const(f, "type"), intfmt,
boolstr(f:istagdelim()), boolstr(f:is_extension()),
boolstr(f:lazy()), f:name(), f:number(),
boolstr(f:lazy()), boolstr(f:packed()), f:name(), f:number(),
linktab:addr(f:containing_type()), subdef,
f:_selector_base(), f:index(),
reftable, reftable + 1

@ -115,6 +115,7 @@ reffed_ptr<FieldDef> DefBuilder::NewFieldDef(const goog::FieldDescriptor* f,
upb_f->set_number(f->number(), &status);
upb_f->set_label(FieldDef::ConvertLabel(f->label()));
upb_f->set_descriptor_type(FieldDef::ConvertDescriptorType(f->type()));
upb_f->set_packed(f->options().packed());
#ifdef UPB_GOOGLE3
upb_f->set_lazy(f->options().lazy());
#endif

@ -30,6 +30,10 @@
#undef private
#undef protected
#ifdef GOOGLE_PROTOBUF_HAS_ARENAS
namespace proto2 { class Arena; }
#endif
#include "upb/def.h"
#include "upb/handlers.h"
#include "upb/shim/shim.h"
@ -448,12 +452,35 @@ class P2R_Handlers {
class RepeatedMessageTypeHandler {
public:
typedef proto2::Message Type;
#ifndef GOOGLE_PROTOBUF_HAS_ARENAS
// AddAllocated() calls this, but only if other objects are sitting
// around waiting for reuse, which we will not do.
static void Delete(Type* t) {
UPB_UNUSED(t);
assert(false);
}
#else
static ::proto2::Arena* GetArena(Type* t) {
return t->GetArena();
}
static void* GetMaybeArenaPointer(Type* t) {
return t->GetMaybeArenaPointer();
}
static inline Type* NewFromPrototype(
const Type* prototype, ::proto2::Arena* arena = NULL) {
return prototype->New(arena);
}
// AddAllocated() calls this, but only if other objects are sitting
// around waiting for reuse, which we will not do.
static void Delete(Type* t, ::proto2::Arena* arena) {
UPB_UNUSED(t);
UPB_UNUSED(arena);
assert(false);
}
static void Merge(const Type& from, Type* to) {
to->MergeFrom(from);
}
#endif
};
// Closure is a RepeatedPtrField<SubMessageType>*, but we access it through

@ -261,11 +261,64 @@ case goog::FieldDescriptor::cpptype: \
return r->offsets_[index];
}
class FieldOffset {
// Base class that provides access to elements of the message as a whole, such
// as the unknown-field set, and is inherited by context classes for specific
// field handlers.
class FieldDataBase {
public:
FieldDataBase(const goog::internal::GeneratedMessageReflection* r)
: unknown_fields_offset_(r->unknown_fields_offset_)
#ifdef GOOGLE_PROTOBUF_HAS_ARENAS
, arena_offset_(r->arena_offset_)
#endif // GOOGLE_PROTOBUF_HAS_ARENAS
{}
#ifdef GOOGLE_PROTOBUF_HAS_ARENAS
goog::Arena* GetArena(const goog::Message& message) const {
if (unknown_fields_offset_ ==
goog::internal::GeneratedMessageReflection::
kUnknownFieldSetInMetadata) {
const goog::internal::InternalMetadataWithArena* metadata =
GetConstPointer<goog::internal::InternalMetadataWithArena>(
&message, arena_offset_);
return metadata->arena();
} else if (arena_offset_ !=
goog::internal::GeneratedMessageReflection::kNoArenaPointer) {
return *GetConstPointer<goog::Arena*>(&message, arena_offset_);
} else {
return NULL;
}
}
goog::UnknownFieldSet* GetUnknownFieldSet(goog::Message* message) const {
if (unknown_fields_offset_ ==
goog::internal::GeneratedMessageReflection::
kUnknownFieldSetInMetadata) {
goog::internal::InternalMetadataWithArena* metadata =
GetPointer<goog::internal::InternalMetadataWithArena>(
message, arena_offset_);
return metadata->mutable_unknown_fields();
}
return GetPointer<goog::UnknownFieldSet>(message, unknown_fields_offset_);
}
#else // ifdef GOOGLE_PROTOBUF_HAS_ARENAS
goog::UnknownFieldSet* GetUnknownFieldSet(goog::Message* message) const {
return GetPointer<goog::UnknownFieldSet>(message, unknown_fields_offset_);
}
#endif // ifdef !GOOGLE_PROTOBUF_HAS_ARENAS
private:
int unknown_fields_offset_;
#ifdef GOOGLE_PROTOBUF_HAS_ARENAS
int arena_offset_;
#endif // GOOGLE_PROTOBUF_HAS_ARENAS
};
class FieldOffset : public FieldDataBase {
public:
FieldOffset(const goog::FieldDescriptor* f,
const goog::internal::GeneratedMessageReflection* r)
: offset_(GetOffset(f, r)), is_repeated_(f->is_repeated()) {
: FieldDataBase(r),
offset_(GetOffset(f, r)), is_repeated_(f->is_repeated()) {
if (!is_repeated_) {
int64_t hasbit = GetHasbit(f, r);
hasbyte_ = hasbit / 8;
@ -293,11 +346,12 @@ case goog::FieldDescriptor::cpptype: \
};
#ifdef GOOGLE_PROTOBUF_HAS_ONEOF
class OneofFieldData {
class OneofFieldData : public FieldDataBase {
public:
OneofFieldData(const goog::FieldDescriptor* f,
const goog::internal::GeneratedMessageReflection* r)
: field_number_offset_(GetOneofDiscriminantOffset(f, r)),
: FieldDataBase(r),
field_number_offset_(GetOneofDiscriminantOffset(f, r)),
field_number_(f->number()) {
const goog::OneofDescriptor* oneof = f->containing_oneof();
@ -343,6 +397,40 @@ case goog::FieldDescriptor::cpptype: \
return GetPointer<int32_t>(message, field_number_offset_);
}
void ClearOneof(goog::Message* m, const FieldOffset* ofs,
int field_number) const {
#ifdef GOOGLE_PROTOBUF_HAS_ARENAS
if (GetArena(*m) != NULL) {
return;
}
#endif
switch (types_.at(field_number)) {
case ONEOF_TYPE_NONE:
break;
case ONEOF_TYPE_STRING:
delete *ofs->GetFieldPointer<std::string*>(m);
break;
case ONEOF_TYPE_MESSAGE:
delete *ofs->GetFieldPointer<goog::Message*>(m);
break;
#ifdef UPB_GOOGLE3
case ONEOF_TYPE_GLOBALSTRING:
delete *ofs->GetFieldPointer<string*>(m);
break;
case ONEOF_TYPE_CORD:
delete *ofs->GetFieldPointer<Cord*>(m);
break;
case ONEOF_TYPE_STRINGPIECE:
delete *ofs->GetFieldPointer<
goog::internal::StringPieceField*>(m);
break;
case ONEOF_TYPE_LAZYFIELD:
delete *ofs->GetFieldPointer<goog::internal::LazyField*>(m);
break;
#endif
}
}
// Returns whether this is different than the previous value of the
// field_number; this implies that the current value was freed (if
// necessary) and the caller should allocate a new instance.
@ -351,30 +439,7 @@ case goog::FieldDescriptor::cpptype: \
if (*field_number == field_number_) {
return false;
} else {
switch (types_.at(*field_number)) {
case ONEOF_TYPE_NONE:
break;
case ONEOF_TYPE_STRING:
delete *ofs->GetFieldPointer<std::string*>(m);
break;
case ONEOF_TYPE_MESSAGE:
delete *ofs->GetFieldPointer<goog::Message*>(m);
break;
#ifdef UPB_GOOGLE3
case ONEOF_TYPE_GLOBALSTRING:
delete *ofs->GetFieldPointer<string*>(m);
break;
case ONEOF_TYPE_CORD:
delete *ofs->GetFieldPointer<Cord*>(m);
break;
case ONEOF_TYPE_STRINGPIECE:
delete *ofs->GetFieldPointer<goog::internal::StringPieceField*>(m);
break;
case ONEOF_TYPE_LAZYFIELD:
delete *ofs->GetFieldPointer<goog::internal::LazyField*>(m);
break;
#endif
}
ClearOneof(m, ofs, *field_number);
*field_number = field_number_;
return true;
}
@ -578,7 +643,6 @@ case goog::FieldDescriptor::cpptype: \
const upb::FieldDef* f)
: FieldOffset(proto2_f, r),
field_number_(f->number()),
unknown_fields_offset_(r->unknown_fields_offset_),
enum_(upb_downcast_enumdef(f->subdef())) {}
bool IsValidValue(int32_t val) const {
@ -587,13 +651,8 @@ case goog::FieldDescriptor::cpptype: \
int32_t field_number() const { return field_number_; }
goog::UnknownFieldSet* mutable_unknown_fields(goog::Message* m) const {
return GetPointer<goog::UnknownFieldSet>(m, unknown_fields_offset_);
}
private:
int32_t field_number_;
size_t unknown_fields_offset_;
const upb::EnumDef* enum_;
};
@ -617,7 +676,7 @@ case goog::FieldDescriptor::cpptype: \
*message_val = val;
data->SetHasbit(m);
} else {
data->mutable_unknown_fields(m)->AddVarint(data->field_number(), val);
data->GetUnknownFieldSet(m)->AddVarint(data->field_number(), val);
}
}
@ -631,7 +690,7 @@ case goog::FieldDescriptor::cpptype: \
data->GetFieldPointer<goog::RepeatedField<int32_t> >(m);
r->Add(val);
} else {
data->mutable_unknown_fields(m)->AddVarint(data->field_number(), val);
data->GetUnknownFieldSet(m)->AddVarint(data->field_number(), val);
}
}
@ -718,7 +777,14 @@ case goog::FieldDescriptor::cpptype: \
T** str = data->GetStringPointer(m);
data->SetHasbit(m);
// If it points to the default instance, we must create a new instance.
if (*str == data->prototype()) *str = new T();
if (*str == data->prototype()) {
*str = new T();
#ifdef GOOGLE_PROTOBUF_HAS_ARENAS
if (data->GetArena(*m)) {
data->GetArena(*m)->Own(*str);
}
#endif
}
(*str)->clear();
// reserve() here appears to hurt performance rather than help.
return *str;
@ -749,6 +815,16 @@ case goog::FieldDescriptor::cpptype: \
T** str = ofs->GetFieldPointer<T*>(m);
if (data->SetOneofHas(m)) {
*str = new T();
#ifdef GOOGLE_PROTOBUF_HAS_ARENAS
// Note that in the main proto2-arenas implementation, the parsing code
// creates ArenaString instances for string field data, and the
// implementation later dynamically converts to ::string if a mutable
// version is requested. To keep complexity down in this binding, we
// create an ordinary string and allow the arena to own its destruction.
if (data->GetArena(*m) != NULL) {
data->GetArena(*m)->Own(*str);
}
#endif
} else {
(*str)->clear();
}
@ -857,7 +933,11 @@ case goog::FieldDescriptor::cpptype: \
data->SetHasbit(m);
goog::Message** subm = data->GetFieldPointer<goog::Message*>(m);
if (*subm == NULL || *subm == data->prototype()) {
#ifdef GOOGLE_PROTOBUF_HAS_ARENAS
*subm = data->prototype()->New(data->GetArena(*m));
#else
*subm = data->prototype()->New();
#endif
}
return *subm;
}
@ -865,14 +945,50 @@ case goog::FieldDescriptor::cpptype: \
class RepeatedMessageTypeHandler {
public:
typedef goog::Message Type;
#ifdef GOOGLE_PROTOBUF_HAS_ARENAS
static ::proto2::Arena* GetArena(Type* t) {
return t->GetArena();
}
static void* GetMaybeArenaPointer(Type* t) {
return t->GetMaybeArenaPointer();
}
static inline Type* NewFromPrototype(
const Type* prototype, ::proto2::Arena* arena = NULL) {
return prototype->New(arena);
}
static void Delete(Type* t, goog::Arena* arena = NULL) {
if (arena == NULL) {
delete t;
}
}
#else // ifdef GOOGLE_PROTOBUF_HAS_ARENAS
static inline Type* NewFromPrototype(const Type* prototype) {
return prototype->New();
}
// AddAllocated() calls this, but only if other objects are sitting
// around waiting for reuse, which we will not do.
static void Delete(Type* t) {
UPB_UNUSED(t);
assert(false);
}
#endif // ifdef GOOGLE_PROTOBUF_HAS_ARENAS
static void Merge(const Type& from, Type* to) {
to->MergeFrom(from);
}
};
#ifdef GOOGLE_PROTOBUF_HAS_ARENAS
// Closure is a RepeatedPtrField<SubMessageType>*, but we access it through
// its base class RepeatedPtrFieldBase*.
static goog::Message* StartRepeatedSubMessage(
goog::internal::RepeatedPtrFieldBase* r,
const SubMessageHandlerData* data) {
goog::Message* submsg = data->prototype()->New(r->GetArenaNoVirtual());
r->AddAllocated<RepeatedMessageTypeHandler>(submsg);
return submsg;
}
#else // ifdef GOOGLE_PROTOBUF_HAS_ARENAS
// Closure is a RepeatedPtrField<SubMessageType>*, but we access it through
// its base class RepeatedPtrFieldBase*.
static goog::Message* StartRepeatedSubMessage(
@ -886,13 +1002,19 @@ case goog::FieldDescriptor::cpptype: \
return submsg;
}
#endif // ifdef GOOGLE_PROTOBUF_HAS_ARENAS
#ifdef GOOGLE_PROTOBUF_HAS_ONEOF
static goog::Message* StartOneofSubMessage(
goog::Message* m, const OneofSubMessageHandlerData* data) {
const FieldOffset* ofs = data;
goog::Message** subm = ofs->GetFieldPointer<goog::Message*>(m);
if (data->SetOneofHas(m)) {
#ifdef GOOGLE_PROTOBUF_HAS_ARENAS
*subm = data->prototype()->New(data->GetArena(*m));
#else
*subm = data->prototype()->New();
#endif
}
return *subm;
}
@ -1123,9 +1245,21 @@ case goog::FieldDescriptor::cpptype: \
LazyMessageExtensionImpl() {}
virtual ~LazyMessageExtensionImpl() {}
#ifdef GOOGLE_PROTOBUF_HAS_ARENAS
virtual LazyMessageExtension* New() const {
return New(NULL);
}
virtual LazyMessageExtension* New(proto2::Arena* arena) const {
LazyMessageExtensionImpl* message =
::proto2::Arena::Create<LazyMessageExtensionImpl>(arena);
return message;
}
#else // ifdef GOOGLE_PROTOBUF_HAS_ARENAS
virtual LazyMessageExtension* New() const {
return new LazyMessageExtensionImpl();
}
#endif // ifdef GOOGLE_PROTOBUF_HAS_ARENAS
virtual const proto2::MessageLite& GetMessage(
const proto2::MessageLite& prototype) const {
@ -1149,6 +1283,12 @@ case goog::FieldDescriptor::cpptype: \
static_cast<const proto2::Message&>(prototype));
}
virtual proto2::MessageLite* UnsafeArenaReleaseMessage(
const proto2::MessageLite& prototype) {
return lazy_field_.UnsafeArenaReleaseByPrototype(
static_cast<const proto2::Message&>(prototype));
}
virtual bool IsInitialized() const { return true; }
virtual int ByteSize() const { return lazy_field_.MessageByteSize(); }
@ -1201,7 +1341,13 @@ case goog::FieldDescriptor::cpptype: \
LazyMessageExtensionImpl* lazy_extension;
if (set->MaybeNewExtension(data->number(), data->field_descriptor(),
&item)) {
#ifdef GOOGLE_PROTOBUF_HAS_ARENAS
lazy_extension =
::proto2::Arena::Create<LazyMessageExtensionImpl>(
m->GetArena());
#else
lazy_extension = new LazyMessageExtensionImpl();
#endif
item->type = UPB_DESCRIPTOR_TYPE_MESSAGE;
item->is_repeated = false;
item->is_lazy = true;

@ -640,6 +640,12 @@ static int lupb_fielddef_number(lua_State *L) {
return 1;
}
static int lupb_fielddef_packed(lua_State *L) {
const upb_fielddef *f = lupb_fielddef_check(L, 1);
lua_pushboolean(L, upb_fielddef_packed(f));
return 1;
}
static int lupb_fielddef_subdef(lua_State *L) {
const upb_fielddef *f = lupb_fielddef_check(L, 1);
if (!upb_fielddef_hassubdef(f))
@ -753,6 +759,12 @@ static int lupb_fielddef_setnumber(lua_State *L) {
return 0;
}
static int lupb_fielddef_setpacked(lua_State *L) {
upb_fielddef *f = lupb_fielddef_checkmutable(L, 1);
upb_fielddef_setpacked(f, lupb_checkbool(L, 2));
return 0;
}
static int lupb_fielddef_setsubdef(lua_State *L) {
upb_fielddef *f = lupb_fielddef_checkmutable(L, 1);
const upb_def *def = NULL;
@ -817,6 +829,7 @@ static const struct luaL_Reg lupb_fielddef_m[] = {
{"lazy", lupb_fielddef_lazy},
{"name", lupb_fielddef_name},
{"number", lupb_fielddef_number},
{"packed", lupb_fielddef_packed},
{"subdef", lupb_fielddef_subdef},
{"subdef_name", lupb_fielddef_subdefname},
{"type", lupb_fielddef_type},
@ -828,6 +841,7 @@ static const struct luaL_Reg lupb_fielddef_m[] = {
{"set_lazy", lupb_fielddef_setlazy},
{"set_name", lupb_fielddef_setname},
{"set_number", lupb_fielddef_setnumber},
{"set_packed", lupb_fielddef_setpacked},
{"set_subdef", lupb_fielddef_setsubdef},
{"set_subdef_name", lupb_fielddef_setsubdefname},
{"set_type", lupb_fielddef_settype},
@ -926,7 +940,7 @@ static int lupb_msgdef_field(lua_State *L) {
if (type == LUA_TNUMBER) {
f = upb_msgdef_itof(m, lua_tointeger(L, 2));
} else if (type == LUA_TSTRING) {
f = upb_msgdef_ntof(m, lua_tostring(L, 2));
f = upb_msgdef_ntofz(m, lua_tostring(L, 2));
} else {
const char *msg = lua_pushfstring(L, "number or string expected, got %s",
luaL_typename(L, 2));
@ -1358,8 +1372,9 @@ const upb_msgdef *lupb_msg_checkdef(lua_State *L, int narg) {
static const upb_fielddef *lupb_msg_checkfield(lua_State *L,
const lupb_msgdef *lmd,
int fieldarg) {
const char *fieldname = luaL_checkstring(L, fieldarg);
const upb_fielddef *f = upb_msgdef_ntof(lmd->md, fieldname);
size_t len;
const char *fieldname = luaL_checklstring(L, fieldarg, &len);
const upb_fielddef *f = upb_msgdef_ntof(lmd->md, fieldname, len);
if (!f) {
const char *msg = lua_pushfstring(L, "no such field: %s", fieldname);

@ -1,2 +0,0 @@
This is PROTOTYPE code -- all interfaces are experimental
and will almost certainly change.

@ -0,0 +1,30 @@
# Ruby extension
To build, run (from the top upb directory):
$ make ruby
$ sudo make install
To test, run:
$ make rubytest
The binding currently supports:
- loading message types from descriptors.
- constructing message instances
- reading and writing their members
- parsing and serializing the messages
- all data types (including nested and repeated)
The binding does *not* currently support:
- defining message types directly in Ruby code.
- generating Ruby code for a .proto file.
- type-checking for setters
- homogenous / type-checked arrays
- default values
Because code generation is not currently implemented, the interface to import
a specific message type is kind of clunky for the moment.

@ -1,9 +1,13 @@
#!/usr/bin/ruby
require 'mkmf'
# Extra args are passed on the command-line.
$CFLAGS += (" " + ARGV[0])
find_header("upb/upb.h", "../../..") or raise "Can't find upb headers"
find_library("upb_pic", "upb_msgdef_new", "../../../lib") or raise "Can't find upb lib"
find_library("upb.pb_pic", "upb_decoder_init", "../../../lib") or raise "Can't find upb.pb lib"
find_library("upb.descriptor_pic", "upb_descreader_init", "../../../lib") or raise "Can't find upb.descriptor lib"
$CFLAGS += " -Wall"
find_library("upb.pb_pic", "upb_pbdecoder_init", "../../../lib") or raise "Can't find upb.pb lib"
create_makefile("upb")

File diff suppressed because it is too large Load Diff

@ -628,6 +628,7 @@ upb_fielddef *upb_fielddef_new(const void *owner) {
f->tagdelim = false;
f->is_extension_ = false;
f->lazy_ = false;
f->packed_ = true;
// For the moment we default this to UPB_INTFMT_VARIABLE, since it will work
// with all integer types and is in some since more "default" since the most
@ -735,6 +736,10 @@ bool upb_fielddef_lazy(const upb_fielddef *f) {
return f->lazy_;
}
bool upb_fielddef_packed(const upb_fielddef *f) {
return f->packed_;
}
const char *upb_fielddef_name(const upb_fielddef *f) {
return upb_def_fullname(UPB_UPCAST(f));
}
@ -1030,6 +1035,11 @@ void upb_fielddef_setlazy(upb_fielddef *f, bool lazy) {
f->lazy_ = lazy;
}
void upb_fielddef_setpacked(upb_fielddef *f, bool packed) {
assert(!upb_fielddef_isfrozen(f));
f->packed_ = packed;
}
void upb_fielddef_setlabel(upb_fielddef *f, upb_label_t label) {
assert(!upb_fielddef_isfrozen(f));
assert(upb_fielddef_checklabel(label));
@ -1341,7 +1351,7 @@ bool upb_msgdef_addfield(upb_msgdef *m, upb_fielddef *f, const void *ref_donor,
upb_status_seterrmsg(s, "field name or number were not set");
return false;
} else if(upb_msgdef_itof(m, upb_fielddef_number(f)) ||
upb_msgdef_ntof(m, upb_fielddef_name(f))) {
upb_msgdef_ntofz(m, upb_fielddef_name(f))) {
upb_status_seterrmsg(s, "duplicate field name or number");
return false;
}
@ -1365,20 +1375,13 @@ const upb_fielddef *upb_msgdef_itof(const upb_msgdef *m, uint32_t i) {
upb_value_getptr(val) : NULL;
}
const upb_fielddef *upb_msgdef_ntof(const upb_msgdef *m, const char *name) {
const upb_fielddef *upb_msgdef_ntof(const upb_msgdef *m, const char *name,
size_t len) {
upb_value val;
return upb_strtable_lookup(&m->ntof, name, &val) ?
return upb_strtable_lookup2(&m->ntof, name, len, &val) ?
upb_value_getptr(val) : NULL;
}
upb_fielddef *upb_msgdef_itof_mutable(upb_msgdef *m, uint32_t i) {
return (upb_fielddef*)upb_msgdef_itof(m, i);
}
upb_fielddef *upb_msgdef_ntof_mutable(upb_msgdef *m, const char *name) {
return (upb_fielddef*)upb_msgdef_ntof(m, name);
}
int upb_msgdef_numfields(const upb_msgdef *m) {
return upb_strtable_count(&m->ntof);
}

@ -324,6 +324,13 @@ UPB_DEFINE_DEF(upb::FieldDef, fielddef, FIELD,
// contain both regular FieldOptions like "lazy" *and* custom options).
bool lazy() const;
// For non-string, non-submessage fields, this indicates whether binary
// protobufs are encoded in packed or non-packed format.
//
// TODO(haberman): see note above about putting options like this into a
// FieldOptions container.
bool packed() const;
// An integer that can be used as an index into an array of fields for
// whatever message this field belongs to. Guaranteed to be less than
// f->containing_type()->field_count(). May only be accessed once the def has
@ -430,11 +437,14 @@ UPB_DEFINE_DEF(upb::FieldDef, fielddef, FIELD,
bool set_containing_type_name(const char *name, Status* status);
bool set_containing_type_name(const std::string& name, Status* status);
// When we freeze, we ensure that this can only be true for length-delimited
// message fields. Prior to freezing this can be true or false with no
// restrictions.
// Defaults to false. When we freeze, we ensure that this can only be true
// for length-delimited message fields. Prior to freezing this can be true or
// false with no restrictions.
void set_lazy(bool lazy);
// Defaults to true. Sets whether this field is encoded in packed format.
void set_packed(bool packed);
// "type" or "descriptor_type" MUST be set explicitly before the fielddef is
// finalized. These setters require that the enum value is valid; if the
// value did not come directly from an enum constant, the caller should
@ -515,6 +525,7 @@ UPB_DEFINE_STRUCT(upb_fielddef, upb_def,
bool type_is_set_; // False until type is explicitly set.
bool is_extension_;
bool lazy_;
bool packed_;
upb_intfmt_t intfmt;
bool tagdelim;
upb_fieldtype_t type_;
@ -525,13 +536,13 @@ UPB_DEFINE_STRUCT(upb_fielddef, upb_def,
));
#define UPB_FIELDDEF_INIT(label, type, intfmt, tagdelim, is_extension, lazy, \
name, num, msgdef, subdef, selector_base, index, \
defaultval, refs, ref2s) \
packed, name, num, msgdef, subdef, selector_base, \
index, defaultval, refs, ref2s) \
{ \
UPB_DEF_INIT(name, UPB_DEF_FIELD, refs, ref2s), defaultval, {msgdef}, \
{subdef}, false, false, \
type == UPB_TYPE_STRING || type == UPB_TYPE_BYTES, true, is_extension, \
lazy, intfmt, tagdelim, type, label, num, selector_base, index \
lazy, packed, intfmt, tagdelim, type, label, num, selector_base, index \
}
UPB_BEGIN_EXTERN_C // {
@ -561,6 +572,7 @@ uint32_t upb_fielddef_number(const upb_fielddef *f);
const char *upb_fielddef_name(const upb_fielddef *f);
bool upb_fielddef_isextension(const upb_fielddef *f);
bool upb_fielddef_lazy(const upb_fielddef *f);
bool upb_fielddef_packed(const upb_fielddef *f);
const upb_msgdef *upb_fielddef_containingtype(const upb_fielddef *f);
upb_msgdef *upb_fielddef_containingtype_mutable(upb_fielddef *f);
const char *upb_fielddef_containingtypename(upb_fielddef *f);
@ -596,6 +608,7 @@ bool upb_fielddef_setcontainingtypename(upb_fielddef *f, const char *name,
upb_status *s);
void upb_fielddef_setisextension(upb_fielddef *f, bool is_extension);
void upb_fielddef_setlazy(upb_fielddef *f, bool lazy);
void upb_fielddef_setpacked(upb_fielddef *f, bool packed);
void upb_fielddef_setintfmt(upb_fielddef *f, upb_intfmt_t fmt);
void upb_fielddef_settagdelim(upb_fielddef *f, bool tag_delim);
void upb_fielddef_setdefaultint64(upb_fielddef *f, int64_t val);
@ -668,9 +681,26 @@ UPB_DEFINE_DEF(upb::MessageDef, msgdef, MSG, UPB_QUOTE(
// These return NULL if the field is not found.
FieldDef* FindFieldByNumber(uint32_t number);
FieldDef* FindFieldByName(const char *name);
FieldDef* FindFieldByName(const char *name, size_t len);
const FieldDef* FindFieldByNumber(uint32_t number) const;
const FieldDef* FindFieldByName(const char* name) const;
const FieldDef* FindFieldByName(const char* name, size_t len) const;
FieldDef* FindFieldByName(const char *name) {
return FindFieldByName(name, strlen(name));
}
const FieldDef* FindFieldByName(const char *name) const {
return FindFieldByName(name, strlen(name));
}
template <class T>
FieldDef* FindFieldByName(const T& str) {
return FindFieldByName(str.c_str(), str.size());
}
template <class T>
const FieldDef* FindFieldByName(const T& str) const {
return FindFieldByName(str.c_str(), str.size());
}
// Returns a new msgdef that is a copy of the given msgdef (and a copy of all
// the fields) but with any references to submessages broken and replaced
@ -759,12 +789,30 @@ bool upb_msgdef_setfullname(upb_msgdef *m, const char *fullname, upb_status *s);
upb_msgdef *upb_msgdef_dup(const upb_msgdef *m, const void *owner);
bool upb_msgdef_addfield(upb_msgdef *m, upb_fielddef *f, const void *ref_donor,
upb_status *s);
// Field lookup in a couple of different variations:
// - itof = int to field
// - ntof = name to field
// - ntofz = name to field, null-terminated string.
const upb_fielddef *upb_msgdef_itof(const upb_msgdef *m, uint32_t i);
const upb_fielddef *upb_msgdef_ntof(const upb_msgdef *m, const char *name);
upb_fielddef *upb_msgdef_itof_mutable(upb_msgdef *m, uint32_t i);
upb_fielddef *upb_msgdef_ntof_mutable(upb_msgdef *m, const char *name);
const upb_fielddef *upb_msgdef_ntof(const upb_msgdef *m, const char *name,
size_t len);
int upb_msgdef_numfields(const upb_msgdef *m);
UPB_INLINE const upb_fielddef *upb_msgdef_ntofz(const upb_msgdef *m,
const char *name) {
return upb_msgdef_ntof(m, name, strlen(name));
}
UPB_INLINE upb_fielddef *upb_msgdef_itof_mutable(upb_msgdef *m, uint32_t i) {
return (upb_fielddef*)upb_msgdef_itof(m, i);
}
UPB_INLINE upb_fielddef *upb_msgdef_ntof_mutable(upb_msgdef *m,
const char *name, size_t len) {
return (upb_fielddef *)upb_msgdef_ntof(m, name, len);
}
// upb_msg_iter i;
// for(upb_msg_begin(&i, m); !upb_msg_done(&i); upb_msg_next(&i)) {
// upb_fielddef *f = upb_msg_iter_field(&i);
@ -1027,6 +1075,12 @@ inline bool FieldDef::lazy() const {
inline void FieldDef::set_lazy(bool lazy) {
upb_fielddef_setlazy(this, lazy);
}
inline bool FieldDef::packed() const {
return upb_fielddef_packed(this);
}
inline void FieldDef::set_packed(bool packed) {
upb_fielddef_setpacked(this, packed);
}
inline const MessageDef* FieldDef::containing_type() const {
return upb_fielddef_containingtype(this);
}
@ -1189,14 +1243,15 @@ inline bool MessageDef::AddField(const reffed_ptr<FieldDef>& f, Status* s) {
inline FieldDef* MessageDef::FindFieldByNumber(uint32_t number) {
return upb_msgdef_itof_mutable(this, number);
}
inline FieldDef* MessageDef::FindFieldByName(const char* name) {
return upb_msgdef_ntof_mutable(this, name);
inline FieldDef* MessageDef::FindFieldByName(const char* name, size_t len) {
return upb_msgdef_ntof_mutable(this, name, len);
}
inline const FieldDef* MessageDef::FindFieldByNumber(uint32_t number) const {
return upb_msgdef_itof(this, number);
}
inline const FieldDef* MessageDef::FindFieldByName(const char* name) const {
return upb_msgdef_ntof(this, name);
inline const FieldDef *MessageDef::FindFieldByName(const char *name,
size_t len) const {
return upb_msgdef_ntof(this, name, len);
}
inline MessageDef* MessageDef::Dup(const void *owner) const {
return upb_msgdef_dup(this, owner);

@ -40,87 +40,87 @@ static const upb_msgdef msgs[20] = {
};
static const upb_fielddef fields[81] = {
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, "aggregate_value", 8, &msgs[18], NULL, 15, 6, {0},&reftables[40], &reftables[41]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BOOL, 0, false, false, false, "allow_alias", 2, &msgs[3], NULL, 6, 1, {0},&reftables[42], &reftables[43]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BOOL, 0, false, false, false, "cc_generic_services", 16, &msgs[10], NULL, 17, 6, {0},&reftables[44], &reftables[45]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_ENUM, 0, false, false, false, "ctype", 1, &msgs[7], UPB_UPCAST(&enums[2]), 6, 1, {0},&reftables[46], &reftables[47]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, "default_value", 7, &msgs[6], NULL, 16, 7, {0},&reftables[48], &reftables[49]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_STRING, 0, false, false, false, "dependency", 3, &msgs[8], NULL, 30, 8, {0},&reftables[50], &reftables[51]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BOOL, 0, false, false, false, "deprecated", 3, &msgs[7], NULL, 8, 3, {0},&reftables[52], &reftables[53]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_DOUBLE, 0, false, false, false, "double_value", 6, &msgs[18], NULL, 11, 4, {0},&reftables[54], &reftables[55]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_INT32, UPB_INTFMT_VARIABLE, false, false, false, "end", 2, &msgs[1], NULL, 3, 1, {0},&reftables[56], &reftables[57]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, "enum_type", 4, &msgs[0], UPB_UPCAST(&msgs[2]), 16, 2, {0},&reftables[58], &reftables[59]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, "enum_type", 5, &msgs[8], UPB_UPCAST(&msgs[2]), 13, 1, {0},&reftables[60], &reftables[61]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, "experimental_map_key", 9, &msgs[7], NULL, 10, 5, {0},&reftables[62], &reftables[63]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, "extendee", 2, &msgs[6], NULL, 7, 2, {0},&reftables[64], &reftables[65]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, "extension", 7, &msgs[8], UPB_UPCAST(&msgs[6]), 19, 3, {0},&reftables[66], &reftables[67]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, "extension", 6, &msgs[0], UPB_UPCAST(&msgs[6]), 22, 4, {0},&reftables[68], &reftables[69]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, "extension_range", 5, &msgs[0], UPB_UPCAST(&msgs[1]), 19, 3, {0},&reftables[70], &reftables[71]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, "field", 2, &msgs[0], UPB_UPCAST(&msgs[6]), 10, 0, {0},&reftables[72], &reftables[73]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, "file", 1, &msgs[9], UPB_UPCAST(&msgs[8]), 5, 0, {0},&reftables[74], &reftables[75]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, "go_package", 11, &msgs[10], NULL, 14, 5, {0},&reftables[76], &reftables[77]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, "identifier_value", 3, &msgs[18], NULL, 6, 1, {0},&reftables[78], &reftables[79]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, "input_type", 2, &msgs[12], NULL, 7, 2, {0},&reftables[80], &reftables[81]),
UPB_FIELDDEF_INIT(UPB_LABEL_REQUIRED, UPB_TYPE_BOOL, 0, false, false, false, "is_extension", 2, &msgs[19], NULL, 5, 1, {0},&reftables[82], &reftables[83]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BOOL, 0, false, false, false, "java_generate_equals_and_hash", 20, &msgs[10], NULL, 20, 9, {0},&reftables[84], &reftables[85]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BOOL, 0, false, false, false, "java_generic_services", 17, &msgs[10], NULL, 18, 7, {0},&reftables[86], &reftables[87]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BOOL, 0, false, false, false, "java_multiple_files", 10, &msgs[10], NULL, 13, 4, {0},&reftables[88], &reftables[89]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, "java_outer_classname", 8, &msgs[10], NULL, 9, 2, {0},&reftables[90], &reftables[91]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, "java_package", 1, &msgs[10], NULL, 6, 1, {0},&reftables[92], &reftables[93]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_ENUM, 0, false, false, false, "label", 4, &msgs[6], UPB_UPCAST(&enums[0]), 11, 4, {0},&reftables[94], &reftables[95]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BOOL, 0, false, false, false, "lazy", 5, &msgs[7], NULL, 9, 4, {0},&reftables[96], &reftables[97]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, "leading_comments", 3, &msgs[17], NULL, 8, 2, {0},&reftables[98], &reftables[99]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, "location", 1, &msgs[16], UPB_UPCAST(&msgs[17]), 5, 0, {0},&reftables[100], &reftables[101]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BOOL, 0, false, false, false, "message_set_wire_format", 1, &msgs[11], NULL, 6, 1, {0},&reftables[102], &reftables[103]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, "message_type", 4, &msgs[8], UPB_UPCAST(&msgs[0]), 10, 0, {0},&reftables[104], &reftables[105]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, "method", 2, &msgs[14], UPB_UPCAST(&msgs[12]), 6, 0, {0},&reftables[106], &reftables[107]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, "name", 1, &msgs[8], NULL, 22, 6, {0},&reftables[108], &reftables[109]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, "name", 1, &msgs[14], NULL, 8, 2, {0},&reftables[110], &reftables[111]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, "name", 2, &msgs[18], UPB_UPCAST(&msgs[19]), 5, 0, {0},&reftables[112], &reftables[113]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, "name", 1, &msgs[4], NULL, 4, 1, {0},&reftables[114], &reftables[115]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, "name", 1, &msgs[0], NULL, 24, 6, {0},&reftables[116], &reftables[117]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, "name", 1, &msgs[12], NULL, 4, 1, {0},&reftables[118], &reftables[119]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, "name", 1, &msgs[2], NULL, 8, 2, {0},&reftables[120], &reftables[121]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, "name", 1, &msgs[6], NULL, 4, 1, {0},&reftables[122], &reftables[123]),
UPB_FIELDDEF_INIT(UPB_LABEL_REQUIRED, UPB_TYPE_STRING, 0, false, false, false, "name_part", 1, &msgs[19], NULL, 2, 0, {0},&reftables[124], &reftables[125]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_INT64, UPB_INTFMT_VARIABLE, false, false, false, "negative_int_value", 5, &msgs[18], NULL, 10, 3, {0},&reftables[126], &reftables[127]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, "nested_type", 3, &msgs[0], UPB_UPCAST(&msgs[0]), 13, 1, {0},&reftables[128], &reftables[129]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BOOL, 0, false, false, false, "no_standard_descriptor_accessor", 2, &msgs[11], NULL, 7, 2, {0},&reftables[130], &reftables[131]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_INT32, UPB_INTFMT_VARIABLE, false, false, false, "number", 3, &msgs[6], NULL, 10, 3, {0},&reftables[132], &reftables[133]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_INT32, UPB_INTFMT_VARIABLE, false, false, false, "number", 2, &msgs[4], NULL, 7, 2, {0},&reftables[134], &reftables[135]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_ENUM, 0, false, false, false, "optimize_for", 9, &msgs[10], UPB_UPCAST(&enums[3]), 12, 3, {0},&reftables[136], &reftables[137]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_MESSAGE, 0, false, false, false, "options", 7, &msgs[0], UPB_UPCAST(&msgs[11]), 23, 5, {0},&reftables[138], &reftables[139]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_MESSAGE, 0, false, false, false, "options", 3, &msgs[2], UPB_UPCAST(&msgs[3]), 7, 1, {0},&reftables[140], &reftables[141]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_MESSAGE, 0, false, false, false, "options", 8, &msgs[6], UPB_UPCAST(&msgs[7]), 3, 0, {0},&reftables[142], &reftables[143]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_MESSAGE, 0, false, false, false, "options", 3, &msgs[4], UPB_UPCAST(&msgs[5]), 3, 0, {0},&reftables[144], &reftables[145]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_MESSAGE, 0, false, false, false, "options", 8, &msgs[8], UPB_UPCAST(&msgs[10]), 20, 4, {0},&reftables[146], &reftables[147]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_MESSAGE, 0, false, false, false, "options", 3, &msgs[14], UPB_UPCAST(&msgs[15]), 7, 1, {0},&reftables[148], &reftables[149]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_MESSAGE, 0, false, false, false, "options", 4, &msgs[12], UPB_UPCAST(&msgs[13]), 3, 0, {0},&reftables[150], &reftables[151]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, "output_type", 3, &msgs[12], NULL, 10, 3, {0},&reftables[152], &reftables[153]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, "package", 2, &msgs[8], NULL, 25, 7, {0},&reftables[154], &reftables[155]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BOOL, 0, false, false, false, "packed", 2, &msgs[7], NULL, 7, 2, {0},&reftables[156], &reftables[157]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_INT32, UPB_INTFMT_VARIABLE, false, false, false, "path", 1, &msgs[17], NULL, 4, 0, {0},&reftables[158], &reftables[159]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_UINT64, UPB_INTFMT_VARIABLE, false, false, false, "positive_int_value", 4, &msgs[18], NULL, 9, 2, {0},&reftables[160], &reftables[161]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_INT32, UPB_INTFMT_VARIABLE, false, false, false, "public_dependency", 10, &msgs[8], NULL, 35, 9, {0},&reftables[162], &reftables[163]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BOOL, 0, false, false, false, "py_generic_services", 18, &msgs[10], NULL, 19, 8, {0},&reftables[164], &reftables[165]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, "service", 6, &msgs[8], UPB_UPCAST(&msgs[14]), 16, 2, {0},&reftables[166], &reftables[167]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_MESSAGE, 0, false, false, false, "source_code_info", 9, &msgs[8], UPB_UPCAST(&msgs[16]), 21, 5, {0},&reftables[168], &reftables[169]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_INT32, UPB_INTFMT_VARIABLE, false, false, false, "span", 2, &msgs[17], NULL, 7, 1, {0},&reftables[170], &reftables[171]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_INT32, UPB_INTFMT_VARIABLE, false, false, false, "start", 1, &msgs[1], NULL, 2, 0, {0},&reftables[172], &reftables[173]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BYTES, 0, false, false, false, "string_value", 7, &msgs[18], NULL, 12, 5, {0},&reftables[174], &reftables[175]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, "trailing_comments", 4, &msgs[17], NULL, 11, 3, {0},&reftables[176], &reftables[177]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_ENUM, 0, false, false, false, "type", 5, &msgs[6], UPB_UPCAST(&enums[1]), 12, 5, {0},&reftables[178], &reftables[179]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, "type_name", 6, &msgs[6], NULL, 13, 6, {0},&reftables[180], &reftables[181]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, "uninterpreted_option", 999, &msgs[5], UPB_UPCAST(&msgs[18]), 5, 0, {0},&reftables[182], &reftables[183]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, "uninterpreted_option", 999, &msgs[15], UPB_UPCAST(&msgs[18]), 5, 0, {0},&reftables[184], &reftables[185]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, "uninterpreted_option", 999, &msgs[3], UPB_UPCAST(&msgs[18]), 5, 0, {0},&reftables[186], &reftables[187]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, "uninterpreted_option", 999, &msgs[13], UPB_UPCAST(&msgs[18]), 5, 0, {0},&reftables[188], &reftables[189]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, "uninterpreted_option", 999, &msgs[10], UPB_UPCAST(&msgs[18]), 5, 0, {0},&reftables[190], &reftables[191]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, "uninterpreted_option", 999, &msgs[11], UPB_UPCAST(&msgs[18]), 5, 0, {0},&reftables[192], &reftables[193]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, "uninterpreted_option", 999, &msgs[7], UPB_UPCAST(&msgs[18]), 5, 0, {0},&reftables[194], &reftables[195]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, "value", 2, &msgs[2], UPB_UPCAST(&msgs[4]), 6, 0, {0},&reftables[196], &reftables[197]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BOOL, 0, false, false, false, "weak", 10, &msgs[7], NULL, 13, 6, {0},&reftables[198], &reftables[199]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_INT32, UPB_INTFMT_VARIABLE, false, false, false, "weak_dependency", 11, &msgs[8], NULL, 38, 10, {0},&reftables[200], &reftables[201]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, false, "aggregate_value", 8, &msgs[18], NULL, 15, 6, {0},&reftables[40], &reftables[41]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BOOL, 0, false, false, false, false, "allow_alias", 2, &msgs[3], NULL, 6, 1, {0},&reftables[42], &reftables[43]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BOOL, 0, false, false, false, false, "cc_generic_services", 16, &msgs[10], NULL, 17, 6, {0},&reftables[44], &reftables[45]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_ENUM, 0, false, false, false, false, "ctype", 1, &msgs[7], UPB_UPCAST(&enums[2]), 6, 1, {0},&reftables[46], &reftables[47]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, false, "default_value", 7, &msgs[6], NULL, 16, 7, {0},&reftables[48], &reftables[49]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_STRING, 0, false, false, false, false, "dependency", 3, &msgs[8], NULL, 30, 8, {0},&reftables[50], &reftables[51]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BOOL, 0, false, false, false, false, "deprecated", 3, &msgs[7], NULL, 8, 3, {0},&reftables[52], &reftables[53]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_DOUBLE, 0, false, false, false, false, "double_value", 6, &msgs[18], NULL, 11, 4, {0},&reftables[54], &reftables[55]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_INT32, UPB_INTFMT_VARIABLE, false, false, false, false, "end", 2, &msgs[1], NULL, 3, 1, {0},&reftables[56], &reftables[57]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, false, "enum_type", 4, &msgs[0], UPB_UPCAST(&msgs[2]), 16, 2, {0},&reftables[58], &reftables[59]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, false, "enum_type", 5, &msgs[8], UPB_UPCAST(&msgs[2]), 13, 1, {0},&reftables[60], &reftables[61]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, false, "experimental_map_key", 9, &msgs[7], NULL, 10, 5, {0},&reftables[62], &reftables[63]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, false, "extendee", 2, &msgs[6], NULL, 7, 2, {0},&reftables[64], &reftables[65]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, false, "extension", 7, &msgs[8], UPB_UPCAST(&msgs[6]), 19, 3, {0},&reftables[66], &reftables[67]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, false, "extension", 6, &msgs[0], UPB_UPCAST(&msgs[6]), 22, 4, {0},&reftables[68], &reftables[69]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, false, "extension_range", 5, &msgs[0], UPB_UPCAST(&msgs[1]), 19, 3, {0},&reftables[70], &reftables[71]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, false, "field", 2, &msgs[0], UPB_UPCAST(&msgs[6]), 10, 0, {0},&reftables[72], &reftables[73]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, false, "file", 1, &msgs[9], UPB_UPCAST(&msgs[8]), 5, 0, {0},&reftables[74], &reftables[75]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, false, "go_package", 11, &msgs[10], NULL, 14, 5, {0},&reftables[76], &reftables[77]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, false, "identifier_value", 3, &msgs[18], NULL, 6, 1, {0},&reftables[78], &reftables[79]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, false, "input_type", 2, &msgs[12], NULL, 7, 2, {0},&reftables[80], &reftables[81]),
UPB_FIELDDEF_INIT(UPB_LABEL_REQUIRED, UPB_TYPE_BOOL, 0, false, false, false, false, "is_extension", 2, &msgs[19], NULL, 5, 1, {0},&reftables[82], &reftables[83]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BOOL, 0, false, false, false, false, "java_generate_equals_and_hash", 20, &msgs[10], NULL, 20, 9, {0},&reftables[84], &reftables[85]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BOOL, 0, false, false, false, false, "java_generic_services", 17, &msgs[10], NULL, 18, 7, {0},&reftables[86], &reftables[87]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BOOL, 0, false, false, false, false, "java_multiple_files", 10, &msgs[10], NULL, 13, 4, {0},&reftables[88], &reftables[89]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, false, "java_outer_classname", 8, &msgs[10], NULL, 9, 2, {0},&reftables[90], &reftables[91]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, false, "java_package", 1, &msgs[10], NULL, 6, 1, {0},&reftables[92], &reftables[93]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_ENUM, 0, false, false, false, false, "label", 4, &msgs[6], UPB_UPCAST(&enums[0]), 11, 4, {0},&reftables[94], &reftables[95]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BOOL, 0, false, false, false, false, "lazy", 5, &msgs[7], NULL, 9, 4, {0},&reftables[96], &reftables[97]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, false, "leading_comments", 3, &msgs[17], NULL, 8, 2, {0},&reftables[98], &reftables[99]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, false, "location", 1, &msgs[16], UPB_UPCAST(&msgs[17]), 5, 0, {0},&reftables[100], &reftables[101]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BOOL, 0, false, false, false, false, "message_set_wire_format", 1, &msgs[11], NULL, 6, 1, {0},&reftables[102], &reftables[103]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, false, "message_type", 4, &msgs[8], UPB_UPCAST(&msgs[0]), 10, 0, {0},&reftables[104], &reftables[105]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, false, "method", 2, &msgs[14], UPB_UPCAST(&msgs[12]), 6, 0, {0},&reftables[106], &reftables[107]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, false, "name", 1, &msgs[8], NULL, 22, 6, {0},&reftables[108], &reftables[109]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, false, "name", 1, &msgs[14], NULL, 8, 2, {0},&reftables[110], &reftables[111]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, false, "name", 2, &msgs[18], UPB_UPCAST(&msgs[19]), 5, 0, {0},&reftables[112], &reftables[113]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, false, "name", 1, &msgs[4], NULL, 4, 1, {0},&reftables[114], &reftables[115]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, false, "name", 1, &msgs[0], NULL, 24, 6, {0},&reftables[116], &reftables[117]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, false, "name", 1, &msgs[12], NULL, 4, 1, {0},&reftables[118], &reftables[119]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, false, "name", 1, &msgs[2], NULL, 8, 2, {0},&reftables[120], &reftables[121]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, false, "name", 1, &msgs[6], NULL, 4, 1, {0},&reftables[122], &reftables[123]),
UPB_FIELDDEF_INIT(UPB_LABEL_REQUIRED, UPB_TYPE_STRING, 0, false, false, false, false, "name_part", 1, &msgs[19], NULL, 2, 0, {0},&reftables[124], &reftables[125]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_INT64, UPB_INTFMT_VARIABLE, false, false, false, false, "negative_int_value", 5, &msgs[18], NULL, 10, 3, {0},&reftables[126], &reftables[127]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, false, "nested_type", 3, &msgs[0], UPB_UPCAST(&msgs[0]), 13, 1, {0},&reftables[128], &reftables[129]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BOOL, 0, false, false, false, false, "no_standard_descriptor_accessor", 2, &msgs[11], NULL, 7, 2, {0},&reftables[130], &reftables[131]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_INT32, UPB_INTFMT_VARIABLE, false, false, false, false, "number", 3, &msgs[6], NULL, 10, 3, {0},&reftables[132], &reftables[133]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_INT32, UPB_INTFMT_VARIABLE, false, false, false, false, "number", 2, &msgs[4], NULL, 7, 2, {0},&reftables[134], &reftables[135]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_ENUM, 0, false, false, false, false, "optimize_for", 9, &msgs[10], UPB_UPCAST(&enums[3]), 12, 3, {0},&reftables[136], &reftables[137]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_MESSAGE, 0, false, false, false, false, "options", 7, &msgs[0], UPB_UPCAST(&msgs[11]), 23, 5, {0},&reftables[138], &reftables[139]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_MESSAGE, 0, false, false, false, false, "options", 3, &msgs[2], UPB_UPCAST(&msgs[3]), 7, 1, {0},&reftables[140], &reftables[141]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_MESSAGE, 0, false, false, false, false, "options", 8, &msgs[6], UPB_UPCAST(&msgs[7]), 3, 0, {0},&reftables[142], &reftables[143]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_MESSAGE, 0, false, false, false, false, "options", 3, &msgs[4], UPB_UPCAST(&msgs[5]), 3, 0, {0},&reftables[144], &reftables[145]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_MESSAGE, 0, false, false, false, false, "options", 8, &msgs[8], UPB_UPCAST(&msgs[10]), 20, 4, {0},&reftables[146], &reftables[147]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_MESSAGE, 0, false, false, false, false, "options", 3, &msgs[14], UPB_UPCAST(&msgs[15]), 7, 1, {0},&reftables[148], &reftables[149]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_MESSAGE, 0, false, false, false, false, "options", 4, &msgs[12], UPB_UPCAST(&msgs[13]), 3, 0, {0},&reftables[150], &reftables[151]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, false, "output_type", 3, &msgs[12], NULL, 10, 3, {0},&reftables[152], &reftables[153]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, false, "package", 2, &msgs[8], NULL, 25, 7, {0},&reftables[154], &reftables[155]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BOOL, 0, false, false, false, false, "packed", 2, &msgs[7], NULL, 7, 2, {0},&reftables[156], &reftables[157]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_INT32, UPB_INTFMT_VARIABLE, false, false, false, true, "path", 1, &msgs[17], NULL, 4, 0, {0},&reftables[158], &reftables[159]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_UINT64, UPB_INTFMT_VARIABLE, false, false, false, false, "positive_int_value", 4, &msgs[18], NULL, 9, 2, {0},&reftables[160], &reftables[161]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_INT32, UPB_INTFMT_VARIABLE, false, false, false, false, "public_dependency", 10, &msgs[8], NULL, 35, 9, {0},&reftables[162], &reftables[163]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BOOL, 0, false, false, false, false, "py_generic_services", 18, &msgs[10], NULL, 19, 8, {0},&reftables[164], &reftables[165]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, false, "service", 6, &msgs[8], UPB_UPCAST(&msgs[14]), 16, 2, {0},&reftables[166], &reftables[167]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_MESSAGE, 0, false, false, false, false, "source_code_info", 9, &msgs[8], UPB_UPCAST(&msgs[16]), 21, 5, {0},&reftables[168], &reftables[169]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_INT32, UPB_INTFMT_VARIABLE, false, false, false, true, "span", 2, &msgs[17], NULL, 7, 1, {0},&reftables[170], &reftables[171]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_INT32, UPB_INTFMT_VARIABLE, false, false, false, false, "start", 1, &msgs[1], NULL, 2, 0, {0},&reftables[172], &reftables[173]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BYTES, 0, false, false, false, false, "string_value", 7, &msgs[18], NULL, 12, 5, {0},&reftables[174], &reftables[175]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, false, "trailing_comments", 4, &msgs[17], NULL, 11, 3, {0},&reftables[176], &reftables[177]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_ENUM, 0, false, false, false, false, "type", 5, &msgs[6], UPB_UPCAST(&enums[1]), 12, 5, {0},&reftables[178], &reftables[179]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_STRING, 0, false, false, false, false, "type_name", 6, &msgs[6], NULL, 13, 6, {0},&reftables[180], &reftables[181]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, false, "uninterpreted_option", 999, &msgs[5], UPB_UPCAST(&msgs[18]), 5, 0, {0},&reftables[182], &reftables[183]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, false, "uninterpreted_option", 999, &msgs[15], UPB_UPCAST(&msgs[18]), 5, 0, {0},&reftables[184], &reftables[185]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, false, "uninterpreted_option", 999, &msgs[3], UPB_UPCAST(&msgs[18]), 5, 0, {0},&reftables[186], &reftables[187]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, false, "uninterpreted_option", 999, &msgs[13], UPB_UPCAST(&msgs[18]), 5, 0, {0},&reftables[188], &reftables[189]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, false, "uninterpreted_option", 999, &msgs[10], UPB_UPCAST(&msgs[18]), 5, 0, {0},&reftables[190], &reftables[191]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, false, "uninterpreted_option", 999, &msgs[11], UPB_UPCAST(&msgs[18]), 5, 0, {0},&reftables[192], &reftables[193]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, false, "uninterpreted_option", 999, &msgs[7], UPB_UPCAST(&msgs[18]), 5, 0, {0},&reftables[194], &reftables[195]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_MESSAGE, 0, false, false, false, false, "value", 2, &msgs[2], UPB_UPCAST(&msgs[4]), 6, 0, {0},&reftables[196], &reftables[197]),
UPB_FIELDDEF_INIT(UPB_LABEL_OPTIONAL, UPB_TYPE_BOOL, 0, false, false, false, false, "weak", 10, &msgs[7], NULL, 13, 6, {0},&reftables[198], &reftables[199]),
UPB_FIELDDEF_INIT(UPB_LABEL_REPEATED, UPB_TYPE_INT32, UPB_INTFMT_VARIABLE, false, false, false, false, "weak_dependency", 11, &msgs[8], NULL, 38, 10, {0},&reftables[200], &reftables[201]),
};
static const upb_enumdef enums[4] = {

@ -272,6 +272,9 @@ static bool field_startmsg(void *closure, const void *hd) {
r->f = upb_fielddef_new(&r->defs);
free(r->default_string);
r->default_string = NULL;
// fielddefs default to packed, but descriptors default to non-packed.
upb_fielddef_setpacked(r->f, false);
return true;
}
@ -378,6 +381,13 @@ static bool field_onlazy(void *closure, const void *hd, bool val) {
return true;
}
static bool field_onpacked(void *closure, const void *hd, bool val) {
UPB_UNUSED(hd);
upb_descreader *r = closure;
upb_fielddef_setpacked(r->f, val);
return true;
}
static bool field_ontype(void *closure, const void *hd, int32_t val) {
UPB_UNUSED(hd);
upb_descreader *r = closure;
@ -552,6 +562,7 @@ static void reghandlers(const void *closure, upb_handlers *h) {
&field_ondefaultval, NULL);
} else if (m == D(FieldOptions)) {
upb_handlers_setbool(h, D(FieldOptions_lazy), &field_onlazy, NULL);
upb_handlers_setbool(h, D(FieldOptions_packed), &field_onpacked, NULL);
}
}

@ -149,7 +149,7 @@ const upb_pbdecodermethod *upb_pbdecodermethod_new(
}
/* compiler *******************************************************************/
/* bytecode compiler **********************************************************/
// Data used only at compilation time.
typedef struct {
@ -575,8 +575,8 @@ static void putsel(compiler *c, opcode op, upb_selector_t sel,
// Puts an opcode to call a callback, but only if a callback actually exists for
// this field and handler type.
static void putcb(compiler *c, opcode op, const upb_handlers *h,
const upb_fielddef *f, upb_handlertype_t type) {
static void maybeput(compiler *c, opcode op, const upb_handlers *h,
const upb_fielddef *f, upb_handlertype_t type) {
putsel(c, op, getsel(f, type), h);
}
@ -589,40 +589,165 @@ static bool haslazyhandlers(const upb_handlers *h, const upb_fielddef *f) {
upb_handlers_gethandler(h, getsel(f, UPB_HANDLER_ENDSTR));
}
/* bytecode compiler code generation ******************************************/
// Symbolic names for our local labels.
#define LABEL_LOOPSTART 1 // Top of a repeated field loop.
#define LABEL_LOOPBREAK 2 // To jump out of a repeated loop
#define LABEL_FIELD 3 // Jump backward to find the most recent field.
#define LABEL_ENDMSG 4 // To reach the OP_ENDMSG instr for this msg.
// Generates bytecode to parse a single non-lazy message field.
static void generate_msgfield(compiler *c, const upb_fielddef *f,
upb_pbdecodermethod *method) {
const upb_handlers *h = upb_pbdecodermethod_desthandlers(method);
const upb_pbdecodermethod *sub_m = find_submethod(c, method, f);
if (!sub_m) {
// Don't emit any code for this field at all; it will be parsed as an
// unknown field.
return;
}
label(c, LABEL_FIELD);
int wire_type =
(upb_fielddef_descriptortype(f) == UPB_DESCRIPTOR_TYPE_MESSAGE)
? UPB_WIRE_TYPE_DELIMITED
: UPB_WIRE_TYPE_START_GROUP;
if (upb_fielddef_isseq(f)) {
putop(c, OP_CHECKDELIM, LABEL_ENDMSG);
putchecktag(c, f, wire_type, LABEL_DISPATCH);
dispatchtarget(c, method, f, wire_type);
putop(c, OP_PUSHTAGDELIM, 0);
putop(c, OP_STARTSEQ, getsel(f, UPB_HANDLER_STARTSEQ));
label(c, LABEL_LOOPSTART);
putpush(c, f);
putop(c, OP_STARTSUBMSG, getsel(f, UPB_HANDLER_STARTSUBMSG));
putop(c, OP_CALL, sub_m);
putop(c, OP_POP);
maybeput(c, OP_ENDSUBMSG, h, f, UPB_HANDLER_ENDSUBMSG);
if (wire_type == UPB_WIRE_TYPE_DELIMITED) {
putop(c, OP_SETDELIM);
}
putop(c, OP_CHECKDELIM, LABEL_LOOPBREAK);
putchecktag(c, f, wire_type, LABEL_LOOPBREAK);
putop(c, OP_BRANCH, -LABEL_LOOPSTART);
label(c, LABEL_LOOPBREAK);
putop(c, OP_POP);
maybeput(c, OP_ENDSEQ, h, f, UPB_HANDLER_ENDSEQ);
} else {
putop(c, OP_CHECKDELIM, LABEL_ENDMSG);
putchecktag(c, f, wire_type, LABEL_DISPATCH);
dispatchtarget(c, method, f, wire_type);
putpush(c, f);
putop(c, OP_STARTSUBMSG, getsel(f, UPB_HANDLER_STARTSUBMSG));
putop(c, OP_CALL, sub_m);
putop(c, OP_POP);
maybeput(c, OP_ENDSUBMSG, h, f, UPB_HANDLER_ENDSUBMSG);
if (wire_type == UPB_WIRE_TYPE_DELIMITED) {
putop(c, OP_SETDELIM);
}
}
}
// Generates bytecode to parse a single string or lazy submessage field.
static void generate_delimfield(compiler *c, const upb_fielddef *f,
upb_pbdecodermethod *method) {
const upb_handlers *h = upb_pbdecodermethod_desthandlers(method);
label(c, LABEL_FIELD);
if (upb_fielddef_isseq(f)) {
putop(c, OP_CHECKDELIM, LABEL_ENDMSG);
putchecktag(c, f, UPB_WIRE_TYPE_DELIMITED, LABEL_DISPATCH);
dispatchtarget(c, method, f, UPB_WIRE_TYPE_DELIMITED);
putop(c, OP_PUSHTAGDELIM, 0);
putop(c, OP_STARTSEQ, getsel(f, UPB_HANDLER_STARTSEQ));
label(c, LABEL_LOOPSTART);
putop(c, OP_PUSHLENDELIM);
putop(c, OP_STARTSTR, getsel(f, UPB_HANDLER_STARTSTR));
// Need to emit even if no handler to skip past the string.
putop(c, OP_STRING, getsel(f, UPB_HANDLER_STRING));
putop(c, OP_POP);
maybeput(c, OP_ENDSTR, h, f, UPB_HANDLER_ENDSTR);
putop(c, OP_SETDELIM);
putop(c, OP_CHECKDELIM, LABEL_LOOPBREAK);
putchecktag(c, f, UPB_WIRE_TYPE_DELIMITED, LABEL_LOOPBREAK);
putop(c, OP_BRANCH, -LABEL_LOOPSTART);
label(c, LABEL_LOOPBREAK);
putop(c, OP_POP);
maybeput(c, OP_ENDSEQ, h, f, UPB_HANDLER_ENDSEQ);
} else {
putop(c, OP_CHECKDELIM, LABEL_ENDMSG);
putchecktag(c, f, UPB_WIRE_TYPE_DELIMITED, LABEL_DISPATCH);
dispatchtarget(c, method, f, UPB_WIRE_TYPE_DELIMITED);
putop(c, OP_PUSHLENDELIM);
putop(c, OP_STARTSTR, getsel(f, UPB_HANDLER_STARTSTR));
putop(c, OP_STRING, getsel(f, UPB_HANDLER_STRING));
putop(c, OP_POP);
maybeput(c, OP_ENDSTR, h, f, UPB_HANDLER_ENDSTR);
putop(c, OP_SETDELIM);
}
}
// Generates bytecode to parse a single primitive field.
static void generate_primitivefield(compiler *c, const upb_fielddef *f,
upb_pbdecodermethod *method) {
label(c, LABEL_FIELD);
const upb_handlers *h = upb_pbdecodermethod_desthandlers(method);
upb_descriptortype_t descriptor_type = upb_fielddef_descriptortype(f);
// From a decoding perspective, ENUM is the same as INT32.
if (descriptor_type == UPB_DESCRIPTOR_TYPE_ENUM)
descriptor_type = UPB_DESCRIPTOR_TYPE_INT32;
opcode parse_type = (opcode)descriptor_type;
// TODO(haberman): generate packed or non-packed first depending on "packed"
// setting in the fielddef. This will favor (in speed) whichever was
// specified.
assert((int)parse_type >= 0 && parse_type <= OP_MAX);
upb_selector_t sel = getsel(f, upb_handlers_getprimitivehandlertype(f));
int wire_type = upb_pb_native_wire_types[upb_fielddef_descriptortype(f)];
if (upb_fielddef_isseq(f)) {
putop(c, OP_CHECKDELIM, LABEL_ENDMSG);
putchecktag(c, f, UPB_WIRE_TYPE_DELIMITED, LABEL_DISPATCH);
dispatchtarget(c, method, f, UPB_WIRE_TYPE_DELIMITED);
putop(c, OP_PUSHLENDELIM);
putop(c, OP_STARTSEQ, getsel(f, UPB_HANDLER_STARTSEQ)); // Packed
label(c, LABEL_LOOPSTART);
putop(c, parse_type, sel);
putop(c, OP_CHECKDELIM, LABEL_LOOPBREAK);
putop(c, OP_BRANCH, -LABEL_LOOPSTART);
dispatchtarget(c, method, f, wire_type);
putop(c, OP_PUSHTAGDELIM, 0);
putop(c, OP_STARTSEQ, getsel(f, UPB_HANDLER_STARTSEQ)); // Non-packed
label(c, LABEL_LOOPSTART);
putop(c, parse_type, sel);
putop(c, OP_CHECKDELIM, LABEL_LOOPBREAK);
putchecktag(c, f, wire_type, LABEL_LOOPBREAK);
putop(c, OP_BRANCH, -LABEL_LOOPSTART);
label(c, LABEL_LOOPBREAK);
putop(c, OP_POP); // Packed and non-packed join.
maybeput(c, OP_ENDSEQ, h, f, UPB_HANDLER_ENDSEQ);
putop(c, OP_SETDELIM); // Could remove for non-packed by dup ENDSEQ.
} else {
putop(c, OP_CHECKDELIM, LABEL_ENDMSG);
putchecktag(c, f, wire_type, LABEL_DISPATCH);
dispatchtarget(c, method, f, wire_type);
putop(c, parse_type, sel);
}
}
// Adds bytecode for parsing the given message to the given decoderplan,
// while adding all dispatch targets to this message's dispatch table.
static void compile_method(compiler *c, upb_pbdecodermethod *method) {
assert(method);
// Symbolic names for our local labels.
const int LABEL_LOOPSTART = 1; // Top of a repeated field loop.
const int LABEL_LOOPBREAK = 2; // To jump out of a repeated loop
const int LABEL_FIELD = 3; // Jump backward to find the most recent field.
const int LABEL_ENDMSG = 4; // To reach the OP_ENDMSG instr for this msg.
// Index is descriptor type.
static const uint8_t native_wire_types[] = {
UPB_WIRE_TYPE_END_GROUP, // ENDGROUP
UPB_WIRE_TYPE_64BIT, // DOUBLE
UPB_WIRE_TYPE_32BIT, // FLOAT
UPB_WIRE_TYPE_VARINT, // INT64
UPB_WIRE_TYPE_VARINT, // UINT64
UPB_WIRE_TYPE_VARINT, // INT32
UPB_WIRE_TYPE_64BIT, // FIXED64
UPB_WIRE_TYPE_32BIT, // FIXED32
UPB_WIRE_TYPE_VARINT, // BOOL
UPB_WIRE_TYPE_DELIMITED, // STRING
UPB_WIRE_TYPE_START_GROUP, // GROUP
UPB_WIRE_TYPE_DELIMITED, // MESSAGE
UPB_WIRE_TYPE_DELIMITED, // BYTES
UPB_WIRE_TYPE_VARINT, // UINT32
UPB_WIRE_TYPE_VARINT, // ENUM
UPB_WIRE_TYPE_32BIT, // SFIXED32
UPB_WIRE_TYPE_64BIT, // SFIXED64
UPB_WIRE_TYPE_VARINT, // SINT32
UPB_WIRE_TYPE_VARINT, // SINT64
};
// Clear all entries in the dispatch table.
upb_inttable_uninit(&method->dispatch);
upb_inttable_init(&method->dispatch, UPB_CTYPE_UINT64);
@ -637,128 +762,15 @@ static void compile_method(compiler *c, upb_pbdecodermethod *method) {
upb_msg_iter i;
for(upb_msg_begin(&i, md); !upb_msg_done(&i); upb_msg_next(&i)) {
const upb_fielddef *f = upb_msg_iter_field(&i);
upb_descriptortype_t descriptor_type = upb_fielddef_descriptortype(f);
upb_fieldtype_t type = upb_fielddef_type(f);
// From a decoding perspective, ENUM is the same as INT32.
if (descriptor_type == UPB_DESCRIPTOR_TYPE_ENUM)
descriptor_type = UPB_DESCRIPTOR_TYPE_INT32;
if (type == UPB_TYPE_MESSAGE && !(haslazyhandlers(h, f) && c->lazy)) {
const upb_pbdecodermethod *sub_m = find_submethod(c, method, f);
if (!sub_m) {
// Don't emit any code for this field at all; it will be parsed as an
// unknown field.
continue;
}
label(c, LABEL_FIELD);
int wire_type = (descriptor_type == UPB_DESCRIPTOR_TYPE_MESSAGE)
? UPB_WIRE_TYPE_DELIMITED
: UPB_WIRE_TYPE_START_GROUP;
if (upb_fielddef_isseq(f)) {
putop(c, OP_CHECKDELIM, LABEL_ENDMSG);
putchecktag(c, f, wire_type, LABEL_DISPATCH);
dispatchtarget(c, method, f, wire_type);
putop(c, OP_PUSHTAGDELIM, 0);
putop(c, OP_STARTSEQ, getsel(f, UPB_HANDLER_STARTSEQ));
label(c, LABEL_LOOPSTART);
putpush(c, f);
putop(c, OP_STARTSUBMSG, getsel(f, UPB_HANDLER_STARTSUBMSG));
putop(c, OP_CALL, sub_m);
putop(c, OP_POP);
putcb(c, OP_ENDSUBMSG, h, f, UPB_HANDLER_ENDSUBMSG);
if (wire_type == UPB_WIRE_TYPE_DELIMITED) {
putop(c, OP_SETDELIM);
}
putop(c, OP_CHECKDELIM, LABEL_LOOPBREAK);
putchecktag(c, f, wire_type, LABEL_LOOPBREAK);
putop(c, OP_BRANCH, -LABEL_LOOPSTART);
label(c, LABEL_LOOPBREAK);
putop(c, OP_POP);
putcb(c, OP_ENDSEQ, h, f, UPB_HANDLER_ENDSEQ);
} else {
putop(c, OP_CHECKDELIM, LABEL_ENDMSG);
putchecktag(c, f, wire_type, LABEL_DISPATCH);
dispatchtarget(c, method, f, wire_type);
putpush(c, f);
putop(c, OP_STARTSUBMSG, getsel(f, UPB_HANDLER_STARTSUBMSG));
putop(c, OP_CALL, sub_m);
putop(c, OP_POP);
putcb(c, OP_ENDSUBMSG, h, f, UPB_HANDLER_ENDSUBMSG);
if (wire_type == UPB_WIRE_TYPE_DELIMITED) {
putop(c, OP_SETDELIM);
}
}
generate_msgfield(c, f, method);
} else if (type == UPB_TYPE_STRING || type == UPB_TYPE_BYTES ||
type == UPB_TYPE_MESSAGE) {
label(c, LABEL_FIELD);
if (upb_fielddef_isseq(f)) {
putop(c, OP_CHECKDELIM, LABEL_ENDMSG);
putchecktag(c, f, UPB_WIRE_TYPE_DELIMITED, LABEL_DISPATCH);
dispatchtarget(c, method, f, UPB_WIRE_TYPE_DELIMITED);
putop(c, OP_PUSHTAGDELIM, 0);
putop(c, OP_STARTSEQ, getsel(f, UPB_HANDLER_STARTSEQ));
label(c, LABEL_LOOPSTART);
putop(c, OP_PUSHLENDELIM);
putop(c, OP_STARTSTR, getsel(f, UPB_HANDLER_STARTSTR));
// Need to emit even if no handler to skip past the string.
putop(c, OP_STRING, getsel(f, UPB_HANDLER_STRING));
putop(c, OP_POP);
putcb(c, OP_ENDSTR, h, f, UPB_HANDLER_ENDSTR);
putop(c, OP_SETDELIM);
putop(c, OP_CHECKDELIM, LABEL_LOOPBREAK);
putchecktag(c, f, UPB_WIRE_TYPE_DELIMITED, LABEL_LOOPBREAK);
putop(c, OP_BRANCH, -LABEL_LOOPSTART);
label(c, LABEL_LOOPBREAK);
putop(c, OP_POP);
putcb(c, OP_ENDSEQ, h, f, UPB_HANDLER_ENDSEQ);
} else {
putop(c, OP_CHECKDELIM, LABEL_ENDMSG);
putchecktag(c, f, UPB_WIRE_TYPE_DELIMITED, LABEL_DISPATCH);
dispatchtarget(c, method, f, UPB_WIRE_TYPE_DELIMITED);
putop(c, OP_PUSHLENDELIM);
putop(c, OP_STARTSTR, getsel(f, UPB_HANDLER_STARTSTR));
putop(c, OP_STRING, getsel(f, UPB_HANDLER_STRING));
putop(c, OP_POP);
putcb(c, OP_ENDSTR, h, f, UPB_HANDLER_ENDSTR);
putop(c, OP_SETDELIM);
}
generate_delimfield(c, f, method);
} else {
label(c, LABEL_FIELD);
opcode parse_type = (opcode)descriptor_type;
assert((int)parse_type >= 0 && parse_type <= OP_MAX);
upb_selector_t sel = getsel(f, upb_handlers_getprimitivehandlertype(f));
int wire_type = native_wire_types[upb_fielddef_descriptortype(f)];
if (upb_fielddef_isseq(f)) {
putop(c, OP_CHECKDELIM, LABEL_ENDMSG);
putchecktag(c, f, UPB_WIRE_TYPE_DELIMITED, LABEL_DISPATCH);
dispatchtarget(c, method, f, UPB_WIRE_TYPE_DELIMITED);
putop(c, OP_PUSHLENDELIM);
putop(c, OP_STARTSEQ, getsel(f, UPB_HANDLER_STARTSEQ)); // Packed
label(c, LABEL_LOOPSTART);
putop(c, parse_type, sel);
putop(c, OP_CHECKDELIM, LABEL_LOOPBREAK);
putop(c, OP_BRANCH, -LABEL_LOOPSTART);
dispatchtarget(c, method, f, wire_type);
putop(c, OP_PUSHTAGDELIM, 0);
putop(c, OP_STARTSEQ, getsel(f, UPB_HANDLER_STARTSEQ)); // Non-packed
label(c, LABEL_LOOPSTART);
putop(c, parse_type, sel);
putop(c, OP_CHECKDELIM, LABEL_LOOPBREAK);
putchecktag(c, f, wire_type, LABEL_LOOPBREAK);
putop(c, OP_BRANCH, -LABEL_LOOPSTART);
label(c, LABEL_LOOPBREAK);
putop(c, OP_POP); // Packed and non-packed join.
putcb(c, OP_ENDSEQ, h, f, UPB_HANDLER_ENDSEQ);
putop(c, OP_SETDELIM); // Could remove for non-packed by dup ENDSEQ.
} else {
putop(c, OP_CHECKDELIM, LABEL_ENDMSG);
putchecktag(c, f, wire_type, LABEL_DISPATCH);
dispatchtarget(c, method, f, wire_type);
putop(c, parse_type, sel);
}
generate_primitivefield(c, f, method);
}
}

@ -23,7 +23,7 @@
//
// Note: this mode requires that we can shell out to gcc.
//
// 2. Run the test once locally. This will load the JIT code by building a
// 2. Run the test locally. This will load the JIT code by building a
// .so (/tmp/upb-jit-code.so) and using dlopen, so more of the tooling will
// work properly (like GDB).
//

@ -61,17 +61,21 @@
| add DELIMEND, DECODER->buf
|.endmacro
|
| // OPT: use "call rel32" where possible.
| // Calls an external C function at address "addr".
|.macro callp, addr
|| {
|| //int64_t ofs = (int64_t)addr - (int64_t)upb_status_init;
|| //if (ofs > (1 << 30) || ofs < -(1 << 30)) {
| mov64 rax, (uintptr_t)addr
|
| // Stack must be 16-byte aligned (x86-64 ABI requires this).
| //
| // OPT: possibly remove this by statically ensuring correct alignment.
| //
| // OPT: use "call rel32" where possible.
| push r12
| mov r12, rsp
| and rsp, 0xfffffffffffffff0UL // Align stack.
| call rax
|| //} else {
| // call &addr
|| //}
|| }
| mov rsp, r12
| pop r12
|.endmacro
|
|.macro ld64, val
@ -208,12 +212,6 @@ static void emit_static_asm(jitcompiler *jc) {
| push r12
| push rbx
|
| // Align stack.
| // Since the JIT can call other functions (the JIT'ted code is not a leaf
| // function) we must respect alignment rules. All x86-64 systems require
| // 16-byte stack alignment.
| sub rsp, 8
|
| mov rbx, ARG2_64 // Preserve JIT method.
|
| mov DECODER, rdi
@ -234,7 +232,6 @@ static void emit_static_asm(jitcompiler *jc) {
| mov rax, DECODER->size_param
| mov qword DECODER->call_len, 0
|1:
| add rsp, 8 // Counter previous alignment.
| pop rbx
| pop r12
| pop r13
@ -270,7 +267,6 @@ static void emit_static_asm(jitcompiler *jc) {
| // Must NOT do this before the memcpy(), otherwise memcpy() will
| // clobber the stack we are trying to save!
| mov rsp, DECODER->saved_rsp
| add rsp, 8 // Counter previous alignment.
| pop rbx
| pop r12
| pop r13

@ -1,421 +1,496 @@
/*
* upb - a minimalist implementation of protocol buffers.
*
* Copyright (c) 2009 Google Inc. See LICENSE for details.
* Copyright (c) 2014 Google Inc. See LICENSE for details.
* Author: Josh Haberman <jhaberman@gmail.com>
*
* Since we are implementing pure handlers (ie. without any out-of-band access
* to pre-computed lengths), we have to buffer all submessages before we can
* emit even their first byte.
*
* Not knowing the size of submessages also means we can't write a perfect
* zero-copy implementation, even with buffering. Lengths are stored as
* varints, which means that we don't know how many bytes to reserve for the
* length until we know what the length is.
*
* This leaves us with three main choices:
*
* 1. buffer all submessage data in a temporary buffer, then copy it exactly
* once into the output buffer.
*
* 2. attempt to buffer data directly into the output buffer, estimating how
* many bytes each length will take. When our guesses are wrong, use
* memmove() to grow or shrink the allotted space.
*
* 3. buffer directly into the output buffer, allocating a max length
* ahead-of-time for each submessage length. If we overallocated, we waste
* space, but no memcpy() or memmove() is required. This approach requires
* defining a maximum size for submessages and rejecting submessages that
* exceed that size.
*
* (2) and (3) have the potential to have better performance, but they are more
* complicated and subtle to implement:
*
* (3) requires making an arbitrary choice of the maximum message size; it
* wastes space when submessages are shorter than this and fails
* completely when they are longer. This makes it more finicky and
* requires configuration based on the input. It also makes it impossible
* to perfectly match the output of reference encoders that always use the
* optimal amount of space for each length.
*
* (2) requires guessing the the size upfront, and if multiple lengths are
* guessed wrong the minimum required number of memmove() operations may
* be complicated to compute correctly. Implemented properly, it may have
* a useful amortized or average cost, but more investigation is required
* to determine this and what the optimal algorithm is to achieve it.
*
* (1) makes you always pay for exactly one copy, but its implementation is
* the simplest and its performance is predictable.
*
* So for now, we implement (1) only. If we wish to optimize later, we should
* be able to do it without affecting users.
*
* The strategy is to buffer the segments of data that do *not* depend on
* unknown lengths in one buffer, and keep a separate buffer of segment pointers
* and lengths. When the top-level submessage ends, we can go beginning to end,
* alternating the writing of lengths with memcpy() of the rest of the data.
* At the top level though, no buffering is required.
*/
#include "upb/pb/encoder.h"
#include "upb/pb/varint.int.h"
#include <stdlib.h>
#include "upb/descriptor.h"
/* Functions for calculating sizes of wire values. ****************************/
static size_t upb_v_uint64_t_size(uint64_t val) {
#ifdef __GNUC__
int high_bit = 63 - __builtin_clzll(val); // 0-based, undef if val == 0.
#else
int high_bit = 0;
uint64_t tmp = val;
while(tmp >>= 1) high_bit++;
#endif
return val == 0 ? 1 : high_bit / 7 + 1;
}
static size_t upb_v_int32_t_size(int32_t val) {
// v_uint32's are sign-extended to maintain wire compatibility with int64s.
return upb_v_uint64_t_size((int64_t)val);
/* low-level buffering ********************************************************/
// Low-level functions for interacting with the output buffer.
// TODO(haberman): handle pushback
static void putbuf(upb_pb_encoder *e, const char *buf, size_t len) {
size_t n = upb_bytessink_putbuf(e->output_, e->subc, buf, len, NULL);
UPB_ASSERT_VAR(n, n == len);
}
static size_t upb_v_uint32_t_size(uint32_t val) {
return upb_v_uint64_t_size(val);
static upb_pb_encoder_segment *top(upb_pb_encoder *e) {
return &e->segbuf[*e->top];
}
static size_t upb_f_uint64_t_size(uint64_t val) {
(void)val; // Length is independent of value.
return sizeof(uint64_t);
// Call to ensure that at least "bytes" bytes are available for writing at
// e->ptr. Returns false if the bytes could not be allocated.
static bool reserve(upb_pb_encoder *e, size_t bytes) {
if ((e->limit - e->ptr) < bytes) {
size_t needed = bytes + (e->ptr - e->buf);
size_t old_size = e->limit - e->buf;
size_t new_size = old_size;
while (new_size < needed) {
new_size *= 2;
}
char *realloc_from = (e->buf == e->initbuf) ? NULL : e->buf;
char *new_buf = realloc(realloc_from, new_size);
if (new_buf == NULL) {
return false;
}
if (realloc_from == NULL) {
memcpy(new_buf, e->initbuf, old_size);
}
e->ptr = new_buf + (e->ptr - e->buf);
e->runbegin = new_buf + (e->runbegin - e->buf);
e->limit = new_buf + new_size;
e->buf = new_buf;
}
return true;
}
static size_t upb_f_uint32_t_size(uint32_t val) {
(void)val; // Length is independent of value.
return sizeof(uint32_t);
// Call when "bytes" bytes have been writte at e->ptr. The caller *must* have
// previously called reserve() with at least this many bytes.
static void advance(upb_pb_encoder *e, size_t bytes) {
assert((e->limit - e->ptr) >= bytes);
e->ptr += bytes;
}
// Call when all of the bytes for a handler have been written. Flushes the
// bytes if possible and necessary, returning false if this failed.
static bool commit(upb_pb_encoder *e) {
if (!e->top) {
// We aren't inside a delimited region. Flush our accumulated bytes to
// the output.
//
// TODO(haberman): in the future we may want to delay flushing for
// efficiency reasons.
putbuf(e, e->buf, e->ptr - e->buf);
e->ptr = e->buf;
}
/* Functions to write wire values. ********************************************/
return true;
}
// Since we know in advance the longest that the value could be, we always make
// sure that our buffer is long enough. This saves us from having to perform
// bounds checks.
// Writes the given bytes to the buffer, handling reserve/advance.
static bool encode_bytes(upb_pb_encoder *e, const void *data, size_t len) {
if (!reserve(e, len)) {
return false;
}
// Puts a varint (wire type: UPB_WIRE_TYPE_VARINT).
static uint8_t *upb_put_v_uint64_t(uint8_t *buf, uint64_t val)
{
do {
uint8_t byte = val & 0x7f;
val >>= 7;
if(val) byte |= 0x80;
*buf++ = byte;
} while(val);
return buf;
memcpy(e->ptr, data, len);
advance(e, len);
return true;
}
// Puts an unsigned 32-bit varint, verbatim. Never uses the high 64 bits.
static uint8_t *upb_put_v_uint32_t(uint8_t *buf, uint32_t val)
{
return upb_put_v_uint64_t(buf, val);
// Finish the current run by adding the run totals to the segment and message
// length.
static void accumulate(upb_pb_encoder *e) {
assert(e->ptr >= e->runbegin);
size_t run_len = e->ptr - e->runbegin;
e->segptr->seglen += run_len;
top(e)->msglen += run_len;
e->runbegin = e->ptr;
}
// Puts a signed 32-bit varint, first sign-extending to 64-bits. We do this to
// maintain wire-compatibility with 64-bit signed integers.
static uint8_t *upb_put_v_int32_t(uint8_t *buf, int32_t val)
{
return upb_put_v_uint64_t(buf, (int64_t)val);
// Call to indicate the start of delimited region for which the full length is
// not yet known. All data will be buffered until the length is known.
// Delimited regions may be nested; their lengths will all be tracked properly.
static bool start_delim(upb_pb_encoder *e) {
if (e->top) {
// We are already buffering, advance to the next segment and push it on the
// stack.
accumulate(e);
if (++e->top == e->stacklimit) {
// TODO(haberman): grow stack?
return false;
}
if (++e->segptr == e->seglimit) {
upb_pb_encoder_segment *realloc_from =
(e->segbuf == e->seginitbuf) ? NULL : e->segbuf;
size_t old_size =
(e->seglimit - e->segbuf) * sizeof(upb_pb_encoder_segment);
size_t new_size = old_size * 2;
upb_pb_encoder_segment *new_buf = realloc(realloc_from, new_size);
if (new_buf == NULL) {
return false;
}
if (realloc_from == NULL) {
memcpy(new_buf, e->seginitbuf, old_size);
}
e->segptr = new_buf + (e->segptr - e->segbuf);
e->seglimit = new_buf + (new_size / sizeof(upb_pb_encoder_segment));
e->segbuf = new_buf;
}
} else {
// We were previously at the top level, start buffering.
e->segptr = e->segbuf;
e->top = e->stack;
e->runbegin = e->ptr;
}
*e->top = e->segptr - e->segbuf;
e->segptr->seglen = 0;
e->segptr->msglen = 0;
return true;
}
static void upb_put32(uint8_t *buf, uint32_t val) {
buf[0] = val & 0xff;
buf[1] = (val >> 8) & 0xff;
buf[2] = (val >> 16) & 0xff;
buf[3] = (val >> 24);
// Call to indicate the end of a delimited region. We now know the length of
// the delimited region. If we are not nested inside any other delimited
// regions, we can now emit all of the buffered data we accumulated.
static bool end_delim(upb_pb_encoder *e) {
accumulate(e);
size_t msglen = top(e)->msglen;
if (e->top == e->stack) {
// All lengths are now available, emit all buffered data.
char buf[UPB_PB_VARINT_MAX_LEN];
upb_pb_encoder_segment *s;
const char *ptr = e->buf;
for (s = e->segbuf; s <= e->segptr; s++) {
size_t lenbytes = upb_vencode64(s->msglen, buf);
putbuf(e, buf, lenbytes);
putbuf(e, ptr, s->seglen);
ptr += s->seglen;
}
e->ptr = e->buf;
e->top = NULL;
} else {
// Need to keep buffering; propagate length info into enclosing submessages.
--e->top;
top(e)->msglen += msglen + upb_varint_size(msglen);
}
return true;
}
// Puts a fixed-length 32-bit integer (wire type: UPB_WIRE_TYPE_32BIT).
static uint8_t *upb_put_f_uint32_t(uint8_t *buf, uint32_t val)
{
uint8_t *uint32_end = buf + sizeof(uint32_t);
#if UPB_UNALIGNED_READS_OK
*(uint32_t*)buf = val;
#else
upb_put32(buf, val);
#endif
return uint32_end;
/* tag_t **********************************************************************/
// A precomputed (pre-encoded) tag and length.
typedef struct {
uint8_t bytes;
char tag[7];
} tag_t;
// Allocates a new tag for this field, and sets it in these handlerattr.
static void new_tag(upb_handlers *h, const upb_fielddef *f, upb_wiretype_t wt,
upb_handlerattr *attr) {
uint32_t n = upb_fielddef_number(f);
tag_t *tag = malloc(sizeof(tag_t));
tag->bytes = upb_vencode64((n << 3) | wt, tag->tag);
upb_handlerattr_init(attr);
upb_handlerattr_sethandlerdata(attr, tag);
upb_handlers_addcleanup(h, tag, free);
}
// Puts a fixed-length 64-bit integer (wire type: UPB_WIRE_TYPE_64BIT).
static uint8_t *upb_put_f_uint64_t(uint8_t *buf, uint64_t val)
{
uint8_t *uint64_end = buf + sizeof(uint64_t);
#if UPB_UNALIGNED_READS_OK
*(uint64_t*)buf = val;
#else
upb_put32(buf, (uint32_t)val);
upb_put32(buf, (uint32_t)(val >> 32));
#endif
return uint64_end;
static bool encode_tag(upb_pb_encoder *e, const tag_t *tag) {
return encode_bytes(e, tag->tag, tag->bytes);
}
/* Functions to write and calculate sizes for .proto values. ******************/
// Performs zig-zag encoding, which is used by sint32 and sint64.
static uint32_t upb_zzenc_32(int32_t n) { return (n << 1) ^ (n >> 31); }
static uint64_t upb_zzenc_64(int64_t n) { return (n << 1) ^ (n >> 63); }
/* encoding of wire types *****************************************************/
/* Use macros to define a set of two functions for each .proto type:
*
* // Converts and writes a .proto value into buf. "end" indicates the end
* // of the current available buffer (if the buffer does not contain enough
* // space UPB_STATUS_NEED_MORE_DATA is returned). On success, *outbuf will
* // point one past the data that was written.
* uint8_t *upb_put_INT32(uint8_t *buf, int32_t val);
*
* // Returns the number of bytes required to encode val.
* size_t upb_get_INT32_size(int32_t val);
*
* // Given a .proto value s (source) convert it to a wire value.
* uint32_t upb_vtowv_INT32(int32_t s);
*/
static bool encode_fixed64(upb_pb_encoder *e, uint64_t val) {
// TODO(haberman): byte-swap for big endian.
return encode_bytes(e, &val, sizeof(uint64_t));
}
#define VTOWV(type, wire_t, val_t) \
static wire_t upb_vtowv_ ## type(val_t s)
static bool encode_fixed32(upb_pb_encoder *e, uint32_t val) {
// TODO(haberman): byte-swap for big endian.
return encode_bytes(e, &val, sizeof(uint32_t));
}
#define PUT(type, v_or_f, wire_t, val_t, member_name) \
static uint8_t *upb_put_ ## type(uint8_t *buf, val_t val) { \
wire_t tmp = upb_vtowv_ ## type(val); \
return upb_put_ ## v_or_f ## _ ## wire_t(buf, tmp); \
static bool encode_varint(upb_pb_encoder *e, uint64_t val) {
if (!reserve(e, UPB_PB_VARINT_MAX_LEN)) {
return false;
}
#define T(type, v_or_f, wire_t, val_t, member_name) \
static size_t upb_get_ ## type ## _size(val_t val) { \
return upb_ ## v_or_f ## _ ## wire_t ## _size(val); \
} \
VTOWV(type, wire_t, val_t); /* prototype for PUT below */ \
PUT(type, v_or_f, wire_t, val_t, member_name) \
VTOWV(type, wire_t, val_t)
T(INT32, v, int32_t, int32_t, int32) { return (uint32_t)s; }
T(INT64, v, uint64_t, int64_t, int64) { return (uint64_t)s; }
T(UINT32, v, uint32_t, uint32_t, uint32) { return s; }
T(UINT64, v, uint64_t, uint64_t, uint64) { return s; }
T(SINT32, v, uint32_t, int32_t, int32) { return upb_zzenc_32(s); }
T(SINT64, v, uint64_t, int64_t, int64) { return upb_zzenc_64(s); }
T(FIXED32, f, uint32_t, uint32_t, uint32) { return s; }
T(FIXED64, f, uint64_t, uint64_t, uint64) { return s; }
T(SFIXED32, f, uint32_t, int32_t, int32) { return (uint32_t)s; }
T(SFIXED64, f, uint64_t, int64_t, int64) { return (uint64_t)s; }
T(BOOL, v, uint32_t, bool, _bool) { return (uint32_t)s; }
T(ENUM, v, uint32_t, int32_t, int32) { return (uint32_t)s; }
T(DOUBLE, f, uint64_t, double, _double) {
upb_value v;
v._double = s;
return v.uint64;
advance(e, upb_vencode64(val, e->ptr));
return true;
}
T(FLOAT, f, uint32_t, float, _float) {
upb_value v;
v._float = s;
return v.uint32;
static uint64_t dbl2uint64(double d) {
uint64_t ret;
memcpy(&ret, &d, sizeof(uint64_t));
return ret;
}
#undef VTOWV
#undef PUT
#undef T
static uint8_t *upb_encode_value(uint8_t *buf, upb_field_type_t ft, upb_value v)
{
#define CASE(t, member_name) \
case UPB_TYPE(t): return upb_put_ ## t(buf, v.member_name);
switch(ft) {
CASE(DOUBLE, _double)
CASE(FLOAT, _float)
CASE(INT32, int32)
CASE(INT64, int64)
CASE(UINT32, uint32)
CASE(UINT64, uint64)
CASE(SINT32, int32)
CASE(SINT64, int64)
CASE(FIXED32, uint32)
CASE(FIXED64, uint64)
CASE(SFIXED32, int32)
CASE(SFIXED64, int64)
CASE(BOOL, _bool)
CASE(ENUM, int32)
default: assert(false); return buf;
static uint32_t flt2uint32(float d) {
uint32_t ret;
memcpy(&ret, &d, sizeof(uint32_t));
return ret;
}
/* encoding of proto types ****************************************************/
static bool startmsg(void *c, const void *hd) {
upb_pb_encoder *e = c;
UPB_UNUSED(hd);
if (e->depth++ == 0) {
upb_bytessink_start(e->output_, 0, &e->subc);
}
#undef CASE
return true;
}
static uint32_t _upb_get_value_size(upb_field_type_t ft, upb_value v)
{
#define CASE(t, member_name) \
case UPB_TYPE(t): return upb_get_ ## t ## _size(v.member_name);
switch(ft) {
CASE(DOUBLE, _double)
CASE(FLOAT, _float)
CASE(INT32, int32)
CASE(INT64, int64)
CASE(UINT32, uint32)
CASE(UINT64, uint64)
CASE(SINT32, int32)
CASE(SINT64, int64)
CASE(FIXED32, uint32)
CASE(FIXED64, uint64)
CASE(SFIXED32, int32)
CASE(SFIXED64, int64)
CASE(BOOL, _bool)
CASE(ENUM, int32)
default: assert(false); return 0;
static bool endmsg(void *c, const void *hd, upb_status *status) {
upb_pb_encoder *e = c;
UPB_UNUSED(hd);
UPB_UNUSED(status);
if (--e->depth == 0) {
upb_bytessink_end(e->output_);
}
#undef CASE
return true;
}
static uint8_t *_upb_put_tag(uint8_t *buf, upb_field_number_t num,
upb_wire_type_t wt)
{
return upb_put_UINT32(buf, wt | (num << 3));
static void *encode_startdelimfield(void *c, const void *hd) {
bool ok = encode_tag(c, hd) && commit(c) && start_delim(c);
return ok ? c : UPB_BREAK;
}
static uint32_t _upb_get_tag_size(upb_field_number_t num)
{
return upb_get_UINT32_size(num << 3);
static bool encode_enddelimfield(void *c, const void *hd) {
UPB_UNUSED(hd);
return end_delim(c);
}
static void *encode_startgroup(void *c, const void *hd) {
return (encode_tag(c, hd) && commit(c)) ? c : UPB_BREAK;
}
/* upb_sizebuilder ************************************************************/
static bool encode_endgroup(void *c, const void *hd) {
return encode_tag(c, hd) && commit(c);
}
struct upb_sizebuilder {
// Accumulating size for the current level.
uint32_t size;
static void *encode_startstr(void *c, const void *hd, size_t size_hint) {
UPB_UNUSED(size_hint);
return encode_startdelimfield(c, hd);
}
// Stack of sizes for our current nesting.
uint32_t stack[UPB_MAX_NESTING], *top;
static size_t encode_strbuf(void *c, const void *hd, const char *buf,
size_t len, const upb_bufhandle *h) {
UPB_UNUSED(hd);
UPB_UNUSED(h);
return encode_bytes(c, buf, len) ? len : 0;
}
// Vector of sizes.
uint32_t *sizes;
int sizes_len;
int sizes_size;
#define T(type, ctype, convert, encode) \
static bool encode_scalar_##type(void *e, const void *hd, ctype val) { \
return encode_tag(e, hd) && encode(e, (convert)(val)) && commit(e); \
} \
static bool encode_packed_##type(void *e, const void *hd, ctype val) { \
UPB_UNUSED(hd); \
return encode(e, (convert)(val)); \
}
upb_status status;
};
T(double, double, dbl2uint64, encode_fixed64)
T(float, float, flt2uint32, encode_fixed32);
T(int64, int64_t, uint64_t, encode_varint);
T(int32, int32_t, uint32_t, encode_varint);
T(fixed64, uint64_t, uint64_t, encode_fixed64);
T(fixed32, uint32_t, uint32_t, encode_fixed32);
T(bool, bool, bool, encode_varint);
T(uint32, uint32_t, uint32_t, encode_varint);
T(uint64, uint64_t, uint64_t, encode_varint);
T(enum, int32_t, uint32_t, encode_varint);
T(sfixed32, int32_t, uint32_t, encode_fixed32);
T(sfixed64, int64_t, uint64_t, encode_fixed64);
T(sint32, int32_t, upb_zzenc_32, encode_varint);
T(sint64, int64_t, upb_zzenc_64, encode_varint);
// upb_sink callbacks.
static upb_sink_status _upb_sizebuilder_valuecb(upb_sink *sink, upb_fielddef *f,
upb_value val,
upb_status *status)
{
(void)status;
upb_sizebuilder *sb = (upb_sizebuilder*)sink;
uint32_t size = 0;
size += _upb_get_tag_size(f->number);
size += _upb_get_value_size(f->type, val);
sb->size += size;
return UPB_SINK_CONTINUE;
}
#undef T
static upb_sink_status _upb_sizebuilder_strcb(upb_sink *sink, upb_fielddef *f,
upb_strptr str,
int32_t start, uint32_t end,
upb_status *status)
{
(void)status;
(void)str; // String data itself is not used.
upb_sizebuilder *sb = (upb_sizebuilder*)sink;
if(start >= 0) {
uint32_t size = 0;
size += _upb_get_tag_size(f->number);
size += upb_get_UINT32_size(end - start);
sb->size += size;
}
return UPB_SINK_CONTINUE;
}
static upb_sink_status _upb_sizebuilder_startcb(upb_sink *sink, upb_fielddef *f,
upb_status *status)
{
(void)status;
(void)f; // Unused (we calculate tag size and delimiter in endcb).
upb_sizebuilder *sb = (upb_sizebuilder*)sink;
if(f->type == UPB_TYPE(MESSAGE)) {
*sb->top = sb->size;
sb->top++;
sb->size = 0;
} else {
assert(f->type == UPB_TYPE(GROUP));
sb->size += _upb_get_tag_size(f->number);
}
return UPB_SINK_CONTINUE;
}
/* code to build the handlers *************************************************/
static void newhandlers_callback(const void *closure, upb_handlers *h) {
UPB_UNUSED(closure);
static upb_sink_status _upb_sizebuilder_endcb(upb_sink *sink, upb_fielddef *f,
upb_status *status)
{
(void)status;
upb_sizebuilder *sb = (upb_sizebuilder*)sink;
if(f->type == UPB_TYPE(MESSAGE)) {
sb->top--;
if(sb->sizes_len == sb->sizes_size) {
sb->sizes_size *= 2;
sb->sizes = realloc(sb->sizes, sb->sizes_size * sizeof(*sb->sizes));
upb_handlers_setstartmsg(h, startmsg, NULL);
upb_handlers_setendmsg(h, endmsg, NULL);
const upb_msgdef *m = upb_handlers_msgdef(h);
upb_msg_iter i;
for(upb_msg_begin(&i, m); !upb_msg_done(&i); upb_msg_next(&i)) {
const upb_fielddef *f = upb_msg_iter_field(&i);
bool packed = upb_fielddef_isseq(f) && upb_fielddef_isprimitive(f) &&
upb_fielddef_packed(f);
upb_handlerattr attr;
upb_wiretype_t wt =
packed ? UPB_WIRE_TYPE_DELIMITED
: upb_pb_native_wire_types[upb_fielddef_descriptortype(f)];
// Pre-encode the tag for this field.
new_tag(h, f, wt, &attr);
if (packed) {
upb_handlers_setstartseq(h, f, encode_startdelimfield, &attr);
upb_handlers_setendseq(h, f, encode_enddelimfield, &attr);
}
uint32_t child_size = sb->size;
uint32_t parent_size = *sb->top;
sb->sizes[sb->sizes_len++] = child_size;
// The size according to the parent includes the tag size and delimiter of
// the submessage.
parent_size += upb_get_UINT32_size(child_size);
parent_size += _upb_get_tag_size(f->number);
// Include size accumulated in parent before child began.
sb->size = child_size + parent_size;
} else {
assert(f->type == UPB_TYPE(GROUP));
// As an optimization, we could just add this number twice in startcb, to
// avoid having to recalculate it.
sb->size += _upb_get_tag_size(f->number);
#define T(upper, lower, upbtype) \
case UPB_DESCRIPTOR_TYPE_##upper: \
if (packed) { \
upb_handlers_set##upbtype(h, f, encode_packed_##lower, &attr); \
} else { \
upb_handlers_set##upbtype(h, f, encode_scalar_##lower, &attr); \
} \
break;
switch (upb_fielddef_descriptortype(f)) {
T(DOUBLE, double, double);
T(FLOAT, float, float);
T(INT64, int64, int64);
T(INT32, int32, int32);
T(FIXED64, fixed64, uint64);
T(FIXED32, fixed32, uint32);
T(BOOL, bool, bool);
T(UINT32, uint32, uint32);
T(UINT64, uint64, uint64);
T(ENUM, enum, int32);
T(SFIXED32, sfixed32, int32);
T(SFIXED64, sfixed64, int64);
T(SINT32, sint32, int32);
T(SINT64, sint64, int64);
case UPB_DESCRIPTOR_TYPE_STRING:
case UPB_DESCRIPTOR_TYPE_BYTES:
upb_handlers_setstartstr(h, f, encode_startstr, &attr);
upb_handlers_setendstr(h, f, encode_enddelimfield, &attr);
upb_handlers_setstring(h, f, encode_strbuf, &attr);
break;
case UPB_DESCRIPTOR_TYPE_MESSAGE:
upb_handlers_setstartsubmsg(h, f, encode_startdelimfield, &attr);
upb_handlers_setendsubmsg(h, f, encode_enddelimfield, &attr);
break;
case UPB_DESCRIPTOR_TYPE_GROUP: {
// Endgroup takes a different tag (wire_type = END_GROUP).
upb_handlerattr attr2;
new_tag(h, f, UPB_WIRE_TYPE_END_GROUP, &attr2);
upb_handlers_setstartsubmsg(h, f, encode_startgroup, &attr);
upb_handlers_setendsubmsg(h, f, encode_endgroup, &attr2);
upb_handlerattr_uninit(&attr2);
break;
}
}
#undef T
upb_handlerattr_uninit(&attr);
}
return UPB_SINK_CONTINUE;
}
upb_sink_callbacks _upb_sizebuilder_sink_vtbl = {
_upb_sizebuilder_valuecb,
_upb_sizebuilder_strcb,
_upb_sizebuilder_startcb,
_upb_sizebuilder_endcb
};
/* upb_sink callbacks *********************************************************/
struct upb_encoder {
upb_sink base;
//upb_bytesink *bytesink;
uint32_t *sizes;
int size_offset;
};
// Within one callback we may need to encode up to two separate values.
#define UPB_ENCODER_BUFSIZE (UPB_MAX_ENCODED_SIZE * 2)
static upb_sink_status _upb_encoder_push_buf(upb_encoder *s, const uint8_t *buf,
size_t len, upb_status *status)
{
// TODO: conjure a upb_strptr that points to buf.
//upb_strptr ptr;
(void)s;
(void)buf;
(void)status;
size_t written = 5;// = upb_bytesink_onbytes(s->bytesink, ptr);
if(written < len) {
// TODO: mark to skip "written" bytes next time.
return UPB_SINK_STOP;
} else {
return UPB_SINK_CONTINUE;
}
/* public API *****************************************************************/
const upb_handlers *upb_pb_encoder_newhandlers(const upb_msgdef *m,
const void *owner) {
return upb_handlers_newfrozen(m, owner, newhandlers_callback, NULL);
}
static upb_sink_status _upb_encoder_valuecb(upb_sink *sink, upb_fielddef *f,
upb_value val, upb_status *status)
{
upb_encoder *s = (upb_encoder*)sink;
uint8_t buf[UPB_ENCODER_BUFSIZE], *ptr = buf;
upb_wire_type_t wt = upb_types[f->type].expected_wire_type;
// TODO: handle packed encoding.
ptr = _upb_put_tag(ptr, f->number, wt);
ptr = upb_encode_value(ptr, f->type, val);
return _upb_encoder_push_buf(s, buf, ptr - buf, status);
#define ARRAYSIZE(x) (sizeof(x) / sizeof(x[0]))
void upb_pb_encoder_init(upb_pb_encoder *e, const upb_handlers *h) {
e->output_ = NULL;
e->subc = NULL;
e->buf = e->initbuf;
e->ptr = e->buf;
e->limit = e->buf + ARRAYSIZE(e->initbuf);
e->segbuf = e->seginitbuf;
e->seglimit = e->segbuf + ARRAYSIZE(e->seginitbuf);
e->stacklimit = e->stack + ARRAYSIZE(e->stack);
upb_sink_reset(&e->input_, h, e);
}
static upb_sink_status _upb_encoder_strcb(upb_sink *sink, upb_fielddef *f,
upb_strptr str,
int32_t start, uint32_t end,
upb_status *status)
{
upb_encoder *s = (upb_encoder*)sink;
uint8_t buf[UPB_ENCODER_BUFSIZE], *ptr = buf;
if(start >= 0) {
ptr = _upb_put_tag(ptr, f->number, UPB_WIRE_TYPE_DELIMITED);
ptr = upb_put_UINT32(ptr, end - start);
void upb_pb_encoder_uninit(upb_pb_encoder *e) {
if (e->buf != e->initbuf) {
free(e->buf);
}
// TODO: properly handle partially consumed strings and partially supplied
// strings.
_upb_encoder_push_buf(s, buf, ptr - buf, status);
return _upb_encoder_push_buf(s, (uint8_t*)upb_string_getrobuf(str), end - start, status);
}
static upb_sink_status _upb_encoder_startcb(upb_sink *sink, upb_fielddef *f,
upb_status *status)
{
upb_encoder *s = (upb_encoder*)sink;
uint8_t buf[UPB_ENCODER_BUFSIZE], *ptr = buf;
if(f->type == UPB_TYPE(GROUP)) {
ptr = _upb_put_tag(ptr, f->number, UPB_WIRE_TYPE_START_GROUP);
} else {
ptr = _upb_put_tag(ptr, f->number, UPB_WIRE_TYPE_DELIMITED);
ptr = upb_put_UINT32(ptr, s->sizes[--s->size_offset]);
if (e->segbuf != e->seginitbuf) {
free(e->segbuf);
}
return _upb_encoder_push_buf(s, buf, ptr - buf, status);
}
static upb_sink_status _upb_encoder_endcb(upb_sink *sink, upb_fielddef *f,
upb_status *status)
{
upb_encoder *s = (upb_encoder*)sink;
uint8_t buf[UPB_ENCODER_BUFSIZE], *ptr = buf;
if(f->type != UPB_TYPE(GROUP)) return UPB_SINK_CONTINUE;
ptr = _upb_put_tag(ptr, f->number, UPB_WIRE_TYPE_END_GROUP);
return _upb_encoder_push_buf(s, buf, ptr - buf, status);
void upb_pb_encoder_resetoutput(upb_pb_encoder *e, upb_bytessink *output) {
upb_pb_encoder_reset(e);
e->output_ = output;
e->subc = output->closure;
}
upb_sink_callbacks _upb_encoder_sink_vtbl = {
_upb_encoder_valuecb,
_upb_encoder_strcb,
_upb_encoder_startcb,
_upb_encoder_endcb
};
void upb_pb_encoder_reset(upb_pb_encoder *e) {
e->segptr = NULL;
e->top = NULL;
e->depth = 0;
}
upb_sink *upb_pb_encoder_input(upb_pb_encoder *e) { return &e->input_; }

@ -7,52 +7,155 @@
* Implements a set of upb_handlers that write protobuf data to the binary wire
* format.
*
* For messages that have any submessages, the encoder needs a buffer
* containing the submessage sizes, so they can be properly written at the
* front of each message. Note that groups do *not* have this requirement.
* This encoder implementation does not have any access to any out-of-band or
* precomputed lengths for submessages, so it must buffer submessages internally
* before it can emit the first byte.
*/
#ifndef UPB_ENCODER_H_
#define UPB_ENCODER_H_
#include "upb/upb.h"
#include "upb/bytestream.h"
#include "upb/sink.h"
#ifdef __cplusplus
extern "C" {
namespace upb {
namespace pb {
class Encoder;
} // namespace pb
} // namespace upb
#endif
/* upb_encoder ****************************************************************/
UPB_DECLARE_TYPE(upb::pb::Encoder, upb_pb_encoder);
// A upb_encoder is a upb_sink that emits data to a upb_bytesink in the protocol
// buffer binary wire format.
struct upb_encoder;
typedef struct upb_encoder upb_encoder;
#define UPB_PBENCODER_MAX_NESTING 100
upb_encoder *upb_encoder_new(upb_msgdef *md);
void upb_encoder_free(upb_encoder *e);
/* upb::pb::Encoder ***********************************************************/
// Resets the given upb_encoder such that is is ready to begin encoding,
// outputting data to "bytesink" (which must live until the encoder is
// reset or destroyed).
void upb_encoder_reset(upb_encoder *e, upb_bytesink *bytesink);
// The output buffer is divided into segments; a segment is a string of data
// that is "ready to go" -- it does not need any varint lengths inserted into
// the middle. The seams between segments are where varints will be inserted
// once they are known.
//
// We also use the concept of a "run", which is a range of encoded bytes that
// occur at a single submessage level. Every segment contains one or more runs.
//
// A segment can span messages. Consider:
//
// .--Submessage lengths---------.
// | | |
// | V V
// V | |--------------- | |-----------------
// Submessages: | |-----------------------------------------------
// Top-level msg: ------------------------------------------------------------
//
// Segments: ----- ------------------- -----------------
// Runs: *---- *--------------*--- *----------------
// (* marks the start)
//
// Note that the top-level menssage is not in any segment because it does not
// have any length preceding it.
//
// A segment is only interrupted when another length needs to be inserted. So
// observe how the second segment spans both the inner submessage and part of
// the next enclosing message.
typedef struct {
UPB_PRIVATE_FOR_CPP
uint32_t msglen; // The length to varint-encode before this segment.
uint32_t seglen; // Length of the segment.
} upb_pb_encoder_segment;
// Returns the upb_sink to which data can be written. The sink is invalidated
// when the encoder is reset or destroyed. Note that if the client wants to
// encode any length-delimited submessages it must first call
// upb_encoder_buildsizes() below.
upb_sink *upb_encoder_sink(upb_encoder *e);
UPB_DEFINE_CLASS0(upb::pb::Encoder,
public:
Encoder(const upb::Handlers* handlers);
~Encoder();
// Call prior to pushing any data with embedded submessages. "src" must yield
// exactly the same data as what will next be encoded, but in reverse order.
// The encoder iterates over this data in order to determine the sizes of the
// submessages. If any errors are returned by the upb_src, the status will
// be saved in *status. If the client is sure that the upb_src will not throw
// any errors, "status" may be NULL.
void upb_encoder_buildsizes(upb_encoder *e, upb_src *src, upb_status *status);
static reffed_ptr<const Handlers> NewHandlers(const upb::MessageDef* msg);
// Resets the state of the printer, so that it will expect to begin a new
// document.
void Reset();
// Resets the output pointer which will serve as our closure.
void ResetOutput(BytesSink* output);
// The input to the encoder.
Sink* input();
private:
UPB_DISALLOW_COPY_AND_ASSIGN(Encoder);
,
UPB_DEFINE_STRUCT0(upb_pb_encoder, UPB_QUOTE(
// Our input and output.
upb_sink input_;
upb_bytessink *output_;
// The "subclosure" -- used as the inner closure as part of the bytessink
// protocol.
void *subc;
// The output buffer and limit, and our current write position. "buf"
// initially points to "initbuf", but is dynamically allocated if we need to
// grow beyond the initial size.
char *buf, *ptr, *limit;
// The beginning of the current run, or undefined if we are at the top level.
char *runbegin;
// The list of segments we are accumulating.
upb_pb_encoder_segment *segbuf, *segptr, *seglimit;
// The stack of enclosing submessages. Each entry in the stack points to the
// segment where this submessage's length is being accumulated.
int stack[UPB_PBENCODER_MAX_NESTING], *top, *stacklimit;
// Depth of startmsg/endmsg calls.
int depth;
// Initial buffers for the output buffer and segment buffer. If we outgrow
// these we will dynamically allocate bigger ones.
char initbuf[256];
upb_pb_encoder_segment seginitbuf[32];
)));
UPB_BEGIN_EXTERN_C
const upb_handlers *upb_pb_encoder_newhandlers(const upb_msgdef *m,
const void *owner);
void upb_pb_encoder_reset(upb_pb_encoder *e);
upb_sink *upb_pb_encoder_input(upb_pb_encoder *p);
void upb_pb_encoder_init(upb_pb_encoder *e, const upb_handlers *h);
void upb_pb_encoder_resetoutput(upb_pb_encoder *e, upb_bytessink *output);
void upb_pb_encoder_uninit(upb_pb_encoder *e);
UPB_END_EXTERN_C
#ifdef __cplusplus
} /* extern "C" */
namespace upb {
namespace pb {
inline Encoder::Encoder(const upb::Handlers* handlers) {
upb_pb_encoder_init(this, handlers);
}
inline Encoder::~Encoder() {
upb_pb_encoder_uninit(this);
}
inline void Encoder::Reset() {
upb_pb_encoder_reset(this);
}
inline void Encoder::ResetOutput(BytesSink* output) {
upb_pb_encoder_resetoutput(this, output);
}
inline Sink* Encoder::input() {
return upb_pb_encoder_input(this);
}
inline reffed_ptr<const Handlers> Encoder::NewHandlers(
const upb::MessageDef *md) {
const Handlers* h = upb_pb_encoder_newhandlers(md, &h);
return reffed_ptr<const Handlers>(h, &h);
}
} // namespace pb
} // namespace upb
#endif
#endif /* UPB_ENCODER_H_ */

@ -7,6 +7,29 @@
#include "upb/pb/varint.int.h"
// Index is descriptor type.
const uint8_t upb_pb_native_wire_types[] = {
UPB_WIRE_TYPE_END_GROUP, // ENDGROUP
UPB_WIRE_TYPE_64BIT, // DOUBLE
UPB_WIRE_TYPE_32BIT, // FLOAT
UPB_WIRE_TYPE_VARINT, // INT64
UPB_WIRE_TYPE_VARINT, // UINT64
UPB_WIRE_TYPE_VARINT, // INT32
UPB_WIRE_TYPE_64BIT, // FIXED64
UPB_WIRE_TYPE_32BIT, // FIXED32
UPB_WIRE_TYPE_VARINT, // BOOL
UPB_WIRE_TYPE_DELIMITED, // STRING
UPB_WIRE_TYPE_START_GROUP, // GROUP
UPB_WIRE_TYPE_DELIMITED, // MESSAGE
UPB_WIRE_TYPE_DELIMITED, // BYTES
UPB_WIRE_TYPE_VARINT, // UINT32
UPB_WIRE_TYPE_VARINT, // ENUM
UPB_WIRE_TYPE_32BIT, // SFIXED32
UPB_WIRE_TYPE_64BIT, // SFIXED64
UPB_WIRE_TYPE_VARINT, // SINT32
UPB_WIRE_TYPE_VARINT, // SINT64
};
// A basic branch-based decoder, uses 32-bit values to get good performance
// on 32-bit architectures (but performs well on 64-bits also).
// This scheme comes from the original Google Protobuf implementation (proto2).

@ -37,6 +37,10 @@ typedef enum {
// wiki document about this).
#define UPB_PB_VARINT_MAX_LEN 10
// Array of the "native" (ie. non-packed-repeated) wire type for the given a
// descriptor type (upb_descriptortype_t).
extern const uint8_t upb_pb_native_wire_types[];
/* Zig-zag encoding/decoding **************************************************/
UPB_INLINE int32_t upb_zzdec_32(uint32_t n) {
@ -129,6 +133,11 @@ UPB_INLINE size_t upb_vencode64(uint64_t val, char *buf) {
return i;
}
UPB_INLINE size_t upb_varint_size(uint64_t val) {
char buf[UPB_PB_VARINT_MAX_LEN];
return upb_vencode64(val, buf);
}
// Encodes a 32-bit varint, *not* sign-extended.
UPB_INLINE uint64_t upb_vencode32(uint32_t val) {
char buf[UPB_PB_VARINT_MAX_LEN];

@ -42,14 +42,36 @@ char *upb_strdup(const char *s) {
return p;
}
static upb_tabkey strkey(const char *str) {
upb_tabkey k;
k.str = (char*)str;
// A type to represent the lookup key of either a strtable or an inttable.
// This is like upb_tabkey, but can carry a size also to allow lookups of
// non-NULL-terminated strings (we don't store string lengths in the table).
typedef struct {
upb_tabkey key;
uint32_t len; // For string keys only.
} lookupkey_t;
static lookupkey_t strkey(const char *str) {
lookupkey_t k;
k.key.str = (char*)str;
k.len = strlen(str);
return k;
}
typedef const upb_tabent *hashfunc_t(const upb_table *t, upb_tabkey key);
typedef bool eqlfunc_t(upb_tabkey k1, upb_tabkey k2);
static lookupkey_t strkey2(const char *str, size_t len) {
lookupkey_t k;
k.key.str = (char*)str;
k.len = len;
return k;
}
static lookupkey_t intkey(uintptr_t key) {
lookupkey_t k;
k.key = upb_intkey(key);
return k;
}
typedef uint32_t hashfunc_t(upb_tabkey key);
typedef bool eqlfunc_t(upb_tabkey k1, lookupkey_t k2);
/* Base table (shared code) ***************************************************/
@ -85,10 +107,14 @@ static upb_tabent *emptyent(upb_table *t) {
while (1) { if (upb_tabent_isempty(--e)) return e; assert(e > t->entries); }
}
static const upb_tabent *findentry(const upb_table *t, upb_tabkey key,
hashfunc_t *hash, eqlfunc_t *eql) {
static upb_tabent *getentry_mutable(upb_table *t, uint32_t hash) {
return (upb_tabent*)upb_getentry(t, hash);
}
static const upb_tabent *findentry(const upb_table *t, lookupkey_t key,
uint32_t hash, eqlfunc_t *eql) {
if (t->size_lg2 == 0) return NULL;
const upb_tabent *e = hash(t, key);
const upb_tabent *e = upb_getentry(t, hash);
if (upb_tabent_isempty(e)) return NULL;
while (1) {
if (eql(e->key, key)) return e;
@ -96,8 +122,13 @@ static const upb_tabent *findentry(const upb_table *t, upb_tabkey key,
}
}
static bool lookup(const upb_table *t, upb_tabkey key, upb_value *v,
hashfunc_t *hash, eqlfunc_t *eql) {
static upb_tabent *findentry_mutable(upb_table *t, lookupkey_t key,
uint32_t hash, eqlfunc_t *eql) {
return (upb_tabent*)findentry(t, key, hash, eql);
}
static bool lookup(const upb_table *t, lookupkey_t key, upb_value *v,
uint32_t hash, eqlfunc_t *eql) {
const upb_tabent *e = findentry(t, key, hash, eql);
if (e) {
if (v) {
@ -110,13 +141,13 @@ static bool lookup(const upb_table *t, upb_tabkey key, upb_value *v,
}
// The given key must not already exist in the table.
static void insert(upb_table *t, upb_tabkey key, upb_value val,
hashfunc_t *hash, eqlfunc_t *eql) {
static void insert(upb_table *t, lookupkey_t key, upb_value val,
uint32_t hash, hashfunc_t *hashfunc, eqlfunc_t *eql) {
UPB_UNUSED(eql);
assert(findentry(t, key, hash, eql) == NULL);
assert(val.ctype == t->ctype);
t->count++;
upb_tabent *mainpos_e = (upb_tabent*)hash(t, key);
upb_tabent *mainpos_e = getentry_mutable(t, hash);
upb_tabent *our_e = mainpos_e;
if (upb_tabent_isempty(mainpos_e)) {
// Our main position is empty; use it.
@ -125,7 +156,7 @@ static void insert(upb_table *t, upb_tabkey key, upb_value val,
// Collision.
upb_tabent *new_e = emptyent(t);
// Head of collider's chain.
upb_tabent *chain = (upb_tabent*)hash(t, mainpos_e->key);
upb_tabent *chain = getentry_mutable(t, hashfunc(mainpos_e->key));
if (chain == mainpos_e) {
// Existing ent is in its main posisiton (it has the same hash as us, and
// is the head of our chain). Insert to new ent and append to this chain.
@ -146,14 +177,14 @@ static void insert(upb_table *t, upb_tabkey key, upb_value val,
our_e->next = NULL;
}
}
our_e->key = key;
our_e->key = key.key;
our_e->val = val.val;
assert(findentry(t, key, hash, eql) == our_e);
}
static bool rm(upb_table *t, upb_tabkey key, upb_value *val,
upb_tabkey *removed, hashfunc_t *hash, eqlfunc_t *eql) {
upb_tabent *chain = (upb_tabent*)hash(t, key);
static bool rm(upb_table *t, lookupkey_t key, upb_value *val,
upb_tabkey *removed, uint32_t hash, eqlfunc_t *eql) {
upb_tabent *chain = getentry_mutable(t, hash);
if (upb_tabent_isempty(chain)) return false;
if (eql(chain->key, key)) {
// Element to remove is at the head of its chain.
@ -210,13 +241,12 @@ static size_t begin(const upb_table *t) {
// A simple "subclass" of upb_table that only adds a hash function for strings.
static const upb_tabent *strhash(const upb_table *t, upb_tabkey key) {
// Could avoid the strlen() by using a hash function that terminates on NULL.
return t->entries + (MurmurHash2(key.str, strlen(key.str), 0) & t->mask);
static uint32_t strhash(upb_tabkey key) {
return MurmurHash2(key.str, strlen(key.str), 0);
}
static bool streql(upb_tabkey k1, upb_tabkey k2) {
return strcmp(k1.str, k2.str) == 0;
static bool streql(upb_tabkey k1, lookupkey_t k2) {
return strncmp(k1.str, k2.key.str, k2.len) == 0 && k1.str[k2.len] == '\0';
}
bool upb_strtable_init(upb_strtable *t, upb_ctype_t ctype) {
@ -252,17 +282,23 @@ bool upb_strtable_insert(upb_strtable *t, const char *k, upb_value v) {
}
}
if ((k = upb_strdup(k)) == NULL) return false;
insert(&t->t, strkey(k), v, &strhash, &streql);
lookupkey_t key = strkey(k);
uint32_t hash = MurmurHash2(key.key.str, key.len, 0);
insert(&t->t, strkey(k), v, hash, &strhash, &streql);
return true;
}
bool upb_strtable_lookup(const upb_strtable *t, const char *key, upb_value *v) {
return lookup(&t->t, strkey(key), v, &strhash, &streql);
bool upb_strtable_lookup2(const upb_strtable *t, const char *key, size_t len,
upb_value *v) {
uint32_t hash = MurmurHash2(key, len, 0);
return lookup(&t->t, strkey2(key, len), v, hash, &streql);
}
bool upb_strtable_remove(upb_strtable *t, const char *key, upb_value *val) {
uint32_t hash = MurmurHash2(key, strlen(key), 0);
upb_tabkey tabkey;
if (rm(&t->t, strkey(key), val, &tabkey, &strhash, &streql)) {
if (rm(&t->t, strkey(key), val, &tabkey, hash, &streql)) {
free((void*)tabkey.str);
return true;
} else {
@ -317,8 +353,10 @@ bool upb_strtable_iter_isequal(const upb_strtable_iter *i1,
// For inttables we use a hybrid structure where small keys are kept in an
// array and large keys are put in the hash table.
static bool inteql(upb_tabkey k1, upb_tabkey k2) {
return k1.num == k2.num;
static uint32_t inthash(upb_tabkey key) { return upb_inthash(key.num); }
static bool inteql(upb_tabkey k1, lookupkey_t k2) {
return k1.num == k2.key.num;
}
static _upb_value *mutable_array(upb_inttable *t) {
@ -330,7 +368,7 @@ static _upb_value *inttable_val(upb_inttable *t, uintptr_t key) {
return upb_arrhas(t->array[key]) ? &(mutable_array(t)[key]) : NULL;
} else {
upb_tabent *e =
(upb_tabent*)findentry(&t->t, upb_intkey(key), &upb_inthash, &inteql);
findentry_mutable(&t->t, intkey(key), upb_inthash(key), &inteql);
return e ? &e->val : NULL;
}
}
@ -402,7 +440,8 @@ bool upb_inttable_insert(upb_inttable *t, uintptr_t key, upb_value val) {
const upb_tabent *e = &t->t.entries[i];
upb_value v;
_upb_value_setval(&v, e->val, t->t.ctype);
insert(&new_table, e->key, v, &upb_inthash, &inteql);
uint32_t hash = upb_inthash(e->key.num);
insert(&new_table, intkey(e->key.num), v, hash, &inthash, &inteql);
}
assert(t->t.count == new_table.count);
@ -410,7 +449,7 @@ bool upb_inttable_insert(upb_inttable *t, uintptr_t key, upb_value val) {
uninit(&t->t);
t->t = new_table;
}
insert(&t->t, upb_intkey(key), val, &upb_inthash, &inteql);
insert(&t->t, intkey(key), val, upb_inthash(key), &inthash, &inteql);
}
check(t);
return true;
@ -446,7 +485,8 @@ bool upb_inttable_remove(upb_inttable *t, uintptr_t key, upb_value *val) {
}
} else {
upb_tabkey removed;
success = rm(&t->t, upb_intkey(key), val, &removed, &upb_inthash, &inteql);
uint32_t hash = upb_inthash(key);
success = rm(&t->t, intkey(key), val, &removed, hash, &inteql);
}
check(t);
return success;

@ -25,6 +25,7 @@
#include <assert.h>
#include <stdint.h>
#include <string.h>
#include "upb.h"
#ifdef __cplusplus
@ -219,20 +220,27 @@ UPB_INLINE bool upb_tabent_isempty(const upb_tabent *e) {
return e->key.num == 0;
}
// Used by some of the unit tests for generic hashing functionality.
uint32_t MurmurHash2(const void * key, size_t len, uint32_t seed);
UPB_INLINE upb_tabkey upb_intkey(uintptr_t key) {
upb_tabkey k = {key}; return k;
upb_tabkey k;
k.num = key;
return k;
}
UPB_INLINE uint32_t upb_inthash(uintptr_t key) {
return (uint32_t)key;
}
UPB_INLINE const upb_tabent *upb_inthash(const upb_table *t, upb_tabkey key) {
return t->entries + ((uint32_t)key.num & t->mask);
static const upb_tabent *upb_getentry(const upb_table *t, uint32_t hash) {
return t->entries + (hash & t->mask);
}
UPB_INLINE bool upb_arrhas(_upb_value v) {
return v.uint64 != (uint64_t)UPB_ARRAY_EMPTYVAL;
}
uint32_t MurmurHash2(const void *key, size_t len, uint32_t seed);
// Initialize and uninitialize a table, respectively. If memory allocation
// failed, false is returned that the table is uninitialized.
bool upb_inttable_init(upb_inttable *table, upb_ctype_t ctype);
@ -259,7 +267,14 @@ bool upb_strtable_insert(upb_strtable *t, const char *key, upb_value val);
// Looks up key in this table, returning "true" if the key was found.
// If v is non-NULL, copies the value for this key into *v.
bool upb_inttable_lookup(const upb_inttable *t, uintptr_t key, upb_value *v);
bool upb_strtable_lookup(const upb_strtable *t, const char *key, upb_value *v);
bool upb_strtable_lookup2(const upb_strtable *t, const char *key, size_t len,
upb_value *v);
// For NULL-terminated strings.
UPB_INLINE bool upb_strtable_lookup(const upb_strtable *t, const char *key,
upb_value *v) {
return upb_strtable_lookup2(t, key, strlen(key), v);
}
// Removes an item from the table. Returns true if the remove was successful,
// and stores the removed item in *val if non-NULL.
@ -302,7 +317,7 @@ UPB_INLINE bool upb_inttable_lookup32(const upb_inttable *t, uint32_t key,
} else {
const upb_tabent *e;
if (t->t.entries == NULL) return false;
for (e = upb_inthash(&t->t, upb_intkey(key)); true; e = e->next) {
for (e = upb_getentry(&t->t, upb_inthash(key)); true; e = e->next) {
if ((uint32_t)e->key.num == key) {
_upb_value_setval(v, e->val, t->t.ctype);
return true;

Loading…
Cancel
Save