Merge branch 'master' into defcleanup

pull/13171/head
Joshua Haberman 6 years ago
commit 48bef61537
  1. 23
      .travis.yml
  2. 1
      BUILD
  3. 457
      Makefile
  4. 7
      kokoro/ubuntu/build.sh
  5. 575
      tools/make_c_api.lua
  6. 43
      tools/upbc.lua

@ -1,23 +0,0 @@
language: cpp
compiler:
- gcc
- clang
install:
- ./travis.sh install
- export PATH=$PATH:$PWD/protoc # used by genfiles_install()
script: ./travis.sh script
after_success: ./travis.sh after_success
after_failure: ./travis.sh after_failure
env:
global:
- secure: "Rk5UxBrSGzyuJsOc7DO3ZTSMj0LbKTpBKS7PlvYIPgcU8+DYd3ZbkvWuJb1qD3CDp0J/9X8XWK2c51taiNlSsiqwS6/ympj7VX/k5VbtG0NauWwMoNZnpB0JHaIW1Qn5O/rI4B0zLCraObD/ythXeFOzevpMbfZcB5DIbNsgD8c="
- secure: "E9G9109pmBVh085+f3ZDFCUqObCjHJymZW/knx0/ABJ3xK9O91RXEEkQTIaKDDasHcx9eujU9yK8F6gFgFulEZLxLvS5if5RFXeK5G1YX/MxWjA6Jh2j0dnSbGdd5Q+Lj8/tBqo50ry59qGYqaUPZ9aCXanal3ymbl2lA96n5qE="
matrix:
- UPB_TRAVIS_BUILD=bare
- UPB_TRAVIS_BUILD=barejit
- UPB_TRAVIS_BUILD=core32
- UPB_TRAVIS_BUILD=lua
- UPB_TRAVIS_BUILD=ndebug
- UPB_TRAVIS_BUILD=coverage
- UPB_TRAVIS_BUILD=genfiles
- UPB_TRAVIS_BUILD=amalgamated

@ -302,7 +302,6 @@ lua_library(
name = "lua/upbc_lib",
srcs = [
"tools/dump_cinit.lua",
"tools/make_c_api.lua",
],
luadeps = [
"lua/upb",

@ -1,457 +0,0 @@
#
# This Makefile builds the upb library as well as associated tests, tools, and
# language extensions.
#
# It does not use autoconf/automake/libtool in order to stay lightweight and
# avoid the need for running ./configure.
#
# Summary of compiler flags you may want to use:
#
# * -UNDEBUG: enables assertions() (makes binary larger and slower)
# * -O0: disable optimizations
# * -g: enable debug symbols
# * -fomit-frame-pointer: makes code smaller and faster by freeing up a reg.
#
# Threading:
# * -DUPB_THREAD_UNSAFE: remove all thread-safety.
.PHONY: all lib clean tests test descriptorgen amalgamate
.PHONY: clean_leave_profile genfiles
# Prevents the deletion of intermediate files.
.SECONDARY:
UPB_MODULES = upb upb.pb upb.json upb.descriptor
UPB_LIBS = $(patsubst %,lib/lib%.a,$(UPB_MODULES))
UPB_PICLIBS = $(patsubst %,lib/lib%_pic.a,$(UPB_MODULES))
# Default rule: build all upb libraries (but no bindings)
default: $(UPB_LIBS)
# All: build absolutely everything
all: lib tests tools/upbc lua python
testall: test pythontest
# Set this to have user-specific flags (especially things like -O0 and -g).
USER_CPPFLAGS?=
# Build with "make WITH_JIT=yes" (or anything besides "no") to enable the JIT.
WITH_JIT=no
# Build with "make UPB_FAIL_WARNINGS=yes" (or anything besides "no") to turn
# warnings into errors.
UPB_FAIL_WARNINGS?=no
# Basic compiler/flag setup.
# We are C89/C++98, with the one exception that we need stdint and "long long."
CC?=cc
CXX?=c++
AR?=ar
CFLAGS=
CXXFLAGS=
INCLUDE=-I.
CSTD=-std=c89 -pedantic -Wno-long-long
CXXSTD=-std=c++98 -pedantic -Wno-long-long
WARNFLAGS=-Wall -Wextra -Wpointer-arith
WARNFLAGS_CXX=$(WARNFLAGS) -Wno-unused-private-field
CPPFLAGS=$(INCLUDE) -DNDEBUG $(USER_CPPFLAGS)
LUA=lua # 5.1 and 5.2 should both be supported
ifneq ($(WITH_JIT), no)
USE_JIT=true
CPPFLAGS += -DUPB_USE_JIT_X64
EXTRA_LIBS += -ldl
endif
ifeq ($(CC), clang)
WARNFLAGS += -Wconditional-uninitialized
endif
ifeq ($(CXX), clang++)
WARNFLAGS_CXX += -Wconditional-uninitialized
endif
ifneq ($(UPB_FAIL_WARNINGS), no)
WARNFLAGS += -Werror -Wno-keyword-macro
WARNFLAGS_CXX += -Werror -Wno-keyword-macro
endif
# Build with "make Q=" to see all commands that are being executed.
Q?=@
# Function to expand a wildcard pattern recursively.
rwildcard=$(strip $(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2)$(filter $(subst *,%,$2),$d)))
ifeq ($(Q), @)
E=@echo
else
E=@:
endif
install:
test -f upb/bindings/ruby/upb.so && cd upb/bindings/ruby && make install
# Dependency generating. #######################################################
-include deps
# Unfortuantely we can't easily generate deps for benchmarks or tests because
# of the scheme we use that compiles the same source file multiple times with
# different -D options, which can include different header files.
dep:
$(E) Regenerating dependencies for upb/...
@set -e
@rm -f deps
@for file in $$(find . -name '*.c'); do \
gcc -MM $$file -MT $${file%.*}.o $(CPPFLAGS) -I. >> deps 2> /dev/null; \
done
clean_leave_profile:
@rm -rf obj lib
@rm -f tests/google_message?.h
@rm -f tests/json/test.upbdefs.o
@rm -f $(TESTS) tests/testmain.o tests/t.* tests/conformance_upb
@rm -rf tools/upbc deps
@rm -rf upb/bindings/python/build
@rm -f upb/bindings/ruby/Makefile
@rm -f upb/bindings/ruby/upb.o
@rm -f upb/bindings/ruby/upb.so
@rm -f upb/bindings/ruby/mkmf.log
@rm -f tests/google_messages.pb.*
@rm -f tests/google_messages.proto.pb
@rm -f upb.c upb.h
@find . | grep dSYM | xargs rm -rf
clean: clean_leave_profile clean_lua
@rm -rf $(call rwildcard,,*.gcno) $(call rwildcard,,*.gcda)
# A little bit of Make voodoo: you can call this from the deps of a patterned
# rule like so:
#
# all: lib/libfoo.bar.a
#
# foo_bar_SRCS = a.c b.c
#
# # This will expand into a.o b.o
# lib/lib%.a: $(call make_objs,o)
# gcc -c -o $@ $^
#
# SECONDEXPANSION: flips on a bit essentially that allows this "secondary
# expansion": it must appear before anything that uses make_objs.
.SECONDEXPANSION:
to_srcs = $(subst .,_,$(1)_SRCS)
pc = %
make_objs = $$(patsubst $$(pc).c,obj/$$(pc).$(1),$$($$(call to_srcs,$$*)))
make_objs_cc = $$(patsubst $$(pc).cc,obj/$$(pc).$(1),$$($$(call to_srcs,$$*)))
# Core libraries (ie. not bindings). ###############################################################
upb_SRCS = \
google/protobuf/descriptor.upb.c \
upb/decode.c \
upb/def.c \
upb/encode.c \
upb/handlers.c \
upb/msg.c \
upb/msgfactory.c \
upb/refcounted.c \
upb/sink.c \
upb/table.c \
upb/upb.c \
upb_descriptor_SRCS = \
upb/descriptor/descriptor.upbdefs.c \
upb/descriptor/reader.c \
upb_pb_SRCS = \
upb/pb/compile_decoder.c \
upb/pb/decoder.c \
upb/pb/encoder.c \
upb/pb/glue.c \
upb/pb/textprinter.c \
upb/pb/varint.c \
# If the JIT is enabled we include its source.
# If Lua is present we can use DynASM to regenerate the .h file.
ifdef USE_JIT
upb_pb_SRCS += upb/pb/compile_decoder_x64.c
# The JIT can't compile with -Wpedantic, since it does some inherently
# platform-specific things like casting between data pointers and function
# pointers. Also DynASM emits some GNU extensions.
obj/upb/pb/compile_decoder_x64.o : CSTD = -std=gnu89
obj/upb/pb/compile_decoder_x64.lo : CSTD = -std=gnu89
upb/pb/compile_decoder_x64.h: upb/pb/compile_decoder_x64.dasc
$(E) DYNASM $<
$(Q) $(LUA) third_party/dynasm/dynasm.lua -c upb/pb/compile_decoder_x64.dasc > upb/pb/compile_decoder_x64.h || (rm upb/pb/compile_decoder_x64.h ; false)
endif
upb_json_SRCS = \
upb/json/parser.c \
upb/json/printer.c \
upb/pb/encoder.c \
upb/pb/varint.c \
upb/sink.c \
# Ideally we could keep this uncommented, but Git apparently sometimes skews
# timestamps slightly at "clone" time, which makes "Make" think that it needs
# to rebuild upb/json/parser.c when it actually doesn't. This would be harmless
# except that the user might not have Ragel installed.
#
# So instead we require an excplicit "make ragel" to rebuild this (for now).
# More pain for people developing upb/json/parser.rl, but less pain for everyone
# else.
#
# upb/json/parser.c: upb/json/parser.rl
# $(E) RAGEL $<
# $(Q) ragel -C -o upb/json/parser.c upb/json/parser.rl
ragel:
$(E) RAGEL upb/json/parser.rl
$(Q) ragel -C -o upb/json/parser.c upb/json/parser.rl
# If the user doesn't specify an -O setting, we use -O3 for critical-path
# code and -Os for the rest.
ifeq (, $(findstring -O, $(USER_CPPFLAGS)))
OPT = -O3
lib/libupb.a : OPT = -Os
lib/libupb.descriptor.a : OPT = -Os
obj/upb/pb/compile_decoder.o : OPT = -Os
obj/upb/pb/compile_decoder_64.o : OPT = -Os
ifdef USE_JIT
obj/upb/pb/compile_decoder_x64.o: OPT=-Os
endif
endif
$(UPB_PICLIBS): lib/lib%_pic.a: $(call make_objs,lo)
$(E) AR $@
$(Q) mkdir -p lib && $(AR) rcs $@ $^
$(UPB_LIBS): lib/lib%.a: $(call make_objs,o)
$(E) AR $@
$(Q) mkdir -p lib && $(AR) rcs $@ $^
obj/%.o: %.c | $$(@D)/.
$(E) CC $<
$(Q) $(CC) $(OPT) $(CSTD) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -c -o $@ $<
obj/%.o: %.cc | $$(@D)/.
$(E) CXX $<
$(Q) $(CXX) $(OPT) $(CXXSTD) $(WARNFLAGS_CXX) $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $<
obj/%.lo: %.c | $$(@D)/.
$(E) 'CC -fPIC' $<
$(Q) $(CC) $(OPT) $(CSTD) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -c -o $@ $< -fPIC
obj/%.lo: %.cc | $$(@D)/.
$(E) CXX -fPIC $<
$(Q) $(CXX) $(OPT) $(CXXSTD) $(WARNFLAGS_CXX) $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $< -fPIC
# Note: mkdir -p is technically susceptible to races when used with make -j.
%/.:
$(Q) mkdir -p $@
# Regenerating the auto-generated files in upb/.
upb/descriptor/descriptor.pb: upb/descriptor/descriptor.proto
$(E) PROTOC upb/descriptor/descriptor.proto
$(Q) protoc upb/descriptor/descriptor.proto -oupb/descriptor/descriptor.pb
# "genfiles" includes Proto schemas we need for tests
# For the moment we check in the *.upbdefs.* generated files so that people
# can build and run the tests without Lua as a build dependency.
genfiles: tools/upbc
@# TODO: replace protoc with upbc when upb can parse .proto files
$(E) PROTOC upb/descriptor/descriptor.proto
$(Q) protoc upb/descriptor/descriptor.proto -oupb/descriptor/descriptor.pb
$(E) PROTOC google/protobuf/descriptor.proto
$(Q) protoc google/protobuf/descriptor.proto -ogoogle/protobuf/descriptor.pb
$(E) UPBC upb/descriptor/descriptor.pb
$(Q) ./tools/upbc --generate-upbdefs upb/descriptor/descriptor.pb
$(E) UPBC google/protobuf/descriptor.pb
$(Q) ./tools/upbc google/protobuf/descriptor.pb
$(E) PROTOC tests/json/test.proto
$(Q) protoc tests/json/test.proto -otests/json/test.proto.pb
$(E) UPBC tests/json/test.proto.pb
$(Q) ./tools/upbc --generate-upbdefs tests/json/test.proto.pb
$(E) DYNASM upb/pb/compile_decoder_x64.dasc
$(Q) $(LUA) third_party/dynasm/dynasm.lua -c upb/pb/compile_decoder_x64.dasc > upb/pb/compile_decoder_x64.h || (rm upb/pb/compile_decoder_x64.h ; false)
# upbc depends on these Lua extensions.
UPBC_LUA_EXTS = \
upb/bindings/lua/upb_c.so \
upb/bindings/lua/upb/pb_c.so \
upb/bindings/lua/upb/table_c.so \
tools/upbc: $(UPBC_LUA_EXTS) Makefile
$(E) ECHO tools/upbc
$(Q) echo "#!/bin/sh" > tools/upbc
$(Q) echo 'BASE=`dirname "$$0"`' >> tools/upbc
$(Q) echo 'export LUA_CPATH="$$BASE/../upb/bindings/lua/?.so"' >> tools/upbc
$(Q) echo 'export LUA_PATH="$$BASE/?.lua;$$BASE/../upb/bindings/lua/?.lua"' >> tools/upbc
$(Q) echo 'lua $$BASE/upbc.lua "$$@"' >> tools/upbc
$(Q) chmod a+x tools/upbc
examples/msg: examples/msg.c $(LIBUPB)
$(E) CC $<
$(Q) $(CC) $(CFLAGS) $(CPPFLAGS) -o $@ $< $(LIBUPB)
# Tests. #######################################################################
# This section contains only the tests that don't depend on any external
# libraries.
C_TESTS = \
tests/pb/test_varint \
tests/test_def \
tests/test_handlers \
CC_TESTS = \
tests/pb/test_decoder \
tests/pb/test_encoder \
tests/json/test_json \
tests/test_cpp \
tests/test_table \
TESTS=$(C_TESTS) $(CC_TESTS)
tests: $(TESTS)
tests/json/test.upbdefs.o: tests/json/test.upbdefs.c
$(E) CC $<
$(Q) $(CC) $(OPT) $(CSTD) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -c -o $@ $<
tests/testmain.o: tests/testmain.cc
$(E) CXX $<
$(Q) $(CXX) $(OPT) $(CXXSTD) $(WARNFLAGS_CXX) $(CXXFLAGS) $(CPPFLAGS) -c -o $@ $<
$(C_TESTS): % : %.c tests/testmain.o $$(LIBS)
$(E) CC $<
$(Q) $(CC) $(OPT) $(CSTD) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -o $@ tests/testmain.o $< $(LIBS)
$(CC_TESTS): % : %.cc tests/testmain.o $$(LIBS)
$(E) CXX $<
$(Q) $(CXX) $(OPT) $(CXXSTD) $(WARNFLAGS_CXX) $(CPPFLAGS) $(CXXFLAGS) -Wno-deprecated -o $@ tests/testmain.o $< $(LIBS)
# Several of these tests don't actually test these libs, but use them
# incidentally to load a descriptor
LOAD_DESCRIPTOR_LIBS = lib/libupb.pb.a lib/libupb.descriptor.a
# Specify which libs each test depends on.
tests/pb/test_varint: LIBS = lib/libupb.pb.a lib/libupb.a $(EXTRA_LIBS)
tests/test_def: LIBS = $(LOAD_DESCRIPTOR_LIBS) lib/libupb.a $(EXTRA_LIBS)
tests/test_handlers: LIBS = lib/libupb.descriptor.a lib/libupb.a $(EXTRA_LIBS)
tests/pb/test_decoder: LIBS = lib/libupb.pb.a lib/libupb.a $(EXTRA_LIBS)
tests/pb/test_encoder: LIBS = lib/libupb.pb.a lib/libupb.descriptor.a lib/libupb.a $(EXTRA_LIBS)
tests/test_cpp: LIBS = $(LOAD_DESCRIPTOR_LIBS) lib/libupb.a $(EXTRA_LIBS)
tests/test_table: LIBS = lib/libupb.a $(EXTRA_LIBS)
tests/json/test_json: LIBS = lib/libupb.a lib/libupb.json.a tests/json/test.upbdefs.o $(EXTRA_LIBS)
tests/test.proto.pb: tests/test.proto
@# TODO: add .proto file parser to upb so this isn't necessary.
protoc tests/test.proto -otests/test.proto.pb
VARIADIC_TESTS= \
tests/t.test_vs_proto2.googlemessage1 \
tests/t.test_vs_proto2.googlemessage2 \
ifeq ($(RUN_UNDER), valgrind)
RUN_UNDER=valgrind --leak-check=full --error-exitcode=1 --track-origins=yes
endif
test:
@set -e # Abort on error.
@find tests -perm -u+x -type f | while read test; do \
if [ -x ./$$test ] ; then \
echo "RUN $$test"; \
$(RUN_UNDER) ./$$test tests/test.proto.pb || exit 1; \
fi \
done;
@echo "All tests passed!"
obj/conformance_protos: obj/conformance_protos.pb tools/upbc
cd obj && ../tools/upbc conformance_protos.pb && touch conformance_protos
obj/conformance_protos.pb: third_party/protobuf/autogen.sh
protoc -Ithird_party/protobuf/conformance -Ithird_party/protobuf/src --include_imports \
third_party/protobuf/conformance/conformance.proto \
third_party/protobuf/src/google/protobuf/test_messages_proto3.proto \
-o obj/conformance_protos.pb
third_party/protouf/autogen.sh: .gitmodules
git submodule init && git submodule update
tests/conformance_upb: tests/conformance_upb.c lib/libupb.a obj/conformance_protos
$(CC) -o tests/conformance_upb tests/conformance_upb.c -Iobj -I. $(CPPFLAGS) $(CFLAGS) obj/conformance.upb.c obj/google/protobuf/*.upb.c lib/libupb.a
# Lua extension ##################################################################
ifeq ($(shell uname), Darwin)
LUA_LDFLAGS = -undefined dynamic_lookup -flat_namespace
else
LUA_LDFLAGS =
endif
LUAEXTS = \
upb/bindings/lua/upb_c.so \
upb/bindings/lua/upb/pb_c.so \
LUATESTS = \
tests/bindings/lua/test_upb.lua \
tests/bindings/lua/test_upb.pb.lua \
.PHONY: clean_lua testlua lua
testlua: lua
@set -e; \
for test in $(LUATESTS) ; do \
echo LUA $$test; \
LUA_PATH="third_party/lunit/?.lua;upb/bindings/lua/?.lua" \
LUA_CPATH=upb/bindings/lua/?.so \
$(RUN_UNDER) lua $$test; \
done
clean: clean_lua
clean_lua:
@rm -f upb/bindings/lua/upb_c.so
@rm -f upb/bindings/lua/upb/pb_c.so
@rm -f upb/bindings/lua/upb/table_c.so
lua: $(LUAEXTS)
# Right now the core upb module depends on all of these.
# It's a TODO to factor this more cleanly in the code.
LUA_LIB_DEPS = \
lib/libupb.pb_pic.a \
lib/libupb.descriptor_pic.a \
lib/libupb_pic.a \
upb/bindings/lua/upb_c.so: upb/bindings/lua/upb.c upb/bindings/lua/def.c upb/bindings/lua/msg.c $(LUA_LIB_DEPS)
$(E) 'CC upb/bindings/lua/{upb,def,msg}.c'
$(Q) $(CC) $(OPT) $(CSTD) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $^ $(LUA_LDFLAGS)
upb/bindings/lua/upb/table_c.so: upb/bindings/lua/upb/table.c lib/libupb_pic.a
$(E) CC upb/bindings/lua/upb/table.c
$(Q) $(CC) $(OPT) $(CSTD) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $^ $(LUA_LDFLAGS)
upb/bindings/lua/upb/pb_c.so: upb/bindings/lua/upb/pb.c $(LUA_LIB_DEPS)
$(E) CC upb/bindings/lua/upb/pb.c
$(Q) $(CC) $(OPT) $(CSTD) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $^ $(LUA_LDFLAGS)
# Amalgamated source (upb.c/upb.h) ############################################
AMALGAMATE_SRCS=$(upb_SRCS) $(upb_descriptor_SRCS) $(upb_pb_SRCS) $(upb_json_SRCS)
amalgamate: upb.c upb.h
upb.c upb.h: $(AMALGAMATE_SRCS)
$(E) AMALGAMATE $@
$(Q) ./tools/amalgamate.py "" "" $^
amalgamated: upb.c upb.h
$(E) CC upb.c
$(Q) $(CC) -o upb.o -c upb.c $(WARNFLAGS)

@ -1,4 +1,11 @@
#!/bin/bash
# Install the latest version of Bazel.
use_bazel.sh latest
# Log the bazel path and version.
which bazel
bazel version
cd $(dirname $0)/../..
bazel test :all

@ -1,575 +0,0 @@
--[[
Code to generate a C API in:
foo.proto -> foo.upb.h
foo.upb.c
This code is evolving very quickly and so there are lots of little things
that aren't perfect right now. As it settles a little more, the code
quality should improve.
--]]
local upb = require "upb"
local dump_cinit = require "dump_cinit"
local export = {}
local typemap = {
[upb.TYPE_BOOL] = "bool",
[upb.TYPE_FLOAT] = "float",
[upb.TYPE_INT32] = "int32_t",
[upb.TYPE_UINT32] = "uint32_t",
[upb.TYPE_DOUBLE] = "double",
[upb.TYPE_INT64] = "int64_t",
[upb.TYPE_UINT64] = "uint64_t",
[upb.TYPE_STRING] = "upb_stringview",
[upb.TYPE_BYTES] = "upb_stringview",
}
function strip_proto(filename)
return string.gsub(filename, '%.proto$','')
end
local function join(...)
return table.concat({...}, ".")
end
local function to_cident(...)
return string.gsub(join(...), "[%./]", "_")
end
local function to_preproc(...)
return string.upper(to_cident(...))
end
-- Strips away last path element, ie:
-- foo.Bar.Baz -> foo.Bar
local function remove_name(name)
local package_end = 0
for i=1,string.len(name) do
if string.byte(name, i) == string.byte(".", 1) then
package_end = i - 1
end
end
return string.sub(name, 1, package_end)
end
local function enum_value_symbol(enumdef, name)
return to_cident(remove_name(enumdef:full_name())) .. "_" .. name
end
local function dump_enum_vals(enumdef, append)
local enum_vals = {}
for k, v in enumdef:values() do
enum_vals[#enum_vals + 1] = {k, v}
end
table.sort(enum_vals, function(a, b) return a[2] < b[2] end)
-- protobuf convention is that enum values are scoped at the level of the
-- enum itself, to follow C++. Ie, if you have the enum:
-- message Foo {
-- enum E {
-- VAL1 = 1;
-- VAL2 = 2;
-- }
-- }
--
-- The name of VAL1 is Foo.VAL1, not Foo.E.VAL1.
--
-- This seems a bit sketchy, but people often name their enum values
-- accordingly, ie:
--
-- enum Foo {
-- FOO_VAL1 = 1;
-- FOO_VAL2 = 2;
-- }
--
-- So if we don't respect this also, we end up with constants that look like:
--
-- GOOGLE_PROTOBUF_FIELDDESCRIPTORPROTO_TYPE_TYPE_DOUBLE = 1
--
-- (notice the duplicated "TYPE").
local cident = to_cident(remove_name(enumdef:full_name()))
for i, pair in ipairs(enum_vals) do
k, v = pair[1], pair[2]
append(' %s = %d', enum_value_symbol(enumdef, k), v)
if i == #enum_vals then
append('\n')
else
append(',\n')
end
end
end
local function field_default(field)
if field:type() == upb.TYPE_MESSAGE then
return "NULL"
elseif field:type() == upb.TYPE_STRING or
field:type() == upb.TYPE_BYTES then
local default = field:default() or ""
return string.format('upb_stringview_make("%s", strlen("%s"))', field:default(), field:default())
elseif field:type() == upb.TYPE_ENUM then
return enum_value_symbol(field:subdef(), field:default())
else
return field:default();
end
end
local function ctype(field, const)
if const then
const = "const "
else
const = ""
end
if field:label() == upb.LABEL_REPEATED then
return const .. "upb_array*"
elseif field:type() == upb.TYPE_MESSAGE then
if field:containing_type():file() == field:subdef():file() then
return const .. to_cident(field:subdef():full_name()) .. "*"
else
return const .. "struct " .. to_cident(field:subdef():full_name()) .. "*"
end
elseif field:type() == upb.TYPE_ENUM then
return to_cident(field:subdef():full_name())
else
return typemap[field:type()] or "void*"
end
end
local function emit_file_warning(filedef, append)
append('/* This file was generated by upbc (the upb compiler) from the input\n')
append(' * file:\n')
append(' *\n')
append(' * %s\n', filedef:name())
append(' *\n')
append(' * Do not edit -- your changes will be discarded when the file is\n')
append(' * regenerated. */\n\n')
end
local function field_layout_rank(field)
-- Order:
-- 1, 2, 3. primitive fields (8, 4, 1 byte)
-- 4. string fields
-- 5. submessage fields
-- 6. repeated fields
--
-- This has the following nice properties:
--
-- 1. padding alignment is (nearly) minimized.
-- 2. fields that might have defaults (1-4) are segregated
-- from fields that are always zero-initialized (5-7).
--
-- We skip oneof fields, because they are emitted in a separate pass.
local rank
if field:containing_oneof() then
rank = 100 -- These go last (actually we skip them).
elseif field:label() == upb.LABEL_REPEATED then
rank = 6
elseif field:type() == upb.TYPE_MESSAGE then
rank = 5
elseif field:type() == upb.TYPE_STRING or field:type() == upb.TYPE_BYTES then
rank = 4
elseif field:type() == upb.TYPE_BOOL then
rank = 3
elseif field:type() == upb.TYPE_FLOAT or
field:type() == upb.TYPE_INT32 or
field:type() == upb.TYPE_UINT32 then
rank = 2
else
rank = 1
end
-- Break ties with field number.
return (rank * 2^29) + field:number()
end
local function sizeof(field)
if field:label() == upb.LABEL_REPEATED or
field:type() == upb.TYPE_MESSAGE then
return {4, 8}
elseif field:type() == upb.TYPE_STRING or field:type() == upb.TYPE_BYTES then
-- upb_stringview
return {8, 16}
elseif field:type() == upb.TYPE_BOOL then
return {1, 1}
elseif field:type() == upb.TYPE_FLOAT or
field:type() == upb.TYPE_INT32 or
field:type() == upb.TYPE_UINT32 then
return {4, 4}
else
return {8, 8}
end
end
local function sizemax(size, max)
max[1] = math.max(max[1], size[1])
max[2] = math.max(max[2], size[2])
end
local function alignup(val, align)
val[1] = math.ceil(val[1] / align[1]) * align[1]
val[2] = math.ceil(val[2] / align[2]) * align[2]
end
local function copysize(size)
return {size[1], size[2]}
end
local function place(offset, size, max)
alignup(offset, size)
local ret = copysize(offset)
-- add size
offset[1] = offset[1] + size[1]
offset[2] = offset[2] + size[2]
-- track max size
sizemax(size, max)
return ret
end
local function get_field_layout_order(msg)
local field_order = {}
-- Sort fields by rank.
for field in msg:fields() do
table.insert(field_order, field)
end
table.sort(field_order, function(a, b)
return field_layout_rank(a) < field_layout_rank(b)
end)
return field_order
end
local function get_oneof_layout_order(msg)
local oneof_order = {}
-- Sort oneofs by name.
for oneof in msg:oneofs() do
table.insert(oneof_order, oneof)
end
table.sort(oneof_order, function(a, b)
return a:name() < b:name()
end)
return oneof_order
end
local function has_hasbit(field)
if field:containing_type():file():syntax() == upb.SYNTAX_PROTO2 then
return field:label() ~= upb.LABEL_REPEATED and not field:containing_oneof()
else
return false
end
end
local function get_message_layout(msg)
local hasbit_count = 0
local hasbit_indexes = {}
local field_order = get_field_layout_order(msg)
local maxsize = {0, 0}
-- Count hasbits.
for _, field in ipairs(field_order) do
if has_hasbit(field) then
hasbit_indexes[field] = hasbit_count
hasbit_count = hasbit_count + 1
end
end
-- Place hasbits at the beginning.
local offset = math.ceil(hasbit_count / 8)
offset = {offset, offset} -- 32, 64 bit
local offsets = {}
-- Place non-oneof fields.
for _, field in ipairs(field_order) do
if not field:containing_oneof() then
offsets[field] = place(offset, sizeof(field), maxsize)
end
end
-- Place oneof fields.
for oneof in msg:oneofs() do
local oneof_maxsize = {0, 0}
-- Calculate max size.
for field in oneof:fields() do
local size = sizeof(field)
sizemax(size, oneof_maxsize)
end
-- Place discriminator enum and data.
local data = place(offset, oneof_maxsize, maxsize)
local case = place(offset, {4, 4}, maxsize)
offsets[oneof] = case
-- Place oneof fields.
for oneof in msg:oneofs() do
for field in oneof:fields() do
offsets[field] = data
end
end
end
-- Align overall size up to max size.
alignup(offset, maxsize)
local size = copysize(offset)
return hasbit_indexes, offsets, size
end
function get_sizeinit(size)
return string.format("UPB_SIZE(%s, %s)", size[1], size[2])
end
local function write_h_file(filedef, append)
emit_file_warning(filedef, append)
local basename_preproc = to_preproc(filedef:name())
append('#ifndef %s_UPB_H_\n', basename_preproc)
append('#define %s_UPB_H_\n\n', basename_preproc)
append('#include <string.h>\n\n')
append('#include "upb/msg.h"\n\n')
append('#include "upb/decode.h"\n')
append('#include "upb/encode.h"\n')
append('#include "upb/port_def.inc"\n')
append('UPB_BEGIN_EXTERN_C\n\n')
-- Forward-declare types defined in this file.
for msg in filedef:defs(upb.DEF_MSG) do
local msgname = to_cident(msg:full_name())
append('struct %s;\n', msgname)
end
for msg in filedef:defs(upb.DEF_MSG) do
local msgname = to_cident(msg:full_name())
append('typedef struct %s %s;\n', msgname, msgname)
end
-- Forward-declare types not in this file, but used as submessages.
for msg in filedef:defs(upb.DEF_MSG) do
for field in msg:fields() do
if field:type() == upb.TYPE_MESSAGE and
field:subdef():file() ~= filedef then
-- Forward declaration for message type declared in another file.
append('struct %s;\n', to_cident(field:subdef():full_name()))
end
end
end
append('\n')
append("/* Enums */\n\n")
for _, def in ipairs(sorted_defs(filedef:defs(upb.DEF_ENUM))) do
local cident = to_cident(def:full_name())
append('typedef enum {\n')
dump_enum_vals(def, append)
append('} %s;\n\n', cident)
end
for msg in filedef:defs(upb.DEF_MSG) do
local hasbit_indexes, offsets, size = get_message_layout(msg)
append("/* %s */\n\n", msg:full_name())
local msgname = to_cident(msg:full_name())
append('extern const upb_msglayout %s_msginit;\n', msgname)
append('UPB_INLINE %s *%s_new(upb_arena *arena) {\n', msgname, msgname)
append(' return (%s *)upb_msg_new(&%s_msginit, arena);\n', msgname, msgname)
append('}\n')
append('UPB_INLINE %s *%s_parsenew(upb_stringview buf, upb_arena *arena) {\n',
msgname, msgname)
append(' %s *ret = %s_new(arena);\n', msgname, msgname)
append(' return (ret && upb_decode(buf, ret, &%s_msginit)) ? ret : NULL;\n', msgname)
append('}\n')
append('UPB_INLINE char *%s_serialize(const %s *msg, upb_arena *arena, size_t *len) {\n',
msgname, msgname)
append(' return upb_encode(msg, &%s_msginit, arena, len);\n', msgname)
append('}\n')
append('\n')
for oneof in msg:oneofs() do
local fullname = to_cident(oneof:containing_type():full_name() .. "." .. oneof:name())
local offset = offsets[oneof]
append('typedef enum {\n')
for field in oneof:fields() do
append(' %s = %d,\n', fullname .. "_" .. field:name(), field:number())
end
append(' %s_NOT_SET = 0,\n', fullname)
append('} %s_oneofcases;\n', fullname)
append('UPB_INLINE %s_oneofcases %s_%s_case(const %s* msg) { ' ..
'return UPB_FIELD_AT(msg, int, %s); }\n',
fullname, msgname, oneof:name(), msgname, get_sizeinit(offset))
append('\n')
end
for field in msg:fields() do
append('UPB_INLINE %s %s_%s(const %s *msg) {',
ctype(field, true), msgname, field:name(), msgname)
if field:containing_oneof() then
local data_offset = offsets[field]
local case_offset = offsets[field:containing_oneof()]
append(' return UPB_READ_ONEOF(msg, %s, %s, %s, %s, %s); }\n',
ctype(field, true), get_sizeinit(data_offset),
get_sizeinit(case_offset), field:number(), field_default(field))
else
append(' return UPB_FIELD_AT(msg, %s, %s); }\n',
ctype(field, true), get_sizeinit(offsets[field]))
end
end
append('\n')
for field in msg:fields() do
append('UPB_INLINE void %s_set_%s(%s *msg, %s value) { ',
msgname, field:name(), msgname, ctype(field))
if field:containing_oneof() then
local data_offset = offsets[field]
local case_offset = offsets[field:containing_oneof()]
append('UPB_WRITE_ONEOF(msg, %s, %s, value, %s, %s); }\n',
ctype(field), get_sizeinit(data_offset), get_sizeinit(case_offset),
field:number())
else
append('UPB_FIELD_AT(msg, %s, %s) = value; }\n',
ctype(field), get_sizeinit(offsets[field]))
end
end
append('\n\n')
end
append('UPB_END_EXTERN_C\n')
append('\n')
append('#include "upb/port_undef.inc"\n');
append('\n')
append('#endif /* %s_UPB_H_ */\n', basename_preproc)
end
local function write_c_file(filedef, hfilename, append)
emit_file_warning(filedef, append)
append('#include <stddef.h>\n')
append('#include "upb/msg.h"\n')
append('#include "%s"\n', hfilename)
for dep in filedef:dependencies() do
local outbase = strip_proto(dep:name())
append('#include "%s.upb.h"\n', outbase)
end
append('\n')
append('#include "upb/port_def.inc"\n')
append('\n')
for msg in filedef:defs(upb.DEF_MSG) do
local msgname = to_cident(msg:full_name())
local fields_array_ref = "NULL"
local submsgs_array_ref = "NULL"
local oneofs_array_ref = "NULL"
local field_count = 0
local submsg_count = 0
local submsg_set = {}
local submsg_indexes = {}
local hasbit_indexes, offsets, size = get_message_layout(msg)
local oneofs_layout_order = get_oneof_layout_order(msg)
local oneof_count = 0
-- Another sorted array in field number order.
local fields_number_order = {}
for field in msg:fields() do
field_count = field_count + 1
table.insert(fields_number_order, field)
if field:type() == upb.TYPE_MESSAGE then
submsg_count = submsg_count + 1
submsg_set[field:subdef()] = true
end
end
table.sort(fields_number_order, function(a, b)
return a:number() < b:number()
end)
if submsg_count > 0 then
-- TODO(haberman): could save a little bit of space by only generating a
-- "submsgs" array for every strongly-connected component.
local submsgs_array_name = msgname .. "_submsgs"
submsgs_array_ref = "&" .. submsgs_array_name .. "[0]"
append('static const upb_msglayout *const %s[%s] = {\n',
submsgs_array_name, submsg_count)
-- Create a deterministically-sorted array of submessage entries.
local submsg_array = {}
for k, v in pairs(submsg_set) do
table.insert(submsg_array, k)
end
table.sort(submsg_array, function(a, b)
return a:full_name() < b:full_name()
end)
for i, submsg in ipairs(submsg_array) do
append(' &%s_msginit,\n', to_cident(submsg:full_name()))
submsg_indexes[submsg] = i - 1
end
append('};\n\n')
end
if field_count > 0 then
local fields_array_name = msgname .. "__fields"
fields_array_ref = "&" .. fields_array_name .. "[0]"
append('static const upb_msglayout_field %s[%s] = {\n',
fields_array_name, field_count)
for _, field in ipairs(fields_number_order) do
local submsg_index = "0"
if field:type() == upb.TYPE_MESSAGE then
submsg_index = submsg_indexes[field:subdef()]
end
local presence = 0
if has_hasbit(field) then
presence = hasbit_indexes[field] + 1
elseif field:containing_oneof() then
local case_ofs = offsets[field:containing_oneof()]
presence = get_sizeinit({(-case_ofs[1]) - 1, (-case_ofs[2]) - 1})
end
append(' {%s, %s, %s, %s, %s, %s},\n',
field:number(),
get_sizeinit(offsets[field]),
presence,
submsg_index,
field:descriptor_type(),
field:label())
end
append('};\n\n')
end
append('const upb_msglayout %s_msginit = {\n', msgname)
append(' %s,\n', submsgs_array_ref)
append(' %s,\n', fields_array_ref)
append(' %s, %s, %s,\n',
get_sizeinit(size), field_count,
'false' -- TODO: extendable
)
append('};\n\n')
end
append('#include "upb/port_undef.inc"\n')
append('\n')
end
function export.write_gencode(filedef, hfilename, append_h, append_c)
write_h_file(filedef, append_h)
write_c_file(filedef, hfilename, append_c)
end
return export

@ -9,7 +9,6 @@
--]]
local dump_cinit = require "dump_cinit"
local make_c_api = require "make_c_api"
local upb = require "upb"
local generate_upbdefs = false
@ -77,40 +76,16 @@ for _, file in ipairs(files) do
os.execute(string.format("mkdir -p `dirname %s`", outbase))
if generate_upbdefs then
-- Legacy generated defs.
local hfile = open(hfilename)
local cfile = open(cfilename)
assert(generate_upbdefs)
-- Legacy generated defs.
local hfile = open(hfilename)
local cfile = open(cfilename)
local happend = dump_cinit.file_appender(hfile)
local cappend = dump_cinit.file_appender(cfile)
local happend = dump_cinit.file_appender(hfile)
local cappend = dump_cinit.file_appender(cfile)
dump_cinit.dump_defs(file, happend, cappend)
dump_cinit.dump_defs(file, happend, cappend)
hfile:close()
cfile:close()
else
-- Write C API.
hfilename = outbase .. ".upb.h"
cfilename = outbase .. ".upb.c"
if os.getenv("UPBC_VERBOSE") then
print("upbc:")
print(string.format(" source file=%s", src))
print(string.format(" output file base=%s", outbase))
print(string.format(" hfilename=%s", hfilename))
print(string.format(" cfilename=%s", cfilename))
end
local hfile = open(hfilename)
local cfile = open(cfilename)
local happend = dump_cinit.file_appender(hfile)
local cappend = dump_cinit.file_appender(cfile)
make_c_api.write_gencode(file, hfilename, happend, cappend)
hfile:close()
cfile:close()
end
hfile:close()
cfile:close()
end

Loading…
Cancel
Save