diff --git a/.travis.yml b/.travis.yml index 917aa91e3e..b1a0cc765d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,3 +11,4 @@ env: - UPB_TRAVIS_BUILD=lua - UPB_TRAVIS_BUILD=coverage - UPB_TRAVIS_BUILD=warnings + - UPB_TRAVIS_BUILD=genfiles diff --git a/Makefile b/Makefile index e75fe54c51..7d1ef11202 100644 --- a/Makefile +++ b/Makefile @@ -95,16 +95,19 @@ clean_leave_profile: @rm -rf obj lib @rm -f tests/google_message?.h @rm -f $(TESTS) tests/testmain.o tests/t.* - @rm -f upb/descriptor.pb + @rm -f upb/descriptor/descriptor.pb @rm -rf tools/upbc deps @rm -rf upb/bindings/python/build @rm -f upb/bindings/ruby/Makefile @rm -f upb/bindings/ruby/upb.o @rm -f upb/bindings/ruby/upb.so @rm -f upb/bindings/ruby/mkmf.log + @rm -f tests/google_messages.pb.* + @rm -f tests/google_messages.proto.pb + @rm -f upb.c upb.h @find . | grep dSYM | xargs rm -rf -clean: clean_leave_profile +clean: clean_leave_profile clean_lua @rm -rf $(call rwildcard,,*.gcno) $(call rwildcard,,*.gcda) # A little bit of Make voodoo: you can call this from the deps of a patterned @@ -166,8 +169,20 @@ upb_json_SRCS = \ upb/json/parser.c \ upb/json/printer.c \ -upb/json/parser.c: upb/json/parser.rl - $(E) RAGEL $< +# Ideally we could keep this uncommented, but Git apparently sometimes skews +# timestamps slightly at "clone" time, which makes "Make" think that it needs +# to rebuild upb/json/parser.c when it actually doesn't. This would be harmless +# except that the user might not have Ragel installed. +# +# So instead we require an excplicit "make ragel" to rebuild this (for now). +# More pain for people developing upb/json/parser.rl, but less pain for everyone +# else. +# +# upb/json/parser.c: upb/json/parser.rl +# $(E) RAGEL $< +# $(Q) ragel -C -o upb/json/parser.c upb/json/parser.rl +ragel: + $(E) RAGEL upb/json/parser.rl $(Q) ragel -C -o upb/json/parser.c upb/json/parser.rl # If the user doesn't specify an -O setting, we use -O3 for critical-path @@ -219,13 +234,24 @@ upb/descriptor/descriptor.pb: upb/descriptor/descriptor.proto @# TODO: replace with upbc protoc upb/descriptor/descriptor.proto -oupb/descriptor/descriptor.pb -descriptorgen: upb/descriptor/descriptor.pb tools/upbc - @# Regenerate descriptor_const.h - ./tools/upbc -o upb/descriptor/descriptor upb/descriptor/descriptor.pb - -tools/upbc: tools/upbc.c $(LIBUPB) - $(E) CC $< - $(Q) $(CC) $(CFLAGS) $(CPPFLAGS) -o $@ $< $(LIBUPB) +genfiles: upb/descriptor/descriptor.pb tools/upbc + ./tools/upbc upb/descriptor/descriptor.pb upb/descriptor/descriptor google_protobuf_descriptor + lua dynasm/dynasm.lua upb/pb/compile_decoder_x64.dasc > upb/pb/compile_decoder_x64.h || (rm upb/pb/compile_decoder_x64.h ; false) + +# upbc depends on these Lua extensions. +UPBC_LUA_EXTS = \ + upb/bindings/lua/upb_c.so \ + upb/bindings/lua/upb/pb_c.so \ + upb/bindings/lua/upb/table_c.so \ + +tools/upbc: $(UPBC_LUA_EXTS) Makefile + $(E) ECHO tools/upbc + $(Q) echo "#!/bin/sh" > tools/upbc + $(Q) echo 'BASE=`dirname "$$0"`' >> tools/upbc + $(Q) echo 'export LUA_CPATH="$$BASE/../upb/bindings/lua/?.so"' >> tools/upbc + $(Q) echo 'export LUA_PATH="$$BASE/?.lua;$$BASE/../upb/bindings/lua/?.lua"' >> tools/upbc + $(Q) echo 'lua $$BASE/upbc.lua "$$@"' >> tools/upbc + $(Q) chmod a+x tools/upbc examples/msg: examples/msg.c $(LIBUPB) $(E) CC $< @@ -379,14 +405,14 @@ tests/bindings/googlepb/test_vs_proto2.googlemessage2: $(GOOGLEPB_TEST_DEPS) \ # Lua extension ################################################################## ifeq ($(shell uname), Darwin) - LUA_LDFLAGS = -undefined dynamic_lookup + LUA_LDFLAGS = -undefined dynamic_lookup -flat_namespace else LUA_LDFLAGS = endif LUAEXTS = \ - upb/bindings/lua/upb.so \ - upb/bindings/lua/upb/pb.so \ + upb/bindings/lua/upb_c.so \ + upb/bindings/lua/upb/pb_c.so \ LUATESTS = \ tests/bindings/lua/test_upb.lua \ @@ -398,23 +424,19 @@ testlua: lua @set -e # Abort on error. @for test in $(LUATESTS) ; do \ echo LUA $$test; \ - LUA_PATH="tests/bindings/lua/lunit/?.lua" \ + LUA_PATH="tests/bindings/lua/lunit/?.lua;upb/bindings/lua/?.lua" \ LUA_CPATH=upb/bindings/lua/?.so \ lua $$test; \ done clean: clean_lua clean_lua: - @rm -f upb/bindings/lua/upb.lua.h - @rm -f upb/bindings/lua/upb.so - @rm -f upb/bindings/lua/upb/pb.so + @rm -f upb/bindings/lua/upb_c.so + @rm -f upb/bindings/lua/upb/pb_c.so + @rm -f upb/bindings/lua/upb/table_c.so lua: $(LUAEXTS) -upb/bindings/lua/upb.lua.h: - $(E) XXD upb/bindings/lua/upb.lua - $(Q) xxd -i < upb/bindings/lua/upb.lua > upb/bindings/lua/upb.lua.h - # Right now the core upb module depends on all of these. # It's a TODO to factor this more cleanly in the code. LUA_LIB_DEPS = \ @@ -422,22 +444,17 @@ LUA_LIB_DEPS = \ lib/libupb.descriptor_pic.a \ lib/libupb_pic.a \ -upb/bindings/lua/upb.so: upb/bindings/lua/upb.c upb/bindings/lua/upb.lua.h $(LUA_LIB_DEPS) +upb/bindings/lua/upb_c.so: upb/bindings/lua/upb.c $(LUA_LIB_DEPS) $(E) CC upb/bindings/lua/upb.c $(Q) $(CC) $(OPT) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $< $(LUA_LDFLAGS) $(LUA_LIB_DEPS) -# TODO: the dependency between upb/pb.so and upb.so is expressed at the -# .so level, which means that the OS will try to load upb.so when upb/pb.so -# is loaded. This is what we want, but getting the paths right is tricky. -# Basically the dynamic linker needs to be able to find upb.so at: -# $(LD_LIBRARY_PATH)/upb/bindings/lua/upb.so -# So the user has to set both LD_LIBRARY_PATH and LUA_CPATH correctly. -# Another option would be to require the Lua program to always require -# "upb" before requiring eg. "upb.pb", and then the dependency would not -# be expressed at the .so level. -upb/bindings/lua/upb/pb.so: upb/bindings/lua/upb/pb.c upb/bindings/lua/upb.so - $(E) CC upb/bindings/lua/upb.pb.c - $(Q) $(CC) $(OPT) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $< upb/bindings/lua/upb.so $(LUA_LDFLAGS) +upb/bindings/lua/upb/table_c.so: upb/bindings/lua/upb/table.c lib/libupb_pic.a + $(E) CC upb/bindings/lua/upb/table.c + $(Q) $(CC) $(OPT) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $< $(LUA_LDFLAGS) + +upb/bindings/lua/upb/pb_c.so: upb/bindings/lua/upb/pb.c $(LUA_LIB_DEPS) + $(E) CC upb/bindings/lua/upb/pb.c + $(Q) $(CC) $(OPT) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $< $(LUA_LDFLAGS) # Python extension ############################################################# diff --git a/tests/bindings/lua/test_upb.pb.lua b/tests/bindings/lua/test_upb.pb.lua index 02b443a662..7c1c0d0606 100644 --- a/tests/bindings/lua/test_upb.pb.lua +++ b/tests/bindings/lua/test_upb.pb.lua @@ -1,6 +1,8 @@ -local upb = require "upb" +-- Require "pb" first to ensure that the transitive require of "upb" is +-- handled properly by the "pb" module. local pb = require "upb.pb" +local upb = require "upb" local lunit = require "lunit" if _VERSION >= 'Lua 5.2' then diff --git a/tools/dump_cinit.lua b/tools/dump_cinit.lua index 13e1f52ecb..dac2498dd7 100644 --- a/tools/dump_cinit.lua +++ b/tools/dump_cinit.lua @@ -10,7 +10,7 @@ --]] -local upbtable = require "upbtable" +local upbtable = require "upb.table" local upb = require "upb" local export = {} diff --git a/tools/upbc.lua b/tools/upbc.lua index f68d25fe2c..955da57d3b 100644 --- a/tools/upbc.lua +++ b/tools/upbc.lua @@ -19,6 +19,12 @@ local upb = require "upb" local src = arg[1] local outbase = arg[2] local basename = arg[3] + +if not (src and outbase and basename) then + print("Usage: upbc ") + return 1 +end + local hfilename = outbase .. ".upb.h" local cfilename = outbase .. ".upb.c" diff --git a/travis.sh b/travis.sh index 502b2f1640..93a5935dc3 100755 --- a/travis.sh +++ b/travis.sh @@ -59,6 +59,21 @@ lua_script() { make -j12 testlua USER_CPPFLAGS=`pkg-config lua5.2 --cflags` } +# Test that generated files don't need to be regenerated. +# +# We would include the Ragel output here too, but we can't really guarantee +# that its output will be stable for multiple versions of the tool, and we +# don't want the test to be brittle. +genfiles_install() { + sudo apt-get update -qq + sudo apt-get install lua5.2 liblua5.2-dev protobuf-compiler +} +genfiles_script() { + make -j12 genfiles USER_CPPFLAGS=`pkg-config lua5.2 --cflags` + # Will fail if any differences were observed. + git diff --exit-code +} + # A run that executes with coverage support and uploads to coveralls.io coverage_install() { sudo apt-get update -qq diff --git a/upb/bindings/lua/upb.c b/upb/bindings/lua/upb.c index b35af24edd..aebf9f76be 100644 --- a/upb/bindings/lua/upb.c +++ b/upb/bindings/lua/upb.c @@ -33,10 +33,6 @@ #include "upb/pb/glue.h" #include "upb/shim/shim.h" -static const char upb_lua[] = { -#include "upb/bindings/lua/upb.lua.h" -}; - // Lua metatable types. #define LUPB_MSG "lupb.msg" #define LUPB_ARRAY "lupb.array" @@ -1771,9 +1767,9 @@ static void lupb_setfieldi(lua_State *L, const char *field, int i) { lua_setfield(L, -2, field); } -int luaopen_upb(lua_State *L) { +int luaopen_upb_c(lua_State *L) { static char module_key; - if (lupb_openlib(L, &module_key, "upb", lupb_toplevel_m)) { + if (lupb_openlib(L, &module_key, "upb_c", lupb_toplevel_m)) { return 1; } @@ -1858,14 +1854,5 @@ int luaopen_upb(lua_State *L) { lupb_setfieldi(L, "HANDLER_STARTSEQ", UPB_HANDLER_STARTSEQ); lupb_setfieldi(L, "HANDLER_ENDSEQ", UPB_HANDLER_ENDSEQ); - // Call the chunk that will define the extra functions on upb, passing our - // package dictionary as the argument. - if (luaL_loadbuffer(L, upb_lua, sizeof(upb_lua), "upb.lua") || - lua_pcall(L, 0, LUA_MULTRET, 0)) { - lua_error(L); - } - lua_pushvalue(L, -2); - lua_call(L, 1, 0); - return 1; // Return package table. } diff --git a/upb/bindings/lua/upb.lua b/upb/bindings/lua/upb.lua index bda9dfee39..4090b37a6f 100644 --- a/upb/bindings/lua/upb.lua +++ b/upb/bindings/lua/upb.lua @@ -1,146 +1,156 @@ +--[[-------------------------------------------------------------------- + + upb - a minimalist implementation of protocol buffers. + + Copyright (c) 2009 Google Inc. See LICENSE for details. + Author: Josh Haberman + +--------------------------------------------------------------------]]-- + +-- Before calling require on "upb_c", we need to load the same library +-- as RTLD_GLOBAL, for the benefit of other C extensions that depend on +-- C functions in the core. -- --- upb - a minimalist implementation of protocol buffers. --- --- Copyright (c) 2009 Google Inc. See LICENSE for details. --- Author: Josh Haberman +-- This has to happen *before* the require call, because if the module +-- is loaded RTLD_LOCAL first, a subsequent load as RTLD_GLOBAL won't +-- have the proper effect, at least on some platforms. +package.loadlib(package.searchpath("upb_c", package.cpath), "*") + +local upb = require("upb_c") + +-- A convenience function for building/linking/freezing defs +-- while maintaining their original order. -- --- Pure-Lua support functions that are part of the "upb" module. --- This file is embedded and packaged into the "upb" C module binary -- it --- should not be installed or used directly! - -return function(upb) - -- A convenience function for building/linking/freezing defs - -- while maintaining their original order. - -- - -- Sample usage: - -- local m1, m2 = upb.build_defs{ - -- upb.MessageDef{full_name = "M1", fields = { - -- upb.FieldDef{ - -- name = "m2", - -- number = 1, - -- type = upb.TYPE_MESSAGE, - -- subdef_name = ".M2" - -- }, - -- } - -- }, - -- upb.MessageDef{full_name = "M2"} - -- } - upb.build_defs = function(defs) - upb.SymbolTable(defs) - -- Lua 5.2 puts unpack in the table library. - return (unpack or table.unpack)(defs) - end +-- Sample usage: +-- local m1, m2 = upb.build_defs{ +-- upb.MessageDef{full_name = "M1", fields = { +-- upb.FieldDef{ +-- name = "m2", +-- number = 1, +-- type = upb.TYPE_MESSAGE, +-- subdef_name = ".M2" +-- }, +-- } +-- }, +-- upb.MessageDef{full_name = "M2"} +-- } +upb.build_defs = function(defs) + upb.SymbolTable(defs) + -- Lua 5.2 puts unpack in the table library. + return (unpack or table.unpack)(defs) +end - local ipairs_iter = function(array, last_index) - local next_index = last_index + 1 - if next_index > #array then - return nil - end - return next_index, array[next_index] +local ipairs_iter = function(array, last_index) + local next_index = last_index + 1 + if next_index > #array then + return nil end + return next_index, array[next_index] +end - -- For iterating over the indexes and values of a upb.Array. - -- - -- for i, val in upb.ipairs(array) do - -- -- ... - -- end - upb.ipairs = function(array) - return ipairs_iter, array, 0 - end +-- For iterating over the indexes and values of a upb.Array. +-- +-- for i, val in upb.ipairs(array) do +-- -- ... +-- end +upb.ipairs = function(array) + return ipairs_iter, array, 0 +end - local set_named = function(obj, init) - for k, v in pairs(init) do - local func = obj["set_" .. k] - if not func then - error("Cannot set member: " .. k) - end - func(obj, v) +local set_named = function(obj, init) + for k, v in pairs(init) do + local func = obj["set_" .. k] + if not func then + error("Cannot set member: " .. k) end + func(obj, v) end +end + +-- Capture references to the functions we're wrapping. +local RealFieldDef = upb.FieldDef +local RealEnumDef = upb.EnumDef +local RealMessageDef = upb.MessageDef +local RealSymbolTable = upb.SymbolTable - -- Capture references to the functions we're wrapping. - local RealFieldDef = upb.FieldDef - local RealEnumDef = upb.EnumDef - local RealMessageDef = upb.MessageDef - local RealSymbolTable = upb.SymbolTable - - -- FieldDef constructor; a wrapper around the real constructor that can - -- set initial properties. - -- - -- User can specify initialization values like so: - -- upb.FieldDef{label=upb.LABEL_REQUIRED, name="my_field", number=5, - -- type=upb.TYPE_INT32, default_value=12, type_name="Foo"} - upb.FieldDef = function(init) - local f = RealFieldDef() - - if init then - -- Other members are often dependent on type, so set that first. - if init.type then - f:set_type(init.type) - init.type = nil - end - - set_named(f, init) +-- FieldDef constructor; a wrapper around the real constructor that can +-- set initial properties. +-- +-- User can specify initialization values like so: +-- upb.FieldDef{label=upb.LABEL_REQUIRED, name="my_field", number=5, +-- type=upb.TYPE_INT32, default_value=12, type_name="Foo"} +upb.FieldDef = function(init) + local f = RealFieldDef() + + if init then + -- Other members are often dependent on type, so set that first. + if init.type then + f:set_type(init.type) + init.type = nil end - return f + set_named(f, init) end + return f +end - -- MessageDef constructor; a wrapper around the real constructor that can - -- set initial properties. - -- - -- User can specify initialization values like so: - -- upb.MessageDef{full_name="MyMessage", extstart=8000, fields={...}} - upb.MessageDef = function(init) - local m = RealMessageDef() - - if init then - for _, f in pairs(init.fields or {}) do - m:add(f) - end - init.fields = nil - set_named(m, init) +-- MessageDef constructor; a wrapper around the real constructor that can +-- set initial properties. +-- +-- User can specify initialization values like so: +-- upb.MessageDef{full_name="MyMessage", extstart=8000, fields={...}} +upb.MessageDef = function(init) + local m = RealMessageDef() + + if init then + for _, f in pairs(init.fields or {}) do + m:add(f) end + init.fields = nil - return m + set_named(m, init) end - -- EnumDef constructor; a wrapper around the real constructor that can - -- set initial properties. - -- - -- User can specify initialization values like so: - -- upb.EnumDef{full_name="MyEnum", - -- values={ - -- {"FOO_VALUE_1", 1}, - -- {"FOO_VALUE_2", 2} - -- } - -- } - upb.EnumDef = function(init) - local e = RealEnumDef() - - if init then - for _, val in pairs(init.values or {}) do - e:add(val[1], val[2]) - end - init.values = nil - - set_named(e, init) + return m +end + +-- EnumDef constructor; a wrapper around the real constructor that can +-- set initial properties. +-- +-- User can specify initialization values like so: +-- upb.EnumDef{full_name="MyEnum", +-- values={ +-- {"FOO_VALUE_1", 1}, +-- {"FOO_VALUE_2", 2} +-- } +-- } +upb.EnumDef = function(init) + local e = RealEnumDef() + + if init then + for _, val in pairs(init.values or {}) do + e:add(val[1], val[2]) end + init.values = nil - return e + set_named(e, init) end - -- SymbolTable constructor; a wrapper around the real constructor that can - -- add an initial set of defs. - upb.SymbolTable = function(defs) - local s = RealSymbolTable() + return e +end - if defs then - s:add(defs) - end +-- SymbolTable constructor; a wrapper around the real constructor that can +-- add an initial set of defs. +upb.SymbolTable = function(defs) + local s = RealSymbolTable() - return s + if defs then + s:add(defs) end + + return s end + +return upb diff --git a/upb/bindings/lua/upb/pb.c b/upb/bindings/lua/upb/pb.c index 920648f0ec..bf82a9b962 100644 --- a/upb/bindings/lua/upb/pb.c +++ b/upb/bindings/lua/upb/pb.c @@ -92,11 +92,9 @@ static const struct luaL_Reg toplevel_m[] = { {NULL, NULL} }; -int luaopen_upb_pb(lua_State *L) { - luaopen_upb(L); - +int luaopen_upb_pb_c(lua_State *L) { static char module_key; - if (lupb_openlib(L, &module_key, "upb.pb", toplevel_m)) { + if (lupb_openlib(L, &module_key, "upb.pb_c", toplevel_m)) { return 1; } diff --git a/upb/bindings/lua/upb/pb.lua b/upb/bindings/lua/upb/pb.lua new file mode 100644 index 0000000000..66b39097ef --- /dev/null +++ b/upb/bindings/lua/upb/pb.lua @@ -0,0 +1,11 @@ +--[[-------------------------------------------------------------------- + + upb - a minimalist implementation of protocol buffers. + + Copyright (c) 2009 Google Inc. See LICENSE for details. + Author: Josh Haberman + +--------------------------------------------------------------------]]-- + +require "upb" +return require "upb.pb_c" diff --git a/upb/bindings/lua/table.c b/upb/bindings/lua/upb/table.c similarity index 99% rename from upb/bindings/lua/table.c rename to upb/bindings/lua/upb/table.c index e15382bce0..0907f581aa 100644 --- a/upb/bindings/lua/table.c +++ b/upb/bindings/lua/upb/table.c @@ -162,7 +162,7 @@ static const struct luaL_Reg lupbtable_toplevel_m[] = { {NULL, NULL} }; -int luaopen_upbtable(lua_State *L) { +int luaopen_upb_table_c(lua_State *L) { static char module_key; if (lupb_openlib(L, &module_key, "upb.table", lupbtable_toplevel_m)) { return 1; diff --git a/upb/bindings/lua/upb/table.lua b/upb/bindings/lua/upb/table.lua new file mode 100644 index 0000000000..8a0a9c9698 --- /dev/null +++ b/upb/bindings/lua/upb/table.lua @@ -0,0 +1,11 @@ +--[[-------------------------------------------------------------------- + + upb - a minimalist implementation of protocol buffers. + + Copyright (c) 2009 Google Inc. See LICENSE for details. + Author: Josh Haberman + +--------------------------------------------------------------------]]-- + +require "upb" +return require "upb.table_c"