From 51cf616dab63ba65c30cc58f0e5a61724aa4f731 Mon Sep 17 00:00:00 2001 From: Josh Haberman Date: Wed, 13 May 2015 14:51:35 -0700 Subject: [PATCH] Changes to Lua module loading, and file generation. This change has several parts: 1. Resurrected tools/upbc. The code was all there but the build was broken for open-source. Now you can type "make tools/upbc" and it will build all necessary Lua modules and create a robust shell script for running upbc. 2. Changed Lua module loading to no longer rely on OS-level .so dependencies. The net effect of this is that you now only need to set LUA_PATH and LUA_CPATH; setting LD_LIBRARY_PATH or rpaths is no longer required. Downside: this drops compatibility with Lua 5.1, since it depends on a feature that only exists in Lua >=5.2 (and LuaJIT). 3. Since upbc works again, I fixed the re-generation of the descriptor files (descriptor.upb.h, descriptor.upb.c). "make genfiles" will re-generate these as well as the JIT code generator. 4. Added a Travis test target that ensures that the checked-in generated files are not out of date. I would do this for the Ragel generated file also, but we can't count on all versions of Ragel to necessarily generate identical output. 5. Changed Makefile to no longer automatically run Ragel to regenerate the JSON parser. This is unfortuante, because it's convenient when you're developing the JSON parser. However, "git clone" sometimes skews the timestamps a little bit so that "make" thinks it needs to regenerate these files for a fresh "git clone." This would normally be harmless, but if the user doesn't have Ragel installed, it makes the build fail completely. So now you have to explicitly regenerate the Ragel output. If you want to you can uncomment the auto-generation during development. --- .travis.yml | 1 + Makefile | 87 ++++++---- tests/bindings/lua/test_upb.pb.lua | 4 +- tools/dump_cinit.lua | 2 +- tools/upbc.lua | 6 + travis.sh | 15 ++ upb/bindings/lua/upb.c | 17 +- upb/bindings/lua/upb.lua | 248 +++++++++++++++-------------- upb/bindings/lua/upb/pb.c | 6 +- upb/bindings/lua/upb/pb.lua | 11 ++ upb/bindings/lua/{ => upb}/table.c | 2 +- upb/bindings/lua/upb/table.lua | 11 ++ 12 files changed, 234 insertions(+), 176 deletions(-) create mode 100644 upb/bindings/lua/upb/pb.lua rename upb/bindings/lua/{ => upb}/table.c (99%) create mode 100644 upb/bindings/lua/upb/table.lua diff --git a/.travis.yml b/.travis.yml index 917aa91e3e..b1a0cc765d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,3 +11,4 @@ env: - UPB_TRAVIS_BUILD=lua - UPB_TRAVIS_BUILD=coverage - UPB_TRAVIS_BUILD=warnings + - UPB_TRAVIS_BUILD=genfiles diff --git a/Makefile b/Makefile index e75fe54c51..7d1ef11202 100644 --- a/Makefile +++ b/Makefile @@ -95,16 +95,19 @@ clean_leave_profile: @rm -rf obj lib @rm -f tests/google_message?.h @rm -f $(TESTS) tests/testmain.o tests/t.* - @rm -f upb/descriptor.pb + @rm -f upb/descriptor/descriptor.pb @rm -rf tools/upbc deps @rm -rf upb/bindings/python/build @rm -f upb/bindings/ruby/Makefile @rm -f upb/bindings/ruby/upb.o @rm -f upb/bindings/ruby/upb.so @rm -f upb/bindings/ruby/mkmf.log + @rm -f tests/google_messages.pb.* + @rm -f tests/google_messages.proto.pb + @rm -f upb.c upb.h @find . | grep dSYM | xargs rm -rf -clean: clean_leave_profile +clean: clean_leave_profile clean_lua @rm -rf $(call rwildcard,,*.gcno) $(call rwildcard,,*.gcda) # A little bit of Make voodoo: you can call this from the deps of a patterned @@ -166,8 +169,20 @@ upb_json_SRCS = \ upb/json/parser.c \ upb/json/printer.c \ -upb/json/parser.c: upb/json/parser.rl - $(E) RAGEL $< +# Ideally we could keep this uncommented, but Git apparently sometimes skews +# timestamps slightly at "clone" time, which makes "Make" think that it needs +# to rebuild upb/json/parser.c when it actually doesn't. This would be harmless +# except that the user might not have Ragel installed. +# +# So instead we require an excplicit "make ragel" to rebuild this (for now). +# More pain for people developing upb/json/parser.rl, but less pain for everyone +# else. +# +# upb/json/parser.c: upb/json/parser.rl +# $(E) RAGEL $< +# $(Q) ragel -C -o upb/json/parser.c upb/json/parser.rl +ragel: + $(E) RAGEL upb/json/parser.rl $(Q) ragel -C -o upb/json/parser.c upb/json/parser.rl # If the user doesn't specify an -O setting, we use -O3 for critical-path @@ -219,13 +234,24 @@ upb/descriptor/descriptor.pb: upb/descriptor/descriptor.proto @# TODO: replace with upbc protoc upb/descriptor/descriptor.proto -oupb/descriptor/descriptor.pb -descriptorgen: upb/descriptor/descriptor.pb tools/upbc - @# Regenerate descriptor_const.h - ./tools/upbc -o upb/descriptor/descriptor upb/descriptor/descriptor.pb - -tools/upbc: tools/upbc.c $(LIBUPB) - $(E) CC $< - $(Q) $(CC) $(CFLAGS) $(CPPFLAGS) -o $@ $< $(LIBUPB) +genfiles: upb/descriptor/descriptor.pb tools/upbc + ./tools/upbc upb/descriptor/descriptor.pb upb/descriptor/descriptor google_protobuf_descriptor + lua dynasm/dynasm.lua upb/pb/compile_decoder_x64.dasc > upb/pb/compile_decoder_x64.h || (rm upb/pb/compile_decoder_x64.h ; false) + +# upbc depends on these Lua extensions. +UPBC_LUA_EXTS = \ + upb/bindings/lua/upb_c.so \ + upb/bindings/lua/upb/pb_c.so \ + upb/bindings/lua/upb/table_c.so \ + +tools/upbc: $(UPBC_LUA_EXTS) Makefile + $(E) ECHO tools/upbc + $(Q) echo "#!/bin/sh" > tools/upbc + $(Q) echo 'BASE=`dirname "$$0"`' >> tools/upbc + $(Q) echo 'export LUA_CPATH="$$BASE/../upb/bindings/lua/?.so"' >> tools/upbc + $(Q) echo 'export LUA_PATH="$$BASE/?.lua;$$BASE/../upb/bindings/lua/?.lua"' >> tools/upbc + $(Q) echo 'lua $$BASE/upbc.lua "$$@"' >> tools/upbc + $(Q) chmod a+x tools/upbc examples/msg: examples/msg.c $(LIBUPB) $(E) CC $< @@ -379,14 +405,14 @@ tests/bindings/googlepb/test_vs_proto2.googlemessage2: $(GOOGLEPB_TEST_DEPS) \ # Lua extension ################################################################## ifeq ($(shell uname), Darwin) - LUA_LDFLAGS = -undefined dynamic_lookup + LUA_LDFLAGS = -undefined dynamic_lookup -flat_namespace else LUA_LDFLAGS = endif LUAEXTS = \ - upb/bindings/lua/upb.so \ - upb/bindings/lua/upb/pb.so \ + upb/bindings/lua/upb_c.so \ + upb/bindings/lua/upb/pb_c.so \ LUATESTS = \ tests/bindings/lua/test_upb.lua \ @@ -398,23 +424,19 @@ testlua: lua @set -e # Abort on error. @for test in $(LUATESTS) ; do \ echo LUA $$test; \ - LUA_PATH="tests/bindings/lua/lunit/?.lua" \ + LUA_PATH="tests/bindings/lua/lunit/?.lua;upb/bindings/lua/?.lua" \ LUA_CPATH=upb/bindings/lua/?.so \ lua $$test; \ done clean: clean_lua clean_lua: - @rm -f upb/bindings/lua/upb.lua.h - @rm -f upb/bindings/lua/upb.so - @rm -f upb/bindings/lua/upb/pb.so + @rm -f upb/bindings/lua/upb_c.so + @rm -f upb/bindings/lua/upb/pb_c.so + @rm -f upb/bindings/lua/upb/table_c.so lua: $(LUAEXTS) -upb/bindings/lua/upb.lua.h: - $(E) XXD upb/bindings/lua/upb.lua - $(Q) xxd -i < upb/bindings/lua/upb.lua > upb/bindings/lua/upb.lua.h - # Right now the core upb module depends on all of these. # It's a TODO to factor this more cleanly in the code. LUA_LIB_DEPS = \ @@ -422,22 +444,17 @@ LUA_LIB_DEPS = \ lib/libupb.descriptor_pic.a \ lib/libupb_pic.a \ -upb/bindings/lua/upb.so: upb/bindings/lua/upb.c upb/bindings/lua/upb.lua.h $(LUA_LIB_DEPS) +upb/bindings/lua/upb_c.so: upb/bindings/lua/upb.c $(LUA_LIB_DEPS) $(E) CC upb/bindings/lua/upb.c $(Q) $(CC) $(OPT) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $< $(LUA_LDFLAGS) $(LUA_LIB_DEPS) -# TODO: the dependency between upb/pb.so and upb.so is expressed at the -# .so level, which means that the OS will try to load upb.so when upb/pb.so -# is loaded. This is what we want, but getting the paths right is tricky. -# Basically the dynamic linker needs to be able to find upb.so at: -# $(LD_LIBRARY_PATH)/upb/bindings/lua/upb.so -# So the user has to set both LD_LIBRARY_PATH and LUA_CPATH correctly. -# Another option would be to require the Lua program to always require -# "upb" before requiring eg. "upb.pb", and then the dependency would not -# be expressed at the .so level. -upb/bindings/lua/upb/pb.so: upb/bindings/lua/upb/pb.c upb/bindings/lua/upb.so - $(E) CC upb/bindings/lua/upb.pb.c - $(Q) $(CC) $(OPT) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $< upb/bindings/lua/upb.so $(LUA_LDFLAGS) +upb/bindings/lua/upb/table_c.so: upb/bindings/lua/upb/table.c lib/libupb_pic.a + $(E) CC upb/bindings/lua/upb/table.c + $(Q) $(CC) $(OPT) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $< $(LUA_LDFLAGS) + +upb/bindings/lua/upb/pb_c.so: upb/bindings/lua/upb/pb.c $(LUA_LIB_DEPS) + $(E) CC upb/bindings/lua/upb/pb.c + $(Q) $(CC) $(OPT) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $< $(LUA_LDFLAGS) # Python extension ############################################################# diff --git a/tests/bindings/lua/test_upb.pb.lua b/tests/bindings/lua/test_upb.pb.lua index 02b443a662..7c1c0d0606 100644 --- a/tests/bindings/lua/test_upb.pb.lua +++ b/tests/bindings/lua/test_upb.pb.lua @@ -1,6 +1,8 @@ -local upb = require "upb" +-- Require "pb" first to ensure that the transitive require of "upb" is +-- handled properly by the "pb" module. local pb = require "upb.pb" +local upb = require "upb" local lunit = require "lunit" if _VERSION >= 'Lua 5.2' then diff --git a/tools/dump_cinit.lua b/tools/dump_cinit.lua index 13e1f52ecb..dac2498dd7 100644 --- a/tools/dump_cinit.lua +++ b/tools/dump_cinit.lua @@ -10,7 +10,7 @@ --]] -local upbtable = require "upbtable" +local upbtable = require "upb.table" local upb = require "upb" local export = {} diff --git a/tools/upbc.lua b/tools/upbc.lua index f68d25fe2c..955da57d3b 100644 --- a/tools/upbc.lua +++ b/tools/upbc.lua @@ -19,6 +19,12 @@ local upb = require "upb" local src = arg[1] local outbase = arg[2] local basename = arg[3] + +if not (src and outbase and basename) then + print("Usage: upbc ") + return 1 +end + local hfilename = outbase .. ".upb.h" local cfilename = outbase .. ".upb.c" diff --git a/travis.sh b/travis.sh index 502b2f1640..93a5935dc3 100755 --- a/travis.sh +++ b/travis.sh @@ -59,6 +59,21 @@ lua_script() { make -j12 testlua USER_CPPFLAGS=`pkg-config lua5.2 --cflags` } +# Test that generated files don't need to be regenerated. +# +# We would include the Ragel output here too, but we can't really guarantee +# that its output will be stable for multiple versions of the tool, and we +# don't want the test to be brittle. +genfiles_install() { + sudo apt-get update -qq + sudo apt-get install lua5.2 liblua5.2-dev protobuf-compiler +} +genfiles_script() { + make -j12 genfiles USER_CPPFLAGS=`pkg-config lua5.2 --cflags` + # Will fail if any differences were observed. + git diff --exit-code +} + # A run that executes with coverage support and uploads to coveralls.io coverage_install() { sudo apt-get update -qq diff --git a/upb/bindings/lua/upb.c b/upb/bindings/lua/upb.c index b35af24edd..aebf9f76be 100644 --- a/upb/bindings/lua/upb.c +++ b/upb/bindings/lua/upb.c @@ -33,10 +33,6 @@ #include "upb/pb/glue.h" #include "upb/shim/shim.h" -static const char upb_lua[] = { -#include "upb/bindings/lua/upb.lua.h" -}; - // Lua metatable types. #define LUPB_MSG "lupb.msg" #define LUPB_ARRAY "lupb.array" @@ -1771,9 +1767,9 @@ static void lupb_setfieldi(lua_State *L, const char *field, int i) { lua_setfield(L, -2, field); } -int luaopen_upb(lua_State *L) { +int luaopen_upb_c(lua_State *L) { static char module_key; - if (lupb_openlib(L, &module_key, "upb", lupb_toplevel_m)) { + if (lupb_openlib(L, &module_key, "upb_c", lupb_toplevel_m)) { return 1; } @@ -1858,14 +1854,5 @@ int luaopen_upb(lua_State *L) { lupb_setfieldi(L, "HANDLER_STARTSEQ", UPB_HANDLER_STARTSEQ); lupb_setfieldi(L, "HANDLER_ENDSEQ", UPB_HANDLER_ENDSEQ); - // Call the chunk that will define the extra functions on upb, passing our - // package dictionary as the argument. - if (luaL_loadbuffer(L, upb_lua, sizeof(upb_lua), "upb.lua") || - lua_pcall(L, 0, LUA_MULTRET, 0)) { - lua_error(L); - } - lua_pushvalue(L, -2); - lua_call(L, 1, 0); - return 1; // Return package table. } diff --git a/upb/bindings/lua/upb.lua b/upb/bindings/lua/upb.lua index bda9dfee39..4090b37a6f 100644 --- a/upb/bindings/lua/upb.lua +++ b/upb/bindings/lua/upb.lua @@ -1,146 +1,156 @@ +--[[-------------------------------------------------------------------- + + upb - a minimalist implementation of protocol buffers. + + Copyright (c) 2009 Google Inc. See LICENSE for details. + Author: Josh Haberman + +--------------------------------------------------------------------]]-- + +-- Before calling require on "upb_c", we need to load the same library +-- as RTLD_GLOBAL, for the benefit of other C extensions that depend on +-- C functions in the core. -- --- upb - a minimalist implementation of protocol buffers. --- --- Copyright (c) 2009 Google Inc. See LICENSE for details. --- Author: Josh Haberman +-- This has to happen *before* the require call, because if the module +-- is loaded RTLD_LOCAL first, a subsequent load as RTLD_GLOBAL won't +-- have the proper effect, at least on some platforms. +package.loadlib(package.searchpath("upb_c", package.cpath), "*") + +local upb = require("upb_c") + +-- A convenience function for building/linking/freezing defs +-- while maintaining their original order. -- --- Pure-Lua support functions that are part of the "upb" module. --- This file is embedded and packaged into the "upb" C module binary -- it --- should not be installed or used directly! - -return function(upb) - -- A convenience function for building/linking/freezing defs - -- while maintaining their original order. - -- - -- Sample usage: - -- local m1, m2 = upb.build_defs{ - -- upb.MessageDef{full_name = "M1", fields = { - -- upb.FieldDef{ - -- name = "m2", - -- number = 1, - -- type = upb.TYPE_MESSAGE, - -- subdef_name = ".M2" - -- }, - -- } - -- }, - -- upb.MessageDef{full_name = "M2"} - -- } - upb.build_defs = function(defs) - upb.SymbolTable(defs) - -- Lua 5.2 puts unpack in the table library. - return (unpack or table.unpack)(defs) - end +-- Sample usage: +-- local m1, m2 = upb.build_defs{ +-- upb.MessageDef{full_name = "M1", fields = { +-- upb.FieldDef{ +-- name = "m2", +-- number = 1, +-- type = upb.TYPE_MESSAGE, +-- subdef_name = ".M2" +-- }, +-- } +-- }, +-- upb.MessageDef{full_name = "M2"} +-- } +upb.build_defs = function(defs) + upb.SymbolTable(defs) + -- Lua 5.2 puts unpack in the table library. + return (unpack or table.unpack)(defs) +end - local ipairs_iter = function(array, last_index) - local next_index = last_index + 1 - if next_index > #array then - return nil - end - return next_index, array[next_index] +local ipairs_iter = function(array, last_index) + local next_index = last_index + 1 + if next_index > #array then + return nil end + return next_index, array[next_index] +end - -- For iterating over the indexes and values of a upb.Array. - -- - -- for i, val in upb.ipairs(array) do - -- -- ... - -- end - upb.ipairs = function(array) - return ipairs_iter, array, 0 - end +-- For iterating over the indexes and values of a upb.Array. +-- +-- for i, val in upb.ipairs(array) do +-- -- ... +-- end +upb.ipairs = function(array) + return ipairs_iter, array, 0 +end - local set_named = function(obj, init) - for k, v in pairs(init) do - local func = obj["set_" .. k] - if not func then - error("Cannot set member: " .. k) - end - func(obj, v) +local set_named = function(obj, init) + for k, v in pairs(init) do + local func = obj["set_" .. k] + if not func then + error("Cannot set member: " .. k) end + func(obj, v) end +end + +-- Capture references to the functions we're wrapping. +local RealFieldDef = upb.FieldDef +local RealEnumDef = upb.EnumDef +local RealMessageDef = upb.MessageDef +local RealSymbolTable = upb.SymbolTable - -- Capture references to the functions we're wrapping. - local RealFieldDef = upb.FieldDef - local RealEnumDef = upb.EnumDef - local RealMessageDef = upb.MessageDef - local RealSymbolTable = upb.SymbolTable - - -- FieldDef constructor; a wrapper around the real constructor that can - -- set initial properties. - -- - -- User can specify initialization values like so: - -- upb.FieldDef{label=upb.LABEL_REQUIRED, name="my_field", number=5, - -- type=upb.TYPE_INT32, default_value=12, type_name="Foo"} - upb.FieldDef = function(init) - local f = RealFieldDef() - - if init then - -- Other members are often dependent on type, so set that first. - if init.type then - f:set_type(init.type) - init.type = nil - end - - set_named(f, init) +-- FieldDef constructor; a wrapper around the real constructor that can +-- set initial properties. +-- +-- User can specify initialization values like so: +-- upb.FieldDef{label=upb.LABEL_REQUIRED, name="my_field", number=5, +-- type=upb.TYPE_INT32, default_value=12, type_name="Foo"} +upb.FieldDef = function(init) + local f = RealFieldDef() + + if init then + -- Other members are often dependent on type, so set that first. + if init.type then + f:set_type(init.type) + init.type = nil end - return f + set_named(f, init) end + return f +end - -- MessageDef constructor; a wrapper around the real constructor that can - -- set initial properties. - -- - -- User can specify initialization values like so: - -- upb.MessageDef{full_name="MyMessage", extstart=8000, fields={...}} - upb.MessageDef = function(init) - local m = RealMessageDef() - - if init then - for _, f in pairs(init.fields or {}) do - m:add(f) - end - init.fields = nil - set_named(m, init) +-- MessageDef constructor; a wrapper around the real constructor that can +-- set initial properties. +-- +-- User can specify initialization values like so: +-- upb.MessageDef{full_name="MyMessage", extstart=8000, fields={...}} +upb.MessageDef = function(init) + local m = RealMessageDef() + + if init then + for _, f in pairs(init.fields or {}) do + m:add(f) end + init.fields = nil - return m + set_named(m, init) end - -- EnumDef constructor; a wrapper around the real constructor that can - -- set initial properties. - -- - -- User can specify initialization values like so: - -- upb.EnumDef{full_name="MyEnum", - -- values={ - -- {"FOO_VALUE_1", 1}, - -- {"FOO_VALUE_2", 2} - -- } - -- } - upb.EnumDef = function(init) - local e = RealEnumDef() - - if init then - for _, val in pairs(init.values or {}) do - e:add(val[1], val[2]) - end - init.values = nil - - set_named(e, init) + return m +end + +-- EnumDef constructor; a wrapper around the real constructor that can +-- set initial properties. +-- +-- User can specify initialization values like so: +-- upb.EnumDef{full_name="MyEnum", +-- values={ +-- {"FOO_VALUE_1", 1}, +-- {"FOO_VALUE_2", 2} +-- } +-- } +upb.EnumDef = function(init) + local e = RealEnumDef() + + if init then + for _, val in pairs(init.values or {}) do + e:add(val[1], val[2]) end + init.values = nil - return e + set_named(e, init) end - -- SymbolTable constructor; a wrapper around the real constructor that can - -- add an initial set of defs. - upb.SymbolTable = function(defs) - local s = RealSymbolTable() + return e +end - if defs then - s:add(defs) - end +-- SymbolTable constructor; a wrapper around the real constructor that can +-- add an initial set of defs. +upb.SymbolTable = function(defs) + local s = RealSymbolTable() - return s + if defs then + s:add(defs) end + + return s end + +return upb diff --git a/upb/bindings/lua/upb/pb.c b/upb/bindings/lua/upb/pb.c index 920648f0ec..bf82a9b962 100644 --- a/upb/bindings/lua/upb/pb.c +++ b/upb/bindings/lua/upb/pb.c @@ -92,11 +92,9 @@ static const struct luaL_Reg toplevel_m[] = { {NULL, NULL} }; -int luaopen_upb_pb(lua_State *L) { - luaopen_upb(L); - +int luaopen_upb_pb_c(lua_State *L) { static char module_key; - if (lupb_openlib(L, &module_key, "upb.pb", toplevel_m)) { + if (lupb_openlib(L, &module_key, "upb.pb_c", toplevel_m)) { return 1; } diff --git a/upb/bindings/lua/upb/pb.lua b/upb/bindings/lua/upb/pb.lua new file mode 100644 index 0000000000..66b39097ef --- /dev/null +++ b/upb/bindings/lua/upb/pb.lua @@ -0,0 +1,11 @@ +--[[-------------------------------------------------------------------- + + upb - a minimalist implementation of protocol buffers. + + Copyright (c) 2009 Google Inc. See LICENSE for details. + Author: Josh Haberman + +--------------------------------------------------------------------]]-- + +require "upb" +return require "upb.pb_c" diff --git a/upb/bindings/lua/table.c b/upb/bindings/lua/upb/table.c similarity index 99% rename from upb/bindings/lua/table.c rename to upb/bindings/lua/upb/table.c index e15382bce0..0907f581aa 100644 --- a/upb/bindings/lua/table.c +++ b/upb/bindings/lua/upb/table.c @@ -162,7 +162,7 @@ static const struct luaL_Reg lupbtable_toplevel_m[] = { {NULL, NULL} }; -int luaopen_upbtable(lua_State *L) { +int luaopen_upb_table_c(lua_State *L) { static char module_key; if (lupb_openlib(L, &module_key, "upb.table", lupbtable_toplevel_m)) { return 1; diff --git a/upb/bindings/lua/upb/table.lua b/upb/bindings/lua/upb/table.lua new file mode 100644 index 0000000000..8a0a9c9698 --- /dev/null +++ b/upb/bindings/lua/upb/table.lua @@ -0,0 +1,11 @@ +--[[-------------------------------------------------------------------- + + upb - a minimalist implementation of protocol buffers. + + Copyright (c) 2009 Google Inc. See LICENSE for details. + Author: Josh Haberman + +--------------------------------------------------------------------]]-- + +require "upb" +return require "upb.table_c"