Changes to Lua module loading, and file generation.

This change has several parts:

1. Resurrected tools/upbc.  The code was all there but the build was
   broken for open-source.  Now you can type "make tools/upbc" and
   it will build all necessary Lua modules and create a robust shell
   script for running upbc.

2. Changed Lua module loading to no longer rely on OS-level .so
   dependencies.  The net effect of this is that you now only need
   to set LUA_PATH and LUA_CPATH; setting LD_LIBRARY_PATH or rpaths
   is no longer required.  Downside: this drops compatibility with
   Lua 5.1, since it depends on a feature that only exists in Lua >=5.2
   (and LuaJIT).

3. Since upbc works again, I fixed the re-generation of the descriptor
   files (descriptor.upb.h, descriptor.upb.c).  "make genfiles" will
   re-generate these as well as the JIT code generator.

4. Added a Travis test target that ensures that the checked-in generated
   files are not out of date.  I would do this for the Ragel generated
   file also, but we can't count on all versions of Ragel to necessarily
   generate identical output.

5. Changed Makefile to no longer automatically run Ragel to regenerate
   the JSON parser.  This is unfortuante, because it's convenient when
   you're developing the JSON parser.  However, "git clone" sometimes
   skews the timestamps a little bit so that "make" thinks it needs to
   regenerate these files for a fresh "git clone."  This would normally
   be harmless, but if the user doesn't have Ragel installed, it makes
   the build fail completely.  So now you have to explicitly regenerate
   the Ragel output.  If you want to you can uncomment the auto-generation
   during development.

.travis.yml

@@ -11,3 +11,4 @@ env:
   - UPB_TRAVIS_BUILD=lua
   - UPB_TRAVIS_BUILD=coverage
   - UPB_TRAVIS_BUILD=warnings
+  - UPB_TRAVIS_BUILD=genfiles

Makefile

@@ -95,16 +95,19 @@ clean_leave_profile:
 	@rm -rf obj lib
 	@rm -f tests/google_message?.h
 	@rm -f $(TESTS) tests/testmain.o tests/t.*
-	@rm -f upb/descriptor.pb
+	@rm -f upb/descriptor/descriptor.pb
 	@rm -rf tools/upbc deps
 	@rm -rf upb/bindings/python/build
 	@rm -f upb/bindings/ruby/Makefile
 	@rm -f upb/bindings/ruby/upb.o
 	@rm -f upb/bindings/ruby/upb.so
 	@rm -f upb/bindings/ruby/mkmf.log
+	@rm -f tests/google_messages.pb.*
+	@rm -f tests/google_messages.proto.pb
+	@rm -f upb.c upb.h
 	@find . | grep dSYM | xargs rm -rf
 
-clean: clean_leave_profile
+clean: clean_leave_profile clean_lua
 	@rm -rf $(call rwildcard,,*.gcno) $(call rwildcard,,*.gcda)
 
 # A little bit of Make voodoo: you can call this from the deps of a patterned
@@ -166,8 +169,20 @@ upb_json_SRCS = \
   upb/json/parser.c \
   upb/json/printer.c \
 
-upb/json/parser.c: upb/json/parser.rl
+# Ideally we could keep this uncommented, but Git apparently sometimes skews
-	$(E) RAGEL $<
+# timestamps slightly at "clone" time, which makes "Make" think that it needs
+# to rebuild upb/json/parser.c when it actually doesn't.  This would be harmless
+# except that the user might not have Ragel installed.
+#
+# So instead we require an excplicit "make ragel" to rebuild this (for now).
+# More pain for people developing upb/json/parser.rl, but less pain for everyone
+# else.
+#
+# upb/json/parser.c: upb/json/parser.rl
+# 	$(E) RAGEL $<
+# 	$(Q) ragel -C -o upb/json/parser.c upb/json/parser.rl
+ragel:
+	$(E) RAGEL upb/json/parser.rl
 	$(Q) ragel -C -o upb/json/parser.c upb/json/parser.rl
 
 # If the user doesn't specify an -O setting, we use -O3 for critical-path
@@ -219,13 +234,24 @@ upb/descriptor/descriptor.pb: upb/descriptor/descriptor.proto
 	@# TODO: replace with upbc
 	protoc upb/descriptor/descriptor.proto -oupb/descriptor/descriptor.pb
 
-descriptorgen: upb/descriptor/descriptor.pb tools/upbc
+genfiles: upb/descriptor/descriptor.pb tools/upbc
-	@# Regenerate descriptor_const.h
+	./tools/upbc upb/descriptor/descriptor.pb upb/descriptor/descriptor google_protobuf_descriptor
-	./tools/upbc -o upb/descriptor/descriptor upb/descriptor/descriptor.pb
+	lua dynasm/dynasm.lua upb/pb/compile_decoder_x64.dasc > upb/pb/compile_decoder_x64.h || (rm upb/pb/compile_decoder_x64.h ; false)
-
+
-tools/upbc: tools/upbc.c $(LIBUPB)
+# upbc depends on these Lua extensions.
-	$(E) CC $<
+UPBC_LUA_EXTS = \
-	$(Q) $(CC) $(CFLAGS) $(CPPFLAGS) -o $@ $< $(LIBUPB)
+  upb/bindings/lua/upb_c.so \
+  upb/bindings/lua/upb/pb_c.so \
+  upb/bindings/lua/upb/table_c.so \
+
+tools/upbc: $(UPBC_LUA_EXTS) Makefile
+	$(E) ECHO tools/upbc
+	$(Q) echo "#!/bin/sh" > tools/upbc
+	$(Q) echo 'BASE=`dirname "$$0"`' >> tools/upbc
+	$(Q) echo 'export LUA_CPATH="$$BASE/../upb/bindings/lua/?.so"' >> tools/upbc
+	$(Q) echo 'export LUA_PATH="$$BASE/?.lua;$$BASE/../upb/bindings/lua/?.lua"' >> tools/upbc
+	$(Q) echo 'lua $$BASE/upbc.lua "$$@"' >> tools/upbc
+	$(Q) chmod a+x tools/upbc
 
 examples/msg: examples/msg.c $(LIBUPB)
 	$(E) CC $<
@@ -379,14 +405,14 @@ tests/bindings/googlepb/test_vs_proto2.googlemessage2: $(GOOGLEPB_TEST_DEPS) \
 # Lua extension ##################################################################
 
 ifeq ($(shell uname), Darwin)
-  LUA_LDFLAGS = -undefined dynamic_lookup
+  LUA_LDFLAGS = -undefined dynamic_lookup -flat_namespace
 else
   LUA_LDFLAGS =
 endif
 
 LUAEXTS = \
-  upb/bindings/lua/upb.so \
+  upb/bindings/lua/upb_c.so \
-  upb/bindings/lua/upb/pb.so \
+  upb/bindings/lua/upb/pb_c.so \
 
 LUATESTS = \
   tests/bindings/lua/test_upb.lua \
@@ -398,23 +424,19 @@ testlua: lua
 	@set -e  # Abort on error.
 	@for test in $(LUATESTS) ; do \
 	  echo LUA $$test; \
-	  LUA_PATH="tests/bindings/lua/lunit/?.lua" \
+	  LUA_PATH="tests/bindings/lua/lunit/?.lua;upb/bindings/lua/?.lua" \
 	    LUA_CPATH=upb/bindings/lua/?.so \
 	    lua $$test; \
 	done
 
 clean: clean_lua
 clean_lua:
-	@rm -f upb/bindings/lua/upb.lua.h
+	@rm -f upb/bindings/lua/upb_c.so
-	@rm -f upb/bindings/lua/upb.so
+	@rm -f upb/bindings/lua/upb/pb_c.so
-	@rm -f upb/bindings/lua/upb/pb.so
+	@rm -f upb/bindings/lua/upb/table_c.so
 
 lua: $(LUAEXTS)
 
-upb/bindings/lua/upb.lua.h:
-	$(E) XXD upb/bindings/lua/upb.lua
-	$(Q) xxd -i < upb/bindings/lua/upb.lua > upb/bindings/lua/upb.lua.h
-
 # Right now the core upb module depends on all of these.
 # It's a TODO to factor this more cleanly in the code.
 LUA_LIB_DEPS = \
@@ -422,22 +444,17 @@ LUA_LIB_DEPS = \
   lib/libupb.descriptor_pic.a \
   lib/libupb_pic.a \
 
-upb/bindings/lua/upb.so: upb/bindings/lua/upb.c upb/bindings/lua/upb.lua.h $(LUA_LIB_DEPS)
+upb/bindings/lua/upb_c.so: upb/bindings/lua/upb.c $(LUA_LIB_DEPS)
 	$(E) CC upb/bindings/lua/upb.c
 	$(Q) $(CC) $(OPT) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $< $(LUA_LDFLAGS) $(LUA_LIB_DEPS)
 
-# TODO: the dependency between upb/pb.so and upb.so is expressed at the
+upb/bindings/lua/upb/table_c.so: upb/bindings/lua/upb/table.c lib/libupb_pic.a
-# .so level, which means that the OS will try to load upb.so when upb/pb.so
+	$(E) CC upb/bindings/lua/upb/table.c
-# is loaded.  This is what we want, but getting the paths right is tricky.
+	$(Q) $(CC) $(OPT) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $< $(LUA_LDFLAGS)
-# Basically the dynamic linker needs to be able to find upb.so at:
+
-#   $(LD_LIBRARY_PATH)/upb/bindings/lua/upb.so
+upb/bindings/lua/upb/pb_c.so: upb/bindings/lua/upb/pb.c $(LUA_LIB_DEPS)
-# So the user has to set both LD_LIBRARY_PATH and LUA_CPATH correctly.
+	$(E) CC upb/bindings/lua/upb/pb.c
-# Another option would be to require the Lua program to always require
+	$(Q) $(CC) $(OPT) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $< $(LUA_LDFLAGS)
-# "upb" before requiring eg. "upb.pb", and then the dependency would not
-# be expressed at the .so level.
-upb/bindings/lua/upb/pb.so: upb/bindings/lua/upb/pb.c upb/bindings/lua/upb.so
-	$(E) CC upb/bindings/lua/upb.pb.c
-	$(Q) $(CC) $(OPT) $(WARNFLAGS) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $< upb/bindings/lua/upb.so $(LUA_LDFLAGS)
 
 
 # Python extension #############################################################

tests/bindings/lua/test_upb.pb.lua

@@ -1,6 +1,8 @@
 
-local upb = require "upb"
+-- Require "pb" first to ensure that the transitive require of "upb" is
+-- handled properly by the "pb" module.
 local pb = require "upb.pb"
+local upb = require "upb"
 local lunit = require "lunit"
 
 if _VERSION >= 'Lua 5.2' then

tools/dump_cinit.lua

@@ -10,7 +10,7 @@
 
 --]]
 
-local upbtable = require "upbtable"
+local upbtable = require "upb.table"
 local upb = require "upb"
 local export = {}
 

tools/upbc.lua

@@ -19,6 +19,12 @@ local upb = require "upb"
 local src = arg[1]
 local outbase = arg[2]
 local basename = arg[3]
+
+if not (src and outbase and basename) then
+  print("Usage: upbc <binary descriptor> <output filename base> <symbol prefix>")
+  return 1
+end
+
 local hfilename = outbase .. ".upb.h"
 local cfilename = outbase .. ".upb.c"
 

travis.sh

@@ -59,6 +59,21 @@ lua_script() {
   make -j12 testlua USER_CPPFLAGS=`pkg-config lua5.2 --cflags`
 }
 
+# Test that generated files don't need to be regenerated.
+#
+# We would include the Ragel output here too, but we can't really guarantee
+# that its output will be stable for multiple versions of the tool, and we
+# don't want the test to be brittle.
+genfiles_install() {
+  sudo apt-get update -qq
+  sudo apt-get install lua5.2 liblua5.2-dev protobuf-compiler
+}
+genfiles_script() {
+  make -j12 genfiles USER_CPPFLAGS=`pkg-config lua5.2 --cflags`
+  # Will fail if any differences were observed.
+  git diff --exit-code
+}
+
 # A run that executes with coverage support and uploads to coveralls.io
 coverage_install() {
   sudo apt-get update -qq

upb/bindings/lua/upb.c

@@ -33,10 +33,6 @@
 #include "upb/pb/glue.h"
 #include "upb/shim/shim.h"
 
-static const char upb_lua[] = {
-#include "upb/bindings/lua/upb.lua.h"
-};
-
 // Lua metatable types.
 #define LUPB_MSG "lupb.msg"
 #define LUPB_ARRAY "lupb.array"
@@ -1771,9 +1767,9 @@ static void lupb_setfieldi(lua_State *L, const char *field, int i) {
   lua_setfield(L, -2, field);
 }
 
-int luaopen_upb(lua_State *L) {
+int luaopen_upb_c(lua_State *L) {
   static char module_key;
-  if (lupb_openlib(L, &module_key, "upb", lupb_toplevel_m)) {
+  if (lupb_openlib(L, &module_key, "upb_c", lupb_toplevel_m)) {
     return 1;
   }
 
@@ -1858,14 +1854,5 @@ int luaopen_upb(lua_State *L) {
   lupb_setfieldi(L, "HANDLER_STARTSEQ",    UPB_HANDLER_STARTSEQ);
   lupb_setfieldi(L, "HANDLER_ENDSEQ",      UPB_HANDLER_ENDSEQ);
 
-  // Call the chunk that will define the extra functions on upb, passing our
-  // package dictionary as the argument.
-  if (luaL_loadbuffer(L, upb_lua, sizeof(upb_lua), "upb.lua") ||
-      lua_pcall(L, 0, LUA_MULTRET, 0)) {
-    lua_error(L);
-  }
-  lua_pushvalue(L, -2);
-  lua_call(L, 1, 0);
-
   return 1;  // Return package table.
 }

upb/bindings/lua/upb.lua

@@ -1,54 +1,63 @@
+--[[--------------------------------------------------------------------
+
+  upb - a minimalist implementation of protocol buffers.
+
+  Copyright (c) 2009 Google Inc.  See LICENSE for details.
+  Author: Josh Haberman <jhaberman@gmail.com>
+
+--------------------------------------------------------------------]]--
+
+-- Before calling require on "upb_c", we need to load the same library
+-- as RTLD_GLOBAL, for the benefit of other C extensions that depend on
+-- C functions in the core.
 --
--- upb - a minimalist implementation of protocol buffers.
+-- This has to happen *before* the require call, because if the module
---
+-- is loaded RTLD_LOCAL first, a subsequent load as RTLD_GLOBAL won't
--- Copyright (c) 2009 Google Inc.  See LICENSE for details.
+-- have the proper effect, at least on some platforms.
--- Author: Josh Haberman <jhaberman@gmail.com>
+package.loadlib(package.searchpath("upb_c", package.cpath), "*")
+
+local upb = require("upb_c")
+
+-- A convenience function for building/linking/freezing defs
+-- while maintaining their original order.
 --
--- Pure-Lua support functions that are part of the "upb" module.
+-- Sample usage:
--- This file is embedded and packaged into the "upb" C module binary -- it
+--   local m1, m2 = upb.build_defs{
--- should not be installed or used directly!
+--     upb.MessageDef{full_name = "M1", fields = {
-
+--         upb.FieldDef{
-return function(upb)
+--           name = "m2",
-  -- A convenience function for building/linking/freezing defs
+--           number = 1,
-  -- while maintaining their original order.
+--           type = upb.TYPE_MESSAGE,
-  --
+--           subdef_name = ".M2"
-  -- Sample usage:
+--         },
-  --   local m1, m2 = upb.build_defs{
+--       }
-  --     upb.MessageDef{full_name = "M1", fields = {
+--     },
-  --         upb.FieldDef{
+--     upb.MessageDef{full_name = "M2"}
-  --           name = "m2",
+--   }
-  --           number = 1,
+upb.build_defs = function(defs)
-  --           type = upb.TYPE_MESSAGE,
-  --           subdef_name = ".M2"
-  --         },
-  --       }
-  --     },
-  --     upb.MessageDef{full_name = "M2"}
-  --   }
-  upb.build_defs = function(defs)
   upb.SymbolTable(defs)
   -- Lua 5.2 puts unpack in the table library.
   return (unpack or table.unpack)(defs)
-  end
+end
 
-  local ipairs_iter = function(array, last_index)
+local ipairs_iter = function(array, last_index)
   local next_index = last_index + 1
   if next_index > #array then
     return nil
   end
   return next_index, array[next_index]
-  end
+end
 
-  -- For iterating over the indexes and values of a upb.Array.
+-- For iterating over the indexes and values of a upb.Array.
-  --
+--
-  -- for i, val in upb.ipairs(array) do
+-- for i, val in upb.ipairs(array) do
-  --   -- ...
+--   -- ...
-  -- end
+-- end
-  upb.ipairs = function(array)
+upb.ipairs = function(array)
   return ipairs_iter, array, 0
-  end
+end
 
-  local set_named = function(obj, init)
+local set_named = function(obj, init)
   for k, v in pairs(init) do
     local func = obj["set_" .. k]
     if not func then
@@ -56,21 +65,21 @@ return function(upb)
     end
     func(obj, v)
   end
-  end
+end
 
-  -- Capture references to the functions we're wrapping.
+-- Capture references to the functions we're wrapping.
-  local RealFieldDef = upb.FieldDef
+local RealFieldDef = upb.FieldDef
-  local RealEnumDef = upb.EnumDef
+local RealEnumDef = upb.EnumDef
-  local RealMessageDef = upb.MessageDef
+local RealMessageDef = upb.MessageDef
-  local RealSymbolTable = upb.SymbolTable
+local RealSymbolTable = upb.SymbolTable
-
+
-  -- FieldDef constructor; a wrapper around the real constructor that can
+-- FieldDef constructor; a wrapper around the real constructor that can
-  -- set initial properties.
+-- set initial properties.
-  --
+--
-  -- User can specify initialization values like so:
+-- User can specify initialization values like so:
-  --   upb.FieldDef{label=upb.LABEL_REQUIRED, name="my_field", number=5,
+--   upb.FieldDef{label=upb.LABEL_REQUIRED, name="my_field", number=5,
-  --                type=upb.TYPE_INT32, default_value=12, type_name="Foo"}
+--                type=upb.TYPE_INT32, default_value=12, type_name="Foo"}
-  upb.FieldDef = function(init)
+upb.FieldDef = function(init)
   local f = RealFieldDef()
 
   if init then
@@ -84,15 +93,15 @@ return function(upb)
   end
 
   return f
-  end
+end
 
 
-  -- MessageDef constructor; a wrapper around the real constructor that can
+-- MessageDef constructor; a wrapper around the real constructor that can
-  -- set initial properties.
+-- set initial properties.
-  --
+--
-  -- User can specify initialization values like so:
+-- User can specify initialization values like so:
-  --   upb.MessageDef{full_name="MyMessage", extstart=8000, fields={...}}
+--   upb.MessageDef{full_name="MyMessage", extstart=8000, fields={...}}
-  upb.MessageDef = function(init)
+upb.MessageDef = function(init)
   local m = RealMessageDef()
 
   if init then
@@ -105,19 +114,19 @@ return function(upb)
   end
 
   return m
-  end
+end
 
-  -- EnumDef constructor; a wrapper around the real constructor that can
+-- EnumDef constructor; a wrapper around the real constructor that can
-  -- set initial properties.
+-- set initial properties.
-  --
+--
-  -- User can specify initialization values like so:
+-- User can specify initialization values like so:
-  --   upb.EnumDef{full_name="MyEnum",
+--   upb.EnumDef{full_name="MyEnum",
-  --     values={
+--     values={
-  --       {"FOO_VALUE_1", 1},
+--       {"FOO_VALUE_1", 1},
-  --       {"FOO_VALUE_2", 2}
+--       {"FOO_VALUE_2", 2}
-  --     }
+--     }
-  --   }
+--   }
-  upb.EnumDef = function(init)
+upb.EnumDef = function(init)
   local e = RealEnumDef()
 
   if init then
@@ -130,11 +139,11 @@ return function(upb)
   end
 
   return e
-  end
+end
 
-  -- SymbolTable constructor; a wrapper around the real constructor that can
+-- SymbolTable constructor; a wrapper around the real constructor that can
-  -- add an initial set of defs.
+-- add an initial set of defs.
-  upb.SymbolTable = function(defs)
+upb.SymbolTable = function(defs)
   local s = RealSymbolTable()
 
   if defs then
@@ -142,5 +151,6 @@ return function(upb)
   end
 
   return s
-  end
 end
+
+return upb

upb/bindings/lua/upb/pb.c

@@ -92,11 +92,9 @@ static const struct luaL_Reg toplevel_m[] = {
   {NULL, NULL}
 };
 
-int luaopen_upb_pb(lua_State *L) {
+int luaopen_upb_pb_c(lua_State *L) {
-  luaopen_upb(L);
-
   static char module_key;
-  if (lupb_openlib(L, &module_key, "upb.pb", toplevel_m)) {
+  if (lupb_openlib(L, &module_key, "upb.pb_c", toplevel_m)) {
     return 1;
   }
 

upb/bindings/lua/upb/pb.lua

@@ -0,0 +1,11 @@
+--[[--------------------------------------------------------------------
+
+  upb - a minimalist implementation of protocol buffers.
+
+  Copyright (c) 2009 Google Inc.  See LICENSE for details.
+  Author: Josh Haberman <jhaberman@gmail.com>
+
+--------------------------------------------------------------------]]--
+
+require "upb"
+return require "upb.pb_c"

upb/bindings/lua/upb/table.c

@@ -162,7 +162,7 @@ static const struct luaL_Reg lupbtable_toplevel_m[] = {
   {NULL, NULL}
 };
 
-int luaopen_upbtable(lua_State *L) {
+int luaopen_upb_table_c(lua_State *L) {
   static char module_key;
   if (lupb_openlib(L, &module_key, "upb.table", lupbtable_toplevel_m)) {
     return 1;

upb/bindings/lua/upb/table.lua

@@ -0,0 +1,11 @@
+--[[--------------------------------------------------------------------
+
+  upb - a minimalist implementation of protocol buffers.
+
+  Copyright (c) 2009 Google Inc.  See LICENSE for details.
+  Author: Josh Haberman <jhaberman@gmail.com>
+
+--------------------------------------------------------------------]]--
+
+require "upb"
+return require "upb.table_c"