Added lua and core32 Travis builds, and rewrote README.md

.travis.yml

@@ -3,4 +3,6 @@ script: ./travis.sh script
 env:
   - UPB_TRAVIS_BUILD=bare
   - UPB_TRAVIS_BUILD=barejit
+  - UPB_TRAVIS_BUILD=core32
   - UPB_TRAVIS_BUILD=withprotobuf
+  - UPB_TRAVIS_BUILD=lua

Makefile

@@ -42,7 +42,7 @@ WITH_JIT=no
 CC=cc
 CXX=c++
 CFLAGS=-std=c99
-CXXFLAGS=-Wno-unused-private-field $(USER_CXXFLAGS)
+CXXFLAGS=-Wno-unused-private-field
 INCLUDE=-I.
 CPPFLAGS=$(INCLUDE) -DNDEBUG -Wall -Wextra -Wno-sign-compare $(USER_CPPFLAGS)
 LDLIBS=-lpthread upb/libupb.a
@@ -87,7 +87,6 @@ clean_leave_profile:
 	@rm -rf obj lib
 	@rm -f benchmark/google_messages.proto.pb benchmark/google_messages.pb.* benchmarks/b.* benchmarks/*.pb*
 	@rm -f $(TESTS) tests/testmain.o tests/t.*
-	@rm -f tests/test.proto.pb
 	@rm -f upb/descriptor.pb
 	@rm -rf tools/upbc deps
 	@rm -rf upb/bindings/python/build
@@ -159,7 +158,7 @@ upb_json_SRCS = \
 
 # If the user doesn't specify an -O setting, we use -O3 for critical-path
 # code and -Os for the rest.
-ifeq (, $(findstring -O, $(USER_CFLAGS)))
+ifeq (, $(findstring -O, $(USER_CPPFLAGS)))
 OPT = -O3
 lib/libupb.a : OPT = -Os
 lib/libupb.descriptor.a : OPT = -Os
@@ -364,18 +363,28 @@ endif
 
 LUAEXTS = \
   upb/bindings/lua/upb.so \
-  upb/bindings/lua/upb.pb.so \
+  upb/bindings/lua/upb/pb.so \
+
+LUATESTS = \
+  tests/bindings/lua/test_upb.lua \
+  tests/bindings/lua/test_upb.pb.lua \
 
 .PHONY: clean_lua testlua lua
 
-testlua:
-	LUA_PATH=tests/bindings/lua/?.lua LUA_CPATH=upb/bindings/lua/?.so lua tests/bindings/lua/upb.lua
+testlua: lua
+	@set -e  # Abort on error.
+	@for test in $(LUATESTS) ; do \
+	  echo LUA $$test; \
+	  LUA_PATH="tests/bindings/lua/lunit/?.lua" \
+	    LUA_CPATH=upb/bindings/lua/?.so \
+	    lua $$test; \
+	done
 
 clean: clean_lua
 clean_lua:
 	@rm -f upb/bindings/lua/upb.lua.h
 	@rm -f upb/bindings/lua/upb.so
-	@rm -f upb/bindings/lua/upb.pb.so
+	@rm -f upb/bindings/lua/upb/pb.so
 
 lua: $(LUAEXTS)
 
@@ -383,13 +392,29 @@ upb/bindings/lua/upb.lua.h:
 	$(E) XXD upb/bindings/lua/upb.lua
 	$(Q) xxd -i < upb/bindings/lua/upb.lua > upb/bindings/lua/upb.lua.h
 
-upb/bindings/lua/upb.so: upb/bindings/lua/upb.c upb/bindings/lua/upb.lua.h lib/libupb.descriptor_pic.a lib/libupb_pic.a lib/libupb.pb_pic.a
-	$(E) CC upb/bindings/lua/upb.c
-	$(Q) $(CC) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $< lib/libupb.pb_pic.a lib/libupb.descriptor_pic.a lib/libupb_pic.a $(LUA_LDFLAGS)
+# Right now the core upb module depends on all of these.
+# It's a TODO to factor this more cleanly in the code.
+LUA_LIB_DEPS = \
+  lib/libupb.pb_pic.a \
+  lib/libupb.descriptor_pic.a \
+  lib/libupb_pic.a \
 
-upb/bindings/lua/upb.pb.so: upb/bindings/lua/upb.pb.c lib/libupb_pic.a lib/libupb.pb_pic.a
+upb/bindings/lua/upb.so: upb/bindings/lua/upb.c upb/bindings/lua/upb.lua.h $(LUA_LIB_DEPS)
+	$(E) CC upb/bindings/lua/upb.c
+	$(Q) $(CC) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $< $(LUA_LDFLAGS) $(LUA_LIB_DEPS)
+
+# TODO: the dependency between upb/pb.so and upb.so is expressed at the
+# .so level, which means that the OS will try to load upb.so when upb/pb.so
+# is loaded.  This is what we want, but getting the paths right is tricky.
+# Basically the dynamic linker needs to be able to find upb.so at:
+#   $(LD_LIBRARY_PATH)/upb/bindings/lua/upb.so
+# So the user has to set both LD_LIBRARY_PATH and LUA_CPATH correctly.
+# Another option would be to require the Lua program to always require
+# "upb" before requiring eg. "upb.pb", and then the dependency would not
+# be expressed at the .so level.
+upb/bindings/lua/upb/pb.so: upb/bindings/lua/upb/pb.c upb/bindings/lua/upb.so
 	$(E) CC upb/bindings/lua/upb.pb.c
-	$(Q) $(CC) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $< $(LUA_LDFLAGS)
+	$(Q) $(CC) $(CPPFLAGS) $(CFLAGS) -fpic -shared -o $@ $< upb/bindings/lua/upb.so $(LUA_LDFLAGS)
 
 
 # Python extension #############################################################

README

@@ -1,69 +0,0 @@
-
-upb - a small, low-level protocol buffer library
-
-BUILDING
-========
-
-To build (the core library is ANSI C99 and has no dependencies):
-$ make
-
-Other useful targets:
-$ make test
-$ make benchmark
-$ make lua (requires lua libraries to be installed)
-
-The tests and benchmarks have the following dependencies
-(Ubuntu package names in parentheses):
-- Google's protobuf compiler + libraries (protobuf-compiler, libprotobuf-dev)
-- Lua binary and libraries (lua5.1, liblua5.1-dev)
-
-Issue tracking is on Google Code:
-  http://code.google.com/p/upb/issues/list
-
-A manual is forthcoming, for now see wiki docs at:
-  https://github.com/haberman/upb/wiki
-
-API
-===
-
-The public C/C++ API is defined by all of the .h files in upb/ except .int.h
-files (which are internal-only).
-
-The .h files define both C and C++ APIs.  Both languages have 100% complete and
-first-class APIs.  The C++ API is a wrapper around the C API, but all of the
-wrapping is done in inline methods in .h files, so there is no overhead to this.
-
-For a more detailed description of the scheme we use to provide both C and C++
-APIs, see: https://github.com/haberman/upb/wiki/CAndCPlusPlusAPI
-
-All of the code that is under upb/ but *not* under upb/bindings/ forms the
-namespace of upb's cross-language public API.  For example, the code in
-upb/descriptor would be exposed as follows:
-
-  * in C/C++:
-     #include "upb/descriptor/X.h"
-
-  * in Lua:
-     require "upb.descriptor"
-
-  * in Python:
-     import upb.descriptor
-
-  * etc.
-
-STABILITY / STATUS
-==================
-
-API and ABI are both subject to change!  Please do not distribute as a shared
-library for this reason (for now at least).
-
-Parsing is becoming stable, but serialization is in very early stages.
-
-The Python bindings are completely broken and Lua only supports schemas for the
-moment.
-
-CONTACT
-=======
-
-Author: Josh Haberman (jhaberman@gmail.com, haberman@google.com)
-See LICENSE for copyright information.

README.md

@@ -0,0 +1,148 @@
+
+# Unleaded - small, fast parsers for the 21st century
+
+[![Build Status](https://travis-ci.org/haberman/upb.svg?branch=master)](https://travis-ci.org/haberman/upb)
+
+Unleaded is a library of fast parsers and serializers.  These
+parsers/serializers are written in C and use every available
+avenue (particularly JIT compilation) to achieve the fastest
+possible speed.  However they are also extremely lightweight
+(less than 100k of object code) and low-overhead.
+
+The library started as a Protocol Buffers library (upb originally
+meant μpb: Micro Protocol Buffers).  It still uses
+protobuf-like schemas as a core abstraction, but **it has expanded
+beyond just Protocol Buffers** to JSON, and other formats are
+planned.
+
+The library itself is written in C, but very idiomatic APIs
+are provided for C++ and popular dynamic languages such as
+Lua.  See the rest of this README for more information about
+these bindings.
+
+Some parts of Unleaded are mature (most notably parsing of
+Protocol Buffers) but others are still immature or nonexistent.
+The core library abstractions are rapidly converging (this
+is saying a lot; it was a long road of about 5 years to make
+this happen), which should make it possible to begin building
+out the encoders and decoders in earnest.
+
+API and ABI are both subject to change!  Please do not distribute
+as a shared library for this reason (for now at least).
+
+## Building the core libraries
+
+The core libraries are pure C99 and have no dependencies.
+
+    $ make
+
+This will create a separate C library for each core library
+in `lib/`.  They are built separately to help your binaries
+slim, so you don't need to link in things you neither want
+or need.
+
+Other useful targets:
+
+    $ make tests
+    $ make test
+
+## How the library is organized
+
+Unleaded tries to stay very small, but also aims to support
+lots of different formats.  We reconcile these goals by
+being *aggresively modular*.  The source tree and the build
+artifacts both reflect this organization:
+
+* **upb**: the core library of handlers and defs (schemas)
+* **upb/pb**: encoders/decoders for Protocol Buffers
+* **upb/json**: encoders/decoders for JSON
+* **upb/descriptor**: building upb defs from protobuf desciptors
+  (ie. descriptor.proto)
+* **upb/bindings/googlepb**: binding to the Google protobuf
+  library.
+* **upb/bindings/lua**: binding to the Lua C API (Lua and LuaJIT).
+* more to come!
+
+## C and C++ API
+
+The public C/C++ API is defined by all of the .h files in
+`upb/` except `.int.h` files (which are internal-only).
+
+The `.h` files define both C and C++ APIs.  Both languages
+have 100% complete and first-class APIs.  The C++ API is a
+wrapper around the C API, but all of the wrapping is done in
+inline methods in `.h` files, so there is no overhead to
+this.
+
+For a more detailed description of the scheme we use to
+provide both C and C++ APIs, see:
+[https://github.com/haberman/upb/wiki/CAndCPlusPlusAPI](CAndCPlusPlusAPI).
+
+All of the code that is under `upb/` but *not* under
+`upb/bindings/` forms the namespace of upb's cross-language
+public API.  For example, the code in upb/descriptor would
+be exposed as follows:
+
+  * **in C/C++:** `#include "upb/descriptor/X.h"`
+  * **in Lua:** `require "upb.descriptor"`
+  * **in Python:** `import upb.descriptor`
+  * etc.
+
+## Google protobuf bindings
+
+Unleaded supports integration with the
+[https://github.com/google/protobuf](Google protobuf library).
+These bindings let you:
+
+* convert protobuf schema objects (`Descriptor`, `FieldDescriptor`, etc).
+  to their Unleaded equivalents (`upb::MessageDef`, `upb::FieldDef`).
+* use Unleaded parsers to populate protobuf generated classes.
+  Unleaded's parsers are much faster than protobuf's `DynamicMessage`.
+  If you are generating C++ with the protobuf compiler, then protobuf's
+  parsers are the same speed or a little faster than Unleaded in JIT
+  mode, but Unleaded will have smaller binaries because you don't
+  have to generate the code ahead of time.
+
+To build the Google protobuf integration you must have the protobuf
+libraries already installed.  Once they are installed run:
+
+    $ make googlepb
+
+To test run:
+
+    $ make googlepbtests
+    $ make test
+
+## Lua bindings
+
+Lua bindings provide Unleaded's functionality to Lua programs.
+The bindings target Lua 5.1, Lua 5.2, LuaJIT, and (soon) Lua 5.3.
+
+Right now the Lua bindings support:
+
+* Building schema objects manually (eg. you can essentially write
+  .proto files natively in Lua).
+* creating message objects.
+* parsing Protocol Buffers into message objects.
+
+Other capabilities (parse/serialize JSON, serialize Protocol Buffers)
+are coming.
+
+To build the Lua bindings, the Lua libraries must be installed.  Once
+they are installed, run:
+
+    $ make lua
+
+Note that if the Lua headers are not in a standard place, you may
+need to pass custom flags:
+
+    $ make lua USER_CPPFLAGS=`pkg-config lua5.2 --cflags`
+
+To test the Lua bindings:
+
+    $ make testlua
+
+## Contact
+
+Author: Josh Haberman ([jhaberman@gmail.com](jhaberman@gmail.com),
+[haberman@google.com](haberman@google.com))

travis.sh

@@ -5,7 +5,8 @@ bare_install() {
   :
 }
 bare_script() {
-  make -j12 tests && make test
+  make -j12 tests
+  make test
 }
 
 # Bare JIT build: no dependencies installed, but JIT enabled.
@@ -13,7 +14,8 @@ barejit_install() {
   :
 }
 barejit_script() {
-  make -j12 tests WITH_JIT=yes && make test
+  make -j12 tests WITH_JIT=yes
+  make test
 }
 
 # Build with Google protobuf support and tests (with JIT).
@@ -26,6 +28,26 @@ withprotobuf_script() {
   make test
 }
 
+# A 32-bit build.  Can only test the core because any dependencies
+# need to be available as 32-bit libs also, which gets hairy fast.
+# Can't enable the JIT because it only supports x64.
+core32_install() {
+  :
+}
+core32_script() {
+  make -j12 tests USER_CPPFLAGS=-m32
+  make test
+}
+
+# A build of Lua and running of Lua tests.
+lua_install() {
+  sudo apt-get update -qq
+  sudo apt-get install lua5.2 liblua5.2-dev
+}
+lua_script() {
+  make -j12 testlua USER_CPPFLAGS=`pkg-config lua5.2 --cflags`
+}
+
 set -e
 set -x
 eval ${UPB_TRAVIS_BUILD}_${1}

upb/bindings/lua/upb.c

@@ -1323,9 +1323,10 @@ static size_t lupb_sizeof(lua_State *L, const upb_fielddef *f) {
     case UPB_TYPE_STRING:
     case UPB_TYPE_BYTES:
     case UPB_TYPE_MESSAGE:
-      lupb_assert(L, false);
-      return 0;
+      break;
   }
+  lupb_assert(L, false);
+  return 0;
 }
 
 static int div_round_up(size_t n, size_t d) {