From 2d44ad4badee8ad39f113de204c61269acd031ad Mon Sep 17 00:00:00 2001 From: Josh Haberman Date: Thu, 1 Nov 2018 11:42:37 -0700 Subject: [PATCH 01/16] Some fixes to the CMake build. --- CMakeLists.txt | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2909931ecd..ba572fa2ea 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -91,12 +91,12 @@ if(LUA_FOUND) set_target_properties(upb_c PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "upb/bindings/lua" + LIBRARY_OUTPUT_DIRECTORY "lua" PREFIX "" SUFFIX ".so") set_target_properties(table_c pb_c PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "upb/bindings/lua/upb" + LIBRARY_OUTPUT_DIRECTORY "lua/upb" PREFIX "" SUFFIX ".so") @@ -123,15 +123,16 @@ if(LUA_FOUND) DEPENDS ${CMAKE_SOURCE_DIR}/upb/bindings/lua/upb.lua COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/upb/bindings/lua/upb.lua - ${CMAKE_CURRENT_BINARY_DIR}/upb/bindings/lua) + ${CMAKE_CURRENT_BINARY_DIR}/lua) add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/lua/upb/pb.lua + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/lua/upb/table.lua + ${CMAKE_CURRENT_BINARY_DIR}/lua/upb/pb.lua DEPENDS ${CMAKE_SOURCE_DIR}/upb/bindings/lua/upb/table.lua ${CMAKE_SOURCE_DIR}/upb/bindings/lua/upb/pb.lua COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/upb/bindings/lua/upb/table.lua ${CMAKE_SOURCE_DIR}/upb/bindings/lua/upb/pb.lua - ${CMAKE_CURRENT_BINARY_DIR}/upb/bindings/lua/upb) + ${CMAKE_CURRENT_BINARY_DIR}/lua/upb) add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/upb/pb/compile_decoder_x64.h @@ -143,6 +144,15 @@ if(LUA_FOUND) > ${CMAKE_CURRENT_BINARY_DIR}/upb/pb/compile_decoder_x64.h ) + add_custom_target( + upbc ALL + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/tools/make_c_api.lua + ${CMAKE_CURRENT_SOURCE_DIR}/tools/upbc.lua + ${CMAKE_CURRENT_BINARY_DIR}/tools/upbc + ${CMAKE_CURRENT_BINARY_DIR}/lua/upb.lua + ${CMAKE_CURRENT_BINARY_DIR}/lua/upb/pb.lua + ) + add_custom_target( genfiles2 ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/upb/pb/compile_decoder_x64.h @@ -170,15 +180,6 @@ if(LUA_FOUND AND PROTOBUF_FOUND) set(protobuf_BUILD_CONFORMANCE ON CACHE BOOL "Build conformance tests" FORCE) add_subdirectory(third_party/protobuf/cmake) - add_custom_target( - upbc ALL - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/tools/make_c_api.lua - ${CMAKE_CURRENT_SOURCE_DIR}/tools/upbc.lua - ${CMAKE_CURRENT_BINARY_DIR}/tools/upbc - ${CMAKE_CURRENT_BINARY_DIR}/lua/upb.lua - ${CMAKE_CURRENT_BINARY_DIR}/lua/upb/pb.lua - ) - add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/conformance.upb.h ${CMAKE_CURRENT_BINARY_DIR}/google/protobuf/test_messages_proto3.upb.h From 9bb0814c48cb4d4df87ca64c9fabba5d15183cc1 Mon Sep 17 00:00:00 2001 From: Josh Haberman Date: Thu, 1 Nov 2018 11:43:30 -0700 Subject: [PATCH 02/16] Added missing "upbc" file. --- tools/upbc | 5 +++++ 1 file changed, 5 insertions(+) create mode 100755 tools/upbc diff --git a/tools/upbc b/tools/upbc new file mode 100755 index 0000000000..bd12250c79 --- /dev/null +++ b/tools/upbc @@ -0,0 +1,5 @@ +#!/bin/sh +BASE=`dirname "$0"` +export LUA_CPATH="$BASE/../lua/?.so" +export LUA_PATH="$BASE/?.lua;$BASE/../lua/?.lua" +lua $BASE/upbc.lua "$@" From 01ed4ce3d4323c38cb3aeb91053a2a3adb4dfc94 Mon Sep 17 00:00:00 2001 From: Josh Haberman Date: Fri, 2 Nov 2018 12:49:15 -0700 Subject: [PATCH 03/16] WIP. --- BUILD | 183 +++++++++++++++++++++++++++++++++++++++++++++++++ WORKSPACE | 15 ++++ build_defs.bzl | 100 +++++++++++++++++++++++++++ lua.BUILD | 102 +++++++++++++++++++++++++++ 4 files changed, 400 insertions(+) create mode 100644 BUILD create mode 100644 WORKSPACE create mode 100644 build_defs.bzl create mode 100644 lua.BUILD diff --git a/BUILD b/BUILD new file mode 100644 index 0000000000..2dd5818ac0 --- /dev/null +++ b/BUILD @@ -0,0 +1,183 @@ +load(":build_defs.bzl", "lua_cclibrary", "lua_library", "lua_binary") + +cc_library( + name = "upb", + srcs = [ + "google/protobuf/descriptor.upb.c", + "upb/decode.c", + "upb/def.c", + "upb/encode.c", + "upb/handlers.c", + "upb/handlers-inl.h", + "upb/msg.c", + "upb/msgfactory.c", + "upb/port_def.inc", + "upb/port_undef.inc", + "upb/refcounted.c", + "upb/sink.c", + "upb/structdefs.int.h", + "upb/structs.int.h", + "upb/table.c", + "upb/table.int.h", + "upb/upb.c", + ], + hdrs = [ + "google/protobuf/descriptor.upb.h", + "upb/decode.h", + "upb/def.h", + "upb/encode.h", + "upb/handlers.h", + "upb/msg.h", + "upb/msgfactory.h", + "upb/refcounted.h", + "upb/sink.h", + "upb/upb.h", + ], +) + +cc_library( + name = "upb_descriptor", + srcs = [ + "upb/descriptor/descriptor.upbdefs.c", + "upb/descriptor/reader.c", + ], + hdrs = [ + "upb/descriptor/descriptor.upbdefs.h", + "upb/descriptor/reader.h", + ], + deps = [":upb"], +) + +cc_library( + name = "upb_pb", + srcs = [ + "upb/pb/compile_decoder.c", + "upb/pb/decoder.c", + "upb/pb/decoder.int.h", + "upb/pb/encoder.c", + "upb/pb/glue.c", + "upb/pb/textprinter.c", + "upb/pb/varint.c", + "upb/pb/varint.int.h", + ], + hdrs = [ + "upb/pb/decoder.h", + "upb/pb/encoder.h", + "upb/pb/glue.h", + "upb/pb/textprinter.h", + ], + deps = [ + ":upb", + ":upb_descriptor", + ], +) + +cc_library( + name = "upb_json", + srcs = [ + "upb/json/parser.c", + "upb/json/printer.c", + ], + hdrs = [ + "upb/json/parser.h", + "upb/json/printer.h", + ], + deps = [":upb"], +) + +lua_cclibrary( + name = "lua/upb_c", + srcs = [ + "upb/bindings/lua/def.c", + "upb/bindings/lua/msg.c", + "upb/bindings/lua/upb.c", + ], + hdrs = [ + "upb/bindings/lua/upb.h", + ], + deps = [ + "upb", + "upb_pb", + ], +) + +lua_library( + name = "lua/upb", + base = "upb/bindings/lua", + srcs = [ + "upb/bindings/lua/upb.lua" + ], + luadeps = [ + "lua/upb_c", + ], +) + +lua_cclibrary( + name = "lua/upb/table_c", + srcs = [ + "upb/bindings/lua/upb/table.c", + ], + luadeps = [ + "lua/upb_c", + ], + deps = [ + "upb", + ], +) + +lua_library( + name = "lua/upb/table", + base = "upb/bindings/lua", + srcs = [ + "upb/bindings/lua/upb/table.lua", + ], + luadeps = [ + "lua/upb", + "lua/upb/table_c", + ], +) + +lua_cclibrary( + name = "lua/upb/pb_c", + srcs = [ + "upb/bindings/lua/upb/pb.c", + ], + luadeps = [ + "lua/upb_c", + ], + deps = [ + "upb_pb", + ], +) + +lua_library( + name = "lua/upb/pb", + base = "upb/bindings/lua", + srcs = [ + "upb/bindings/lua/upb/pb.lua", + ], + luadeps = [ + "lua/upb", + "lua/upb/pb_c", + ], +) + +lua_library( + name = "lua/upbc_lib", + base = "tools", + srcs = [ + "tools/dump_cinit.lua", + "tools/make_c_api.lua", + ], + luadeps = [ + "lua/upb", + ] +) + +lua_binary( + name = "upbc", + main = "tools/upbc.lua", + luadeps = [ + "lua/upbc_lib", + ] +) diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 0000000000..b03116f8d9 --- /dev/null +++ b/WORKSPACE @@ -0,0 +1,15 @@ + +workspace(name = "upb") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "lua", + build_file = "//:lua.BUILD", + sha256 = "b9e2e4aad6789b3b63a056d442f7b39f0ecfca3ae0f1fc0ae4e9614401b69f4b", + strip_prefix = "lua-5.2.4", + urls = [ + "https://mirror.bazel.build/www.lua.org/ftp/lua-5.2.4.tar.gz", + "https://www.lua.org/ftp/lua-5.2.4.tar.gz", + ], +) diff --git a/build_defs.bzl b/build_defs.bzl new file mode 100644 index 0000000000..6023bf6936 --- /dev/null +++ b/build_defs.bzl @@ -0,0 +1,100 @@ + +shell_find_runfiles = """ + # --- begin runfiles.bash initialization --- + # Copy-pasted from Bazel's Bash runfiles library (tools/bash/runfiles/runfiles.bash). + set -euo pipefail + if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then + if [[ -f "$0.runfiles_manifest" ]]; then + export RUNFILES_MANIFEST_FILE="$0.runfiles_manifest" + elif [[ -f "$0.runfiles/MANIFEST" ]]; then + export RUNFILES_MANIFEST_FILE="$0.runfiles/MANIFEST" + elif [[ -f "$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then + export RUNFILES_DIR="$0.runfiles" + fi + fi + if [[ -f "${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then + source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash" + elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then + source "$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " \ + "$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)" + else + echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash" + exit 1 + fi + # --- end runfiles.bash initialization --- +""" + +def _librule(name): + return name + "_lib" + +def lua_cclibrary(name, srcs, hdrs = [], deps = [], luadeps = []): + lib_rule = name + "_lib" + so_rule = "lib" + name + ".so" + so_file = _remove_prefix(name, "lua/") + ".so" + + native.cc_library( + name = _librule(name), + hdrs = hdrs, + srcs = srcs, + deps = deps + [_librule(dep) for dep in luadeps] + ["@lua//:liblua_headers"], + ) + + native.cc_binary( + name = so_rule, + linkshared = True, + deps = [_librule(name)], + ) + + native.genrule( + name = name + "_copy", + srcs = [":" + so_rule], + outs = [so_file], + cmd = "cp $< $@", + ) + + native.filegroup( + name = name, + data = [so_file], + ) + +def _remove_prefix(str, prefix): + if not str.startswith(prefix): + fail("%s doesn't start with %s" % (str, prefix)) + return str[len(prefix):] + +def lua_library(name, srcs, base, luadeps = []): + outs = [_remove_prefix(src, base + "/") for src in srcs] + native.genrule( + name = name + "_copy", + srcs = srcs, + outs = outs, + cmd = "cp $(SRCS) $(@D)", + ) + + native.filegroup( + name = name, + data = outs + luadeps, + ) + +def lua_binary(name, main, luadeps=[]): + script = name + ".sh" + + script_contents = (shell_find_runfiles + """ +BASE=$(dirname $(rlocation upb/upb_c.so)) +export LUA_CPATH="$BASE/?.so" +export LUA_PATH="$BASE/?.lua" +$(rlocation lua/lua) $(rlocation upb/tools/upbc.lua) "$@" +""").replace("$", "$$") + print(native.repository_name()) + + native.genrule( + name = "gen_" + name, + outs = [script], + cmd = "(cat <<'HEREDOC'\n%s\nHEREDOC\n) > $@" % script_contents, + ) + native.sh_binary( + name = name, + srcs = [script], + data = ["@lua//:lua", "@bazel_tools//tools/bash/runfiles", main] + luadeps, + ) + diff --git a/lua.BUILD b/lua.BUILD new file mode 100644 index 0000000000..7be0b594e5 --- /dev/null +++ b/lua.BUILD @@ -0,0 +1,102 @@ +package( + default_visibility = ["//visibility:public"], +) + +cc_library( + name = "liblua_headers", + defines = ["LUA_USE_LINUX"], + hdrs = [ + "src/lauxlib.h", + "src/lua.h", + "src/lua.hpp", + "src/luaconf.h", + "src/lualib.h", + ], + includes = ["src"], +) + +cc_library( + name = "liblua", + srcs = [ + "src/lapi.c", + "src/lapi.h", + "src/lauxlib.c", + "src/lauxlib.h", + "src/lbaselib.c", + "src/lbitlib.c", + "src/lcode.c", + "src/lcode.h", + "src/lcorolib.c", + "src/lctype.c", + "src/lctype.h", + "src/ldblib.c", + "src/ldebug.c", + "src/ldebug.h", + "src/ldo.c", + "src/ldo.h", + "src/ldump.c", + "src/lfunc.c", + "src/lfunc.h", + "src/lgc.c", + "src/lgc.h", + "src/linit.c", + "src/liolib.c", + "src/llex.c", + "src/llex.h", + "src/llimits.h", + "src/lmathlib.c", + "src/lmem.c", + "src/lmem.h", + "src/loadlib.c", + "src/lobject.c", + "src/lobject.h", + "src/lopcodes.c", + "src/lopcodes.h", + "src/loslib.c", + "src/lparser.c", + "src/lparser.h", + "src/lstate.c", + "src/lstate.h", + "src/lstring.c", + "src/lstring.h", + "src/lstrlib.c", + "src/ltable.c", + "src/ltable.h", + "src/ltablib.c", + "src/ltm.c", + "src/ltm.h", + "src/lundump.c", + "src/lundump.h", + "src/lvm.c", + "src/lvm.h", + "src/lzio.c", + "src/lzio.h", + ], + defines = ["LUA_USE_LINUX"], + hdrs = [ + "src/lauxlib.h", + "src/lua.h", + "src/lua.hpp", + "src/luaconf.h", + "src/lualib.h", + ], + includes = ["src"], + linkopts = [ + "-lm", + "-ldl", + ], +) + +cc_binary( + name = "lua", + srcs = [ + "src/lua.c", + ], + deps = [ + ":liblua", + ], + linkopts = [ + "-lreadline", + "-rdynamic", + ], +) From 2655ed9cf5e448e6ce7195e9afb9361c0a901df0 Mon Sep 17 00:00:00 2001 From: Josh Haberman Date: Sat, 3 Nov 2018 13:04:10 -0700 Subject: [PATCH 04/16] Bazel builds everything except conformance_upb. --- BUILD | 275 +++++++++++++++++++++++++++++++++++++++++++-- WORKSPACE | 19 ++++ build_defs.bzl | 106 ++++++++++++++++- upb/json/parser.c | 26 ++--- upb/json/parser.rl | 2 +- 5 files changed, 398 insertions(+), 30 deletions(-) diff --git a/BUILD b/BUILD index 2dd5818ac0..6f295a0e39 100644 --- a/BUILD +++ b/BUILD @@ -1,4 +1,14 @@ -load(":build_defs.bzl", "lua_cclibrary", "lua_library", "lua_binary") +load( + ":build_defs.bzl", + "lua_cclibrary", + "lua_library", + "lua_binary", + "lua_test", + "generated_file_staleness_test", + "upb_amalgamation", +) + +# C/C++ rules ################################################################## cc_library( name = "upb", @@ -33,6 +43,7 @@ cc_library( "upb/sink.h", "upb/upb.h", ], + copts = ["-std=c89", "-pedantic", "-Wno-long-long"], ) cc_library( @@ -46,6 +57,7 @@ cc_library( "upb/descriptor/reader.h", ], deps = [":upb"], + copts = ["-std=c89", "-pedantic", "-Wno-long-long"], ) cc_library( @@ -70,6 +82,7 @@ cc_library( ":upb", ":upb_descriptor", ], + copts = ["-std=c89", "-pedantic", "-Wno-long-long"], ) cc_library( @@ -83,8 +96,114 @@ cc_library( "upb/json/printer.h", ], deps = [":upb"], + copts = ["-std=c89", "-pedantic", "-Wno-long-long"], ) +cc_library( + name = "upb_cc_bindings", + hdrs = [ + "upb/bindings/stdc++/string.h", + ], + deps = [":upb"], +) + +# Amalgamation ################################################################# + +py_binary( + name = "amalgamate", + srcs = ["tools/amalgamate.py"], +) + +upb_amalgamation( + name = "gen_amalgamation", + amalgamator = ":amalgamate", + libs = [ + ":upb", + ":upb_descriptor", + ":upb_pb", + ":upb_json", + ], + outs = [ + "upb.h", + "upb.c", + ], +) + +cc_library( + name = "amalgamation", + hdrs = ["upb.h"], + srcs = ["upb.c"], +) + +# C/C++ tests ################################################################## + +cc_library( + testonly = 1, + name = "upb_test", + hdrs = [ + "tests/upb_test.h", + "tests/test_util.h", + ], + srcs = [ + "tests/testmain.cc", + ], +) + +cc_test( + name = "test_varint", + srcs = ["tests/pb/test_varint.c"], + deps = [":upb_pb", ":upb_test"], +) + +cc_test( + name = "test_def", + srcs = ["tests/test_def.c"], + deps = [":upb_pb", ":upb_test"], +) + +cc_test( + name = "test_handlers", + srcs = ["tests/test_handlers.c"], + deps = [":upb_pb", ":upb_test"], +) + +cc_test( + name = "test_decoder", + srcs = ["tests/pb/test_decoder.cc"], + deps = [":upb_pb", ":upb_test"], +) + +cc_test( + name = "test_encoder", + srcs = ["tests/pb/test_encoder.cc"], + deps = [":upb_pb", ":upb_test", ":upb_cc_bindings"], + data = ["upb/descriptor/descriptor.pb"], +) + +cc_test( + name = "test_cpp", + srcs = ["tests/test_cpp.cc"], + deps = [":upb_descriptor", ":upb", ":upb_pb", ":upb_test"], +) + +cc_test( + name = "test_table", + srcs = ["tests/test_table.cc"], + deps = [":upb", ":upb_test"], +) + +cc_test( + name = "test_json", + srcs = [ + "tests/json/test_json.cc", + "tests/json/test.upbdefs.h", + "tests/json/test.upbdefs.c", + ], + deps = [":upb_json", ":upb_test"], +) + +# Lua libraries. ############################################################### + lua_cclibrary( name = "lua/upb_c", srcs = [ @@ -103,10 +222,10 @@ lua_cclibrary( lua_library( name = "lua/upb", - base = "upb/bindings/lua", srcs = [ - "upb/bindings/lua/upb.lua" + "upb/bindings/lua/upb.lua", ], + strip_prefix = "upb/bindings/lua", luadeps = [ "lua/upb_c", ], @@ -127,10 +246,10 @@ lua_cclibrary( lua_library( name = "lua/upb/table", - base = "upb/bindings/lua", srcs = [ "upb/bindings/lua/upb/table.lua", ], + strip_prefix = "upb/bindings/lua", luadeps = [ "lua/upb", "lua/upb/table_c", @@ -152,10 +271,10 @@ lua_cclibrary( lua_library( name = "lua/upb/pb", - base = "upb/bindings/lua", srcs = [ "upb/bindings/lua/upb/pb.lua", ], + strip_prefix = "upb/bindings/lua", luadeps = [ "lua/upb", "lua/upb/pb_c", @@ -164,20 +283,158 @@ lua_library( lua_library( name = "lua/upbc_lib", - base = "tools", srcs = [ "tools/dump_cinit.lua", "tools/make_c_api.lua", ], + strip_prefix = "tools", luadeps = [ "lua/upb", - ] + "lua/upb/table", + ], +) + +# Lua tests. ################################################################### + +lua_test( + name = "lua/test_upb", + luamain = "tests/bindings/lua/test_upb.lua", + luadeps = ["lua/upb"], +) + +lua_test( + name = "lua/test_upb_pb", + luamain = "tests/bindings/lua/test_upb.pb.lua", + luadeps = ["lua/upb/pb"], ) +# upb compiler ################################################################# + lua_binary( name = "upbc", - main = "tools/upbc.lua", luadeps = [ "lua/upbc_lib", - ] + ], + luamain = "tools/upbc.lua", +) + +# Generated files ############################################################## + +exports_files(["staleness_test.py"]) + +py_library( + name = "staleness_test_lib", + testonly = 1, + srcs = ["staleness_test_lib.py"], +) + +genrule( + name = "make_dynasm_decoder", + srcs = [ + "third_party/dynasm/dynasm.lua", + "third_party/dynasm/dasm_x64.lua", + "third_party/dynasm/dasm_x86.lua", + "upb/pb/compile_decoder_x64.dasc", + ], + outs = ["generated/upb/pb/compile_decoder_x64.h"], + cmd = "LUA_PATH=third_party/dynasm/?.lua $(location @lua//:lua) third_party/dynasm/dynasm.lua -c upb/pb/compile_decoder_x64.dasc > $@", + tools = ["@lua"], +) + +proto_library( + name = "upb_descriptor_proto", + srcs = [ + "upb/descriptor/descriptor.proto" + ], +) + +genrule( + name = "copy_upb_descriptor_pb", + outs = ["generated/upb/descriptor/descriptor.pb"], + srcs = [":upb_descriptor_proto"], + cmd = "cp $< $@", +) + +genrule( + name = "generate_old_upbdefs", + srcs = ["generated/upb/descriptor/descriptor.pb"], + tools = [":upbc"], + outs = [ + "generated/upb/descriptor/descriptor.upbdefs.h", + "generated/upb/descriptor/descriptor.upbdefs.c", + ], + cmd = "UPBC=$$PWD/$(location :upbc); INFILE=$$PWD/$<; cd $(GENDIR)/generated && $$UPBC --generate-upbdefs $$INFILE", +) + +proto_library( + name = "google_descriptor_proto", + srcs = [ + "google/protobuf/descriptor.proto" + ], +) + +genrule( + name = "copy_google_descriptor_pb", + outs = ["generated/google/protobuf/descriptor.pb"], + srcs = [":google_descriptor_proto"], + cmd = "cp $< $@", +) + +genrule( + name = "generate_descriptor_c", + srcs = ["generated/google/protobuf/descriptor.pb"], + tools = [":upbc"], + outs = [ + "generated/google/protobuf/descriptor.upb.h", + "generated/google/protobuf/descriptor.upb.c", + ], + cmd = "UPBC=$$PWD/$(location :upbc); INFILE=$$PWD/$<; cd $(GENDIR)/generated && $$UPBC $$INFILE", +) + +proto_library( + name = "json_test_proto", + srcs = ["tests/json/test.proto"], +) + +genrule( + name = "copy_json_test_proto", + srcs = [":json_test_proto"], + outs = ["generated/tests/json/test.proto.pb"], + cmd = "cp $< $@", +) + +genrule( + name = "generated_json_test_proto_upbdefs", + srcs = ["generated/tests/json/test.proto.pb"], + tools = [":upbc"], + outs = [ + "generated/tests/json/test.upbdefs.h", + "generated/tests/json/test.upbdefs.c", + ], + cmd = "UPBC=$$PWD/$(location :upbc); INFILE=$$PWD/$<; cd $(GENDIR)/generated && $$UPBC --generate-upbdefs $$INFILE", +) + +genrule( + name = "generate_json_ragel", + srcs = ["upb/json/parser.rl"], + outs = ["generated/upb/json/parser.c"], + tools = ["@ragel//:ragel"], + cmd = "$(location @ragel//:ragel) -C -o upb/json/parser.c $< && mv upb/json/parser.c $@", +) + +generated_file_staleness_test( + name = "test_generated_files", + outs = [ + "google/protobuf/descriptor.upb.h", + "google/protobuf/descriptor.upb.c", + "upb/pb/compile_decoder_x64.h", + "upb/descriptor/descriptor.upbdefs.c", + "upb/descriptor/descriptor.pb", + "upb/descriptor/descriptor.upbdefs.h", + "upb/json/parser.c", + "tests/json/test.upbdefs.c", + "tests/json/test.upbdefs.h", + "tests/json/test.proto.pb", + ], + generated_pattern = "generated/%s", ) diff --git a/WORKSPACE b/WORKSPACE index b03116f8d9..299bfa7567 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -13,3 +13,22 @@ http_archive( "https://www.lua.org/ftp/lua-5.2.4.tar.gz", ], ) + +http_archive( + name = "com_google_protobuf", + sha256 = "d7a221b3d4fb4f05b7473795ccea9e05dab3b8721f6286a95fffbffc2d926f8b", + strip_prefix = "protobuf-3.6.1", + urls = [ + "https://github.com/protocolbuffers/protobuf/archive/v3.6.1.zip" + ], +) + +http_archive( + name = "ragel", + sha256 = "5f156edb65d20b856d638dd9ee2dfb43285914d9aa2b6ec779dac0270cd56c3f", + build_file = "//:ragel.BUILD", + strip_prefix = "ragel-6.10", + urls = [ + "http://www.colm.net/files/ragel/ragel-6.10.tar.gz" + ], +) diff --git a/build_defs.bzl b/build_defs.bzl index 6023bf6936..f9cadd4874 100644 --- a/build_defs.bzl +++ b/build_defs.bzl @@ -62,8 +62,8 @@ def _remove_prefix(str, prefix): fail("%s doesn't start with %s" % (str, prefix)) return str[len(prefix):] -def lua_library(name, srcs, base, luadeps = []): - outs = [_remove_prefix(src, base + "/") for src in srcs] +def lua_library(name, srcs, strip_prefix, luadeps = []): + outs = [_remove_prefix(src, strip_prefix + "/") for src in srcs] native.genrule( name = name + "_copy", srcs = srcs, @@ -76,7 +76,7 @@ def lua_library(name, srcs, base, luadeps = []): data = outs + luadeps, ) -def lua_binary(name, main, luadeps=[]): +def _lua_binary_or_test(name, luamain, luadeps, rule): script = name + ".sh" script_contents = (shell_find_runfiles + """ @@ -85,16 +85,110 @@ export LUA_CPATH="$BASE/?.so" export LUA_PATH="$BASE/?.lua" $(rlocation lua/lua) $(rlocation upb/tools/upbc.lua) "$@" """).replace("$", "$$") - print(native.repository_name()) native.genrule( name = "gen_" + name, outs = [script], cmd = "(cat <<'HEREDOC'\n%s\nHEREDOC\n) > $@" % script_contents, ) - native.sh_binary( + rule( name = name, srcs = [script], - data = ["@lua//:lua", "@bazel_tools//tools/bash/runfiles", main] + luadeps, + data = ["@lua//:lua", "@bazel_tools//tools/bash/runfiles", luamain] + luadeps, ) +def lua_binary(name, luamain, luadeps=[]): + _lua_binary_or_test(name, luamain, luadeps, native.sh_binary) + +def lua_test(name, luamain, luadeps=[]): + _lua_binary_or_test(name, luamain, luadeps, native.sh_test) + +def generated_file_staleness_test(name, outs, generated_pattern): + """Tests that checked-in file(s) match the contents of generated file(s). + + The resulting test will verify that all output files exist and have the + correct contents. If the test fails, it can be invoked with --fix to + bring the checked-in files up to date. + + Args: + name: Name of the rule. + outs: the checked-in files that are copied from generated files. + generated_pattern: the pattern for transforming each "out" file into a + generated file. For example, if generated_pattern="generated/%s" then + a file foo.txt will look for generated file generated/foo.txt. + """ + + script_name = name + ".py" + script_src = "//:staleness_test.py" + + # Filter out non-existing rules so Blaze doesn't error out before we even + # run the test. + existing_outs = native.glob(include = outs) + + # The file list contains a few extra bits of information at the end. + # These get unpacked by the Config class in staleness_test_lib.py. + file_list = outs + [generated_pattern, native.package_name() or ".", name] + + native.genrule( + name = name + "_makescript", + outs = [script_name], + srcs = [script_src], + testonly = 1, + cmd = "cat $(location " + script_src + ") > $@; " + + "sed -i 's|INSERT_FILE_LIST_HERE|" + "\\n ".join(file_list) + "|' $@", + ) + + native.py_test( + name = name, + srcs = [script_name], + data = existing_outs + [generated_pattern % file for file in outs], + deps = [ + "//:staleness_test_lib", + ], + ) + +SrcList = provider( + fields = { + 'srcs' : 'list of srcs', + 'hdrs' : 'list of hdrs', + } +) + +def _file_list_aspect_impl(target, ctx): + srcs = [] + hdrs = [] + for src in ctx.rule.attr.srcs: + srcs += src.files.to_list() + for hdr in ctx.rule.attr.hdrs: + hdrs += hdr.files.to_list() + return [SrcList(srcs = srcs, hdrs = hdrs)] + +_file_list_aspect = aspect( + implementation = _file_list_aspect_impl, +) + +def _upb_amalgamation(ctx): + srcs = [] + hdrs = [] + for lib in ctx.attr.libs: + srcs += lib[SrcList].srcs + hdrs += lib[SrcList].hdrs + ctx.actions.run( + inputs = srcs + hdrs, + outputs = ctx.outputs.outs, + arguments = ["", ctx.bin_dir.path + "/"] + [f.path for f in srcs], + progress_message = "Making amalgamation", + executable = ctx.executable.amalgamator, + ) + +upb_amalgamation = rule( + implementation = _upb_amalgamation, + attrs = { + "amalgamator": attr.label( + executable = True, + cfg = "host", + ), + "libs": attr.label_list(aspects = [_file_list_aspect]), + "outs": attr.output_list(), + } +) diff --git a/upb/json/parser.c b/upb/json/parser.c index 3ae4722377..be743662b2 100644 --- a/upb/json/parser.c +++ b/upb/json/parser.c @@ -2366,7 +2366,7 @@ _match: break; case 2: #line 2023 "upb/json/parser.rl" - { p--; {stack[top++] = cs; cs = 24; goto _again;} } + { p--; {stack[top++] = cs; cs = 24;goto _again;} } break; case 3: #line 2027 "upb/json/parser.rl" @@ -2440,17 +2440,17 @@ _match: #line 2082 "upb/json/parser.rl" { if (is_wellknown_msg(parser, UPB_WELLKNOWN_TIMESTAMP)) { - {stack[top++] = cs; cs = 48; goto _again;} + {stack[top++] = cs; cs = 48;goto _again;} } else if (is_wellknown_msg(parser, UPB_WELLKNOWN_DURATION)) { - {stack[top++] = cs; cs = 41; goto _again;} + {stack[top++] = cs; cs = 41;goto _again;} } else { - {stack[top++] = cs; cs = 33; goto _again;} + {stack[top++] = cs; cs = 33;goto _again;} } } break; case 21: #line 2093 "upb/json/parser.rl" - { p--; {stack[top++] = cs; cs = 76; goto _again;} } + { p--; {stack[top++] = cs; cs = 76;goto _again;} } break; case 22: #line 2098 "upb/json/parser.rl" @@ -2538,7 +2538,9 @@ _again: switch ( *__acts++ ) { case 0: #line 2019 "upb/json/parser.rl" - { p--; {cs = stack[--top]; goto _again;} } + { p--; {cs = stack[--top]; if ( p == pe ) + goto _test_eof; +goto _again;} } break; case 26: #line 2111 "upb/json/parser.rl" @@ -2564,7 +2566,7 @@ _again: #line 2139 "upb/json/parser.rl" { end_subobject_full(parser); } break; -#line 2568 "upb/json/parser.c" +#line 2570 "upb/json/parser.c" } } } @@ -2589,7 +2591,7 @@ error: return p - buf; } -bool end(void *closure, const void *hd) { +static bool end(void *closure, const void *hd) { upb_json_parser *parser = closure; /* Prevent compile warning on unused static constants. */ @@ -2603,11 +2605,7 @@ bool end(void *closure, const void *hd) { parse(parser, hd, &eof_ch, 0, NULL); - return parser->current_state >= -#line 2608 "upb/json/parser.c" -105 -#line 2202 "upb/json/parser.rl" -; + return parser->current_state >= 105; } static void json_parser_reset(upb_json_parser *p) { @@ -2622,7 +2620,7 @@ static void json_parser_reset(upb_json_parser *p) { /* Emit Ragel initialization of the parser. */ -#line 2626 "upb/json/parser.c" +#line 2624 "upb/json/parser.c" { cs = json_start; top = 0; diff --git a/upb/json/parser.rl b/upb/json/parser.rl index 29efd09dae..0d59082763 100644 --- a/upb/json/parser.rl +++ b/upb/json/parser.rl @@ -2185,7 +2185,7 @@ error: return p - buf; } -bool end(void *closure, const void *hd) { +static bool end(void *closure, const void *hd) { upb_json_parser *parser = closure; /* Prevent compile warning on unused static constants. */ From a7ebe54eb1cc09cc50cf97af8436835ead4cbbee Mon Sep 17 00:00:00 2001 From: Josh Haberman Date: Sat, 3 Nov 2018 18:18:47 -0700 Subject: [PATCH 05/16] Conformance tests work under Bazel. --- BUILD | 79 +++++++++++++++++++++------------- WORKSPACE | 24 +++++++---- build_defs.bzl | 95 +++++++++++++++++++++++++++++++++++++---- tests/conformance_upb.c | 4 +- tools/upbc.lua | 22 +++++++--- 5 files changed, 170 insertions(+), 54 deletions(-) diff --git a/BUILD b/BUILD index 6f295a0e39..b636e15518 100644 --- a/BUILD +++ b/BUILD @@ -5,7 +5,9 @@ load( "lua_binary", "lua_test", "generated_file_staleness_test", + "make_shell_script", "upb_amalgamation", + "upb_proto_library", ) # C/C++ rules ################################################################## @@ -202,6 +204,43 @@ cc_test( deps = [":upb_json", ":upb_test"], ) +upb_proto_library( + name = "conformance_proto_upb", + deps = [ + "@com_google_protobuf//:conformance_proto", + "@com_google_protobuf//:test_messages_proto3_proto", + ], + upbc = ":upbc", +) + +cc_binary( + name = "conformance_upb", + srcs = [ + "tests/conformance_upb.c", + ], + deps = [ + ":upb", + ":conformance_proto_upb", + ], +) + +make_shell_script( + name = "gen_test_conformance_upb", + out = "test_conformance_upb.sh", + contents = "$(rlocation com_google_protobuf/conformance_test_runner) $(rlocation upb/conformance_upb)", +) + +sh_test( + name = "test_conformance_upb", + srcs = ["test_conformance_upb.sh"], + data = [ + ":conformance_upb", + "@com_google_protobuf//:conformance_test_runner", + "tests/conformance_upb_failures.txt", + "@bazel_tools//tools/bash/runfiles" + ] +) + # Lua libraries. ############################################################### lua_cclibrary( @@ -222,33 +261,21 @@ lua_cclibrary( lua_library( name = "lua/upb", - srcs = [ - "upb/bindings/lua/upb.lua", - ], + srcs = ["upb/bindings/lua/upb.lua"], strip_prefix = "upb/bindings/lua", - luadeps = [ - "lua/upb_c", - ], + luadeps = ["lua/upb_c"], ) lua_cclibrary( name = "lua/upb/table_c", - srcs = [ - "upb/bindings/lua/upb/table.c", - ], - luadeps = [ - "lua/upb_c", - ], - deps = [ - "upb", - ], + srcs = ["upb/bindings/lua/upb/table.c"], + luadeps = ["lua/upb_c"], + deps = ["upb"], ) lua_library( name = "lua/upb/table", - srcs = [ - "upb/bindings/lua/upb/table.lua", - ], + srcs = ["upb/bindings/lua/upb/table.lua"], strip_prefix = "upb/bindings/lua", luadeps = [ "lua/upb", @@ -258,22 +285,14 @@ lua_library( lua_cclibrary( name = "lua/upb/pb_c", - srcs = [ - "upb/bindings/lua/upb/pb.c", - ], - luadeps = [ - "lua/upb_c", - ], - deps = [ - "upb_pb", - ], + srcs = ["upb/bindings/lua/upb/pb.c"], + luadeps = ["lua/upb_c"], + deps = ["upb_pb"], ) lua_library( name = "lua/upb/pb", - srcs = [ - "upb/bindings/lua/upb/pb.lua", - ], + srcs = ["upb/bindings/lua/upb/pb.lua"], strip_prefix = "upb/bindings/lua", luadeps = [ "lua/upb", diff --git a/WORKSPACE b/WORKSPACE index 299bfa7567..b4d19c50a8 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -2,6 +2,7 @@ workspace(name = "upb") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") http_archive( name = "lua", @@ -14,13 +15,12 @@ http_archive( ], ) -http_archive( +git_repository( name = "com_google_protobuf", - sha256 = "d7a221b3d4fb4f05b7473795ccea9e05dab3b8721f6286a95fffbffc2d926f8b", - strip_prefix = "protobuf-3.6.1", - urls = [ - "https://github.com/protocolbuffers/protobuf/archive/v3.6.1.zip" - ], + commit = "25feb59620627b673df76813dfd66e3f565765e7", + #sha256 = "d7a221b3d4fb4f05b7473795ccea9e05dab3b8721f6286a95fffbffc2d926f8b", + remote = "https://github.com/haberman/protobuf.git", + #tag = "conformance-build-tag", ) http_archive( @@ -28,7 +28,13 @@ http_archive( sha256 = "5f156edb65d20b856d638dd9ee2dfb43285914d9aa2b6ec779dac0270cd56c3f", build_file = "//:ragel.BUILD", strip_prefix = "ragel-6.10", - urls = [ - "http://www.colm.net/files/ragel/ragel-6.10.tar.gz" - ], + urls = ["http://www.colm.net/files/ragel/ragel-6.10.tar.gz"], +) + +# Used by protobuf. +http_archive( + name = "bazel_skylib", + sha256 = "bbccf674aa441c266df9894182d80de104cabd19be98be002f6d478aaa31574d", + strip_prefix = "bazel-skylib-2169ae1c374aab4a09aa90e65efe1a3aad4e279b", + urls = ["https://github.com/bazelbuild/bazel-skylib/archive/2169ae1c374aab4a09aa90e65efe1a3aad4e279b.tar.gz"], ) diff --git a/build_defs.bzl b/build_defs.bzl index f9cadd4874..8df5660a64 100644 --- a/build_defs.bzl +++ b/build_defs.bzl @@ -1,5 +1,5 @@ -shell_find_runfiles = """ +_shell_find_runfiles = """ # --- begin runfiles.bash initialization --- # Copy-pasted from Bazel's Bash runfiles library (tools/bash/runfiles/runfiles.bash). set -euo pipefail @@ -62,6 +62,11 @@ def _remove_prefix(str, prefix): fail("%s doesn't start with %s" % (str, prefix)) return str[len(prefix):] +def _remove_suffix(str, suffix): + if not str.endswith(suffix): + fail("%s doesn't end with %s" % (str, suffix)) + return str[:-len(suffix)] + def lua_library(name, srcs, strip_prefix, luadeps = []): outs = [_remove_prefix(src, strip_prefix + "/") for src in srcs] native.genrule( @@ -76,21 +81,28 @@ def lua_library(name, srcs, strip_prefix, luadeps = []): data = outs + luadeps, ) +def make_shell_script(name, contents, out): + script_contents = (_shell_find_runfiles + contents).replace("$", "$$") + native.genrule( + name = "gen_" + name, + outs = [out], + cmd = "(cat <<'HEREDOC'\n%s\nHEREDOC\n) > $@" % script_contents, + ) + def _lua_binary_or_test(name, luamain, luadeps, rule): script = name + ".sh" - script_contents = (shell_find_runfiles + """ + make_shell_script( + name = "gen_" + name, + out = script, + contents = """ BASE=$(dirname $(rlocation upb/upb_c.so)) export LUA_CPATH="$BASE/?.so" export LUA_PATH="$BASE/?.lua" $(rlocation lua/lua) $(rlocation upb/tools/upbc.lua) "$@" -""").replace("$", "$$") - - native.genrule( - name = "gen_" + name, - outs = [script], - cmd = "(cat <<'HEREDOC'\n%s\nHEREDOC\n) > $@" % script_contents, +""" ) + rule( name = name, srcs = [script], @@ -147,6 +159,8 @@ def generated_file_staleness_test(name, outs, generated_pattern): ], ) +# upb_amalgamation() rule, with file_list aspect. + SrcList = provider( fields = { 'srcs' : 'list of srcs', @@ -192,3 +206,68 @@ upb_amalgamation = rule( "outs": attr.output_list(), } ) + +# upb_proto_library() rule + +def _remove_up(string): + if string.startswith("../"): + string = string[3:] + pos = string.find("/") + string = string[pos + 1:] + + return _remove_suffix(string, ".proto") + +def _upb_proto_library_srcs_impl(ctx): + descriptors = [] + outs = [] + for dep in ctx.attr.deps: + if hasattr(dep, 'proto'): + for desc in dep.proto.transitive_descriptor_sets: + descriptors.append(desc) + for src in dep.proto.transitive_sources: + outs.append(ctx.actions.declare_file(_remove_up(src.short_path) + ".upb.h")) + outs.append(ctx.actions.declare_file(_remove_up(src.short_path) + ".upb.c")) + outdir = _remove_suffix(outs[-1].path, _remove_up(src.short_path) + ".upb.c") + + concatenated = ctx.actions.declare_file(ctx.label.name + "_concatenated_descriptor.bin") + descriptor_paths = [d.path for d in descriptors] + + ctx.actions.run_shell( + inputs = descriptors, + outputs = [concatenated], + progress_message = "Concatenating descriptors", + command = "cat %s > %s" % (" ".join(descriptor_paths), concatenated.path), + ) + ctx.actions.run( + inputs = [concatenated], + outputs = outs, + executable = ctx.executable.upbc, + arguments = ["--outdir", outdir, concatenated.path], + progress_message = "Generating upb protos", + ) + + return [DefaultInfo(files = depset(outs))] + +_upb_proto_library_srcs = rule( + implementation = _upb_proto_library_srcs_impl, + attrs = { + "upbc": attr.label( + executable = True, + cfg = "host", + ), + "deps": attr.label_list(), + } +) + +def upb_proto_library(name, deps, upbc): + srcs_rule = name + "_srcs.cc" + _upb_proto_library_srcs( + name = srcs_rule, + upbc = upbc, + deps = deps, + ) + native.cc_library( + name = name, + srcs = [":" + srcs_rule], + deps = [":upb"], + ) diff --git a/tests/conformance_upb.c b/tests/conformance_upb.c index ccc8a94284..e0a7d8ccd4 100644 --- a/tests/conformance_upb.c +++ b/tests/conformance_upb.c @@ -8,8 +8,8 @@ #include #include -#include "conformance.upb.h" -#include "google/protobuf/test_messages_proto3.upb.h" +#include "conformance/conformance.upb.h" +#include "src/google/protobuf/test_messages_proto3.upb.h" int test_count = 0; diff --git a/tools/upbc.lua b/tools/upbc.lua index 8ac4f735ac..adea3734b6 100644 --- a/tools/upbc.lua +++ b/tools/upbc.lua @@ -13,11 +13,17 @@ local make_c_api = require "make_c_api" local upb = require "upb" local generate_upbdefs = false +local outdir = "." -for _, argument in ipairs(arg) do +i = 1 +while i <= #arg do + argument = arg[i] if argument.sub(argument, 1, 2) == "--" then if argument == "--generate-upbdefs" then generate_upbdefs = true + elseif argument == "--outdir" then + i = i + 1 + outdir = arg[i] else print("Unknown flag: " .. argument) return 1 @@ -29,6 +35,7 @@ for _, argument in ipairs(arg) do end src = argument end + i = i + 1 end if not src then @@ -40,6 +47,11 @@ function strip_proto(filename) return string.gsub(filename, '%.proto$','') end +local function open(filename) + local full_name = outdir .. "/" .. filename + return assert(io.open(full_name, "w"), "couldn't open " .. full_name) +end + -- Open input/output files. local f = assert(io.open(src, "r"), "couldn't open input file " .. src) local descriptor = f:read("*all") @@ -67,8 +79,8 @@ for _, file in ipairs(files) do if generate_upbdefs then -- Legacy generated defs. - local hfile = assert(io.open(hfilename, "w"), "couldn't open " .. hfilename) - local cfile = assert(io.open(cfilename, "w"), "couldn't open " .. cfilename) + local hfile = open(hfilename) + local cfile = open(cfilename) local happend = dump_cinit.file_appender(hfile) local cappend = dump_cinit.file_appender(cfile) @@ -90,8 +102,8 @@ for _, file in ipairs(files) do print(string.format(" cfilename=%s", cfilename)) end - local hfile = assert(io.open(hfilename, "w"), "couldn't open " .. hfilename) - local cfile = assert(io.open(cfilename, "w"), "couldn't open " .. cfilename) + local hfile = open(hfilename) + local cfile = open(cfilename) local happend = dump_cinit.file_appender(hfile) local cappend = dump_cinit.file_appender(cfile) From 9abff2b95f2f8650bf0256988233836430b4eddc Mon Sep 17 00:00:00 2001 From: Josh Haberman Date: Sat, 3 Nov 2018 19:37:45 -0700 Subject: [PATCH 06/16] Added missing files from Bazel build. --- ragel.BUILD | 193 ++++++++++++++++++++++++++++++++++++++++++ staleness_test.py | 30 +++++++ staleness_test_lib.py | 158 ++++++++++++++++++++++++++++++++++ 3 files changed, 381 insertions(+) create mode 100644 ragel.BUILD create mode 100644 staleness_test.py create mode 100644 staleness_test_lib.py diff --git a/ragel.BUILD b/ragel.BUILD new file mode 100644 index 0000000000..5485faba0b --- /dev/null +++ b/ragel.BUILD @@ -0,0 +1,193 @@ + +package( + default_visibility = ["//visibility:public"], +) + +cc_binary( + name = "ragel", + srcs = [ + "ragel/rubycodegen.cpp", + "ragel/goipgoto.h", + "ragel/cdtable.h", + "ragel/rubycodegen.h", + "ragel/gotable.h", + "ragel/gocodegen.cpp", + "ragel/rubyfflat.cpp", + "ragel/common.cpp", + "ragel/gofflat.cpp", + "ragel/cdtable.cpp", + "ragel/cdsplit.cpp", + "ragel/rlparse.cpp", + "ragel/csfgoto.cpp", + "ragel/javacodegen.cpp", + "ragel/gocodegen.h", + "ragel/mlgoto.cpp", + "ragel/fsmgraph.cpp", + "ragel/version.h", + "ragel/mlfflat.h", + "ragel/fsmgraph.h", + "ragel/fsmbase.cpp", + "ragel/fsmstate.cpp", + "ragel/gotablish.cpp", + "ragel/rubyflat.cpp", + "ragel/cdfgoto.h", + "ragel/cscodegen.h", + "ragel/mlflat.cpp", + "ragel/rubyflat.h", + "ragel/goftable.h", + "ragel/rbxgoto.cpp", + "ragel/csfflat.cpp", + "ragel/gofgoto.cpp", + "ragel/gofgoto.h", + "ragel/ragel.h", + "ragel/goftable.cpp", + "ragel/cdcodegen.cpp", + "ragel/rlparse.h", + "ragel/cdsplit.h", + "ragel/xmlcodegen.cpp", + "ragel/goipgoto.cpp", + "ragel/dotcodegen.h", + "ragel/gogoto.cpp", + "ragel/csflat.h", + "ragel/csfflat.h", + #"ragel/config.h.in", + "ragel/csipgoto.cpp", + "ragel/mltable.cpp", + "ragel/mlflat.h", + "ragel/csftable.cpp", + "ragel/cdgoto.h", + "ragel/goflat.cpp", + "ragel/rubyfflat.h", + "ragel/mlftable.h", + "ragel/rubyftable.h", + "ragel/fsmap.cpp", + "ragel/redfsm.cpp", + "ragel/goflat.h", + "ragel/parsetree.cpp", + "ragel/fsmmin.cpp", + "ragel/dotcodegen.cpp", + "ragel/redfsm.h", + "ragel/mlcodegen.cpp", + "ragel/cdfgoto.cpp", + "ragel/cssplit.cpp", + "ragel/cstable.cpp", + "ragel/javacodegen.h", + "ragel/parsedata.cpp", + "ragel/buffer.h", + "ragel/gogoto.h", + "ragel/csgoto.h", + "ragel/pcheck.h", + "ragel/rubyftable.cpp", + "ragel/csfgoto.h", + "ragel/common.h", + "ragel/cdftable.h", + "ragel/mlgoto.h", + "ragel/csgoto.cpp", + "ragel/cdflat.h", + "ragel/cdipgoto.h", + "ragel/cstable.h", + "ragel/gendata.h", + "ragel/cdfflat.cpp", + "ragel/gotable.cpp", + "ragel/cdcodegen.h", + "ragel/gendata.cpp", + "ragel/rubytable.h", + "ragel/csflat.cpp", + "ragel/inputdata.h", + "ragel/inputdata.cpp", + "ragel/rubytable.cpp", + "ragel/fsmattach.cpp", + "ragel/csipgoto.h", + "ragel/cscodegen.cpp", + "ragel/cdfflat.h", + "ragel/rbxgoto.h", + "ragel/xmlcodegen.h", + "ragel/gofflat.h", + "ragel/parsedata.h", + "ragel/mlfgoto.h", + "ragel/cdflat.cpp", + "ragel/config.h", + "ragel/rlscan.cpp", + "ragel/mlcodegen.h", + "ragel/mlfflat.cpp", + "ragel/mlftable.cpp", + "ragel/mltable.h", + "ragel/cdipgoto.cpp", + "ragel/cdftable.cpp", + "ragel/parsetree.h", + "ragel/rlscan.h", + "ragel/main.cpp", + "ragel/cssplit.h", + "ragel/mlfgoto.cpp", + "ragel/csftable.h", + "ragel/gotablish.h", + "ragel/cdgoto.cpp", + "aapl/avlmelkey.h", + "aapl/dlistmel.h", + "aapl/avliset.h", + "aapl/avlkeyless.h", + "aapl/sbstset.h", + "aapl/sbsttable.h", + "aapl/quicksort.h", + "aapl/avlitree.h", + "aapl/avlcommon.h", + "aapl/bstset.h", + "aapl/avlmel.h", + "aapl/insertsort.h", + "aapl/dlist.h", + "aapl/avlmap.h", + "aapl/mergesort.h", + "aapl/resize.h", + "aapl/bstcommon.h", + "aapl/bstmap.h", + "aapl/compare.h", + "aapl/svector.h", + "aapl/avlset.h", + "aapl/bsttable.h", + "aapl/avlikeyless.h", + "aapl/bubblesort.h", + "aapl/table.h", + "aapl/avlbasic.h", + "aapl/vector.h", + "aapl/avlimap.h", + "aapl/dlistval.h", + "aapl/dlcommon.h", + "aapl/avlibasic.h", + "aapl/sbstmap.h", + "aapl/avlimel.h", + "aapl/avlimelkey.h", + "aapl/avltree.h", + ], + includes = ["ragel", "aapl"], +) + +config_h_contents = """ +#define PACKAGE "ragel" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "ragel" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "ragel 6.10" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "ragel" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "6.10" + +/* Version number of package */ +#define VERSION "6.10" +""" + +genrule( + name = "gen_config_h", + outs = ["ragel/config.h"], + cmd = "(cat <<'HEREDOC'\n%s\nHEREDOC\n) > $@" % config_h_contents, +) diff --git a/staleness_test.py b/staleness_test.py new file mode 100644 index 0000000000..43176cf8f3 --- /dev/null +++ b/staleness_test.py @@ -0,0 +1,30 @@ +"""The py_test() script for generated_file_staleness_test() rules. + +Note that this file is preprocessed! The INSERT_<...> text below is replaced +with the actual list of files before we actually run the script. +""" + +from __future__ import absolute_import + +import staleness_test_lib +import unittest +import sys + +file_list = """ + INSERT_FILE_LIST_HERE +""".split() + +config = staleness_test_lib.Config(file_list) + + +class TestFilesMatch(unittest.TestCase): + + def testFilesMatch(self): + errors = staleness_test_lib.CheckFilesMatch(config) + self.assertFalse(errors, errors) + + +if len(sys.argv) > 1 and sys.argv[1] == "--fix": + staleness_test_lib.FixFiles(config) +else: + unittest.main() diff --git a/staleness_test_lib.py b/staleness_test_lib.py new file mode 100644 index 0000000000..6d5c3d3068 --- /dev/null +++ b/staleness_test_lib.py @@ -0,0 +1,158 @@ +"""Shared code for validating generated_file_staleness_test() rules. + +This code is used by test scripts generated from +generated_file_staleness_test() rules. +""" + +from __future__ import absolute_import +from __future__ import print_function + +import os +from shutil import copyfile + + +class _FilePair(object): + """Represents a single (target, generated) file pair.""" + + def __init__(self, target, generated): + self.target = target + self.generated = generated + + +class Config(object): + """Represents the configuration for a single staleness test target.""" + + def __init__(self, file_list): + # Duplicate to avoid modifying our arguments. + file_list = list(file_list) + + # The file list contains a few other bits of information at the end. + # This is packed by the code in build_defs.bzl. + self.target_name = file_list.pop() + self.package_name = file_list.pop() + self.pattern = file_list.pop() + + self.file_list = file_list + + +def _GetFilePairs(config): + """Generates the list of file pairs. + + Args: + config: a Config object representing this target's config. + + Returns: + A list of _FilePair objects. + """ + + ret = [] + + has_bazel_genfiles = os.path.exists("bazel-genfiles") + + for filename in config.file_list: + target = os.path.join(config.package_name, filename) + generated = os.path.join(config.package_name, config.pattern % filename) + if has_bazel_genfiles: + generated = os.path.join("bazel-genfiles", generated) + + # Generated files should always exist. Blaze should guarantee this before + # we are run. + if not os.path.isfile(generated): + print("Generated file '%s' does not exist." % generated) + print("Please run this command to generate it:") + print(" bazel build %s:%s" % (config.package_name, config.target_name)) + ret.append(_FilePair(target, generated)) + + return ret + + +def _GetMissingAndStaleFiles(file_pairs): + """Generates lists of missing and stale files. + + Args: + file_pairs: a list of _FilePair objects. + + Returns: + missing_files: a list of _FilePair objects representing missing files. + These target files do not exist at all. + stale_files: a list of _FilePair objects representing stale files. + These target files exist but have stale contents. + """ + + missing_files = [] + stale_files = [] + + for pair in file_pairs: + if not os.path.isfile(pair.target): + missing_files.append(pair) + continue + + generated = open(pair.generated).read() + target = open(pair.target).read() + if generated != target: + stale_files.append(pair) + + return missing_files, stale_files + + +def _CopyFiles(file_pairs): + """Copies all generated files to the corresponding target file. + + The target files must be writable already. + + Args: + file_pairs: a list of _FilePair objects that we want to copy. + """ + + for pair in file_pairs: + target_dir = os.path.dirname(pair.target) + if not os.path.isdir(target_dir): + os.makedirs(target_dir) + copyfile(pair.generated, pair.target) + + +def FixFiles(config): + """Implements the --fix option: overwrites missing or out-of-date files. + + Args: + config: the Config object for this test. + """ + + file_pairs = _GetFilePairs(config) + missing_files, stale_files = _GetMissingAndStaleFiles(file_pairs) + + _CopyFiles(stale_files + missing_files) + + +def CheckFilesMatch(config): + """Checks whether each target file matches the corresponding generated file. + + Args: + config: the Config object for this test. + + Returns: + None if everything matches, otherwise a string error message. + """ + + diff_errors = [] + + file_pairs = _GetFilePairs(config) + missing_files, stale_files = _GetMissingAndStaleFiles(file_pairs) + + for pair in missing_files: + diff_errors.append("File %s does not exist" % pair.target) + continue + + for pair in stale_files: + diff_errors.append("File %s is out of date" % pair.target) + + if diff_errors: + error_msg = "Files out of date!\n\n" + error_msg += "To fix run THIS command:\n" + error_msg += " bazel-bin/%s/%s --fix\n\n" % (config.package_name, + config.target_name) + error_msg += "Errors:\n" + error_msg += " " + "\n ".join(diff_errors) + return error_msg + else: + return None From 966d9d89e6d659cc5942373cd69f024579ea4923 Mon Sep 17 00:00:00 2001 From: Josh Haberman Date: Sat, 3 Nov 2018 19:38:09 -0700 Subject: [PATCH 07/16] Added Kokoro config files. --- kokoro/ubuntu/build.sh | 4 ++++ kokoro/ubuntu/continuous.cfg | 2 ++ kokoro/ubuntu/presubmit.cfg | 2 ++ 3 files changed, 8 insertions(+) create mode 100644 kokoro/ubuntu/build.sh create mode 100644 kokoro/ubuntu/continuous.cfg create mode 100644 kokoro/ubuntu/presubmit.cfg diff --git a/kokoro/ubuntu/build.sh b/kokoro/ubuntu/build.sh new file mode 100644 index 0000000000..bbf8841c56 --- /dev/null +++ b/kokoro/ubuntu/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +cd $(dirname $0)/../.. +bazel test :all diff --git a/kokoro/ubuntu/continuous.cfg b/kokoro/ubuntu/continuous.cfg new file mode 100644 index 0000000000..fa97583b34 --- /dev/null +++ b/kokoro/ubuntu/continuous.cfg @@ -0,0 +1,2 @@ +build_file: "upb/kokoro/ubuntu/build.sh" +timeout_mins: 15 diff --git a/kokoro/ubuntu/presubmit.cfg b/kokoro/ubuntu/presubmit.cfg new file mode 100644 index 0000000000..fa97583b34 --- /dev/null +++ b/kokoro/ubuntu/presubmit.cfg @@ -0,0 +1,2 @@ +build_file: "upb/kokoro/ubuntu/build.sh" +timeout_mins: 15 From 23fa64f10c27bdd4b8261c70a81a7721fd71f69f Mon Sep 17 00:00:00 2001 From: Josh Haberman Date: Mon, 5 Nov 2018 09:34:05 -0800 Subject: [PATCH 08/16] Added script to generate CMake build. Bazel tests it. --- BUILD | 180 ++++++--- CMakeLists.txt | 367 ++++++------------ build_defs.bzl | 2 +- tools/make_cmakelists.py | 234 +++++++++++ staleness_test.py => tools/staleness_test.py | 2 +- .../staleness_test_lib.py | 0 tools/upbc | 5 - 7 files changed, 488 insertions(+), 302 deletions(-) create mode 100755 tools/make_cmakelists.py rename staleness_test.py => tools/staleness_test.py (94%) rename staleness_test_lib.py => tools/staleness_test_lib.py (100%) delete mode 100755 tools/upbc diff --git a/BUILD b/BUILD index b636e15518..458463e089 100644 --- a/BUILD +++ b/BUILD @@ -45,7 +45,11 @@ cc_library( "upb/sink.h", "upb/upb.h", ], - copts = ["-std=c89", "-pedantic", "-Wno-long-long"], + copts = [ + "-std=c89", + "-pedantic", + "-Wno-long-long", + ], ) cc_library( @@ -58,8 +62,12 @@ cc_library( "upb/descriptor/descriptor.upbdefs.h", "upb/descriptor/reader.h", ], + copts = [ + "-std=c89", + "-pedantic", + "-Wno-long-long", + ], deps = [":upb"], - copts = ["-std=c89", "-pedantic", "-Wno-long-long"], ) cc_library( @@ -80,11 +88,15 @@ cc_library( "upb/pb/glue.h", "upb/pb/textprinter.h", ], + copts = [ + "-std=c89", + "-pedantic", + "-Wno-long-long", + ], deps = [ ":upb", ":upb_descriptor", ], - copts = ["-std=c89", "-pedantic", "-Wno-long-long"], ) cc_library( @@ -97,8 +109,12 @@ cc_library( "upb/json/parser.h", "upb/json/printer.h", ], + copts = [ + "-std=c89", + "-pedantic", + "-Wno-long-long", + ], deps = [":upb"], - copts = ["-std=c89", "-pedantic", "-Wno-long-long"], ) cc_library( @@ -118,6 +134,10 @@ py_binary( upb_amalgamation( name = "gen_amalgamation", + outs = [ + "upb.c", + "upb.h", + ], amalgamator = ":amalgamate", libs = [ ":upb", @@ -125,92 +145,115 @@ upb_amalgamation( ":upb_pb", ":upb_json", ], - outs = [ - "upb.h", - "upb.c", - ], ) cc_library( name = "amalgamation", - hdrs = ["upb.h"], srcs = ["upb.c"], + hdrs = ["upb.h"], ) # C/C++ tests ################################################################## cc_library( - testonly = 1, name = "upb_test", - hdrs = [ - "tests/upb_test.h", - "tests/test_util.h", - ], + testonly = 1, srcs = [ "tests/testmain.cc", ], + hdrs = [ + "tests/test_util.h", + "tests/upb_test.h", + ], ) cc_test( name = "test_varint", srcs = ["tests/pb/test_varint.c"], - deps = [":upb_pb", ":upb_test"], + deps = [ + ":upb_pb", + ":upb_test", + ], ) cc_test( name = "test_def", srcs = ["tests/test_def.c"], - deps = [":upb_pb", ":upb_test"], + deps = [ + ":upb_pb", + ":upb_test", + ], ) cc_test( name = "test_handlers", srcs = ["tests/test_handlers.c"], - deps = [":upb_pb", ":upb_test"], + deps = [ + ":upb_pb", + ":upb_test", + ], ) cc_test( name = "test_decoder", srcs = ["tests/pb/test_decoder.cc"], - deps = [":upb_pb", ":upb_test"], + deps = [ + ":upb_pb", + ":upb_test", + ], ) cc_test( name = "test_encoder", srcs = ["tests/pb/test_encoder.cc"], - deps = [":upb_pb", ":upb_test", ":upb_cc_bindings"], data = ["upb/descriptor/descriptor.pb"], + deps = [ + ":upb_cc_bindings", + ":upb_pb", + ":upb_test", + ], ) cc_test( name = "test_cpp", srcs = ["tests/test_cpp.cc"], - deps = [":upb_descriptor", ":upb", ":upb_pb", ":upb_test"], + deps = [ + ":upb", + ":upb_descriptor", + ":upb_pb", + ":upb_test", + ], ) cc_test( name = "test_table", srcs = ["tests/test_table.cc"], - deps = [":upb", ":upb_test"], + deps = [ + ":upb", + ":upb_test", + ], ) cc_test( name = "test_json", srcs = [ - "tests/json/test_json.cc", - "tests/json/test.upbdefs.h", "tests/json/test.upbdefs.c", + "tests/json/test.upbdefs.h", + "tests/json/test_json.cc", + ], + deps = [ + ":upb_json", + ":upb_test", ], - deps = [":upb_json", ":upb_test"], ) upb_proto_library( name = "conformance_proto_upb", + upbc = ":upbc", deps = [ "@com_google_protobuf//:conformance_proto", "@com_google_protobuf//:test_messages_proto3_proto", ], - upbc = ":upbc", ) cc_binary( @@ -219,8 +262,8 @@ cc_binary( "tests/conformance_upb.c", ], deps = [ - ":upb", ":conformance_proto_upb", + ":upb", ], ) @@ -234,11 +277,11 @@ sh_test( name = "test_conformance_upb", srcs = ["test_conformance_upb.sh"], data = [ + "tests/conformance_upb_failures.txt", ":conformance_upb", + "@bazel_tools//tools/bash/runfiles", "@com_google_protobuf//:conformance_test_runner", - "tests/conformance_upb_failures.txt", - "@bazel_tools//tools/bash/runfiles" - ] + ], ) # Lua libraries. ############################################################### @@ -262,8 +305,8 @@ lua_cclibrary( lua_library( name = "lua/upb", srcs = ["upb/bindings/lua/upb.lua"], - strip_prefix = "upb/bindings/lua", luadeps = ["lua/upb_c"], + strip_prefix = "upb/bindings/lua", ) lua_cclibrary( @@ -276,11 +319,11 @@ lua_cclibrary( lua_library( name = "lua/upb/table", srcs = ["upb/bindings/lua/upb/table.lua"], - strip_prefix = "upb/bindings/lua", luadeps = [ "lua/upb", "lua/upb/table_c", ], + strip_prefix = "upb/bindings/lua", ) lua_cclibrary( @@ -293,11 +336,11 @@ lua_cclibrary( lua_library( name = "lua/upb/pb", srcs = ["upb/bindings/lua/upb/pb.lua"], - strip_prefix = "upb/bindings/lua", luadeps = [ "lua/upb", "lua/upb/pb_c", ], + strip_prefix = "upb/bindings/lua", ) lua_library( @@ -306,25 +349,25 @@ lua_library( "tools/dump_cinit.lua", "tools/make_c_api.lua", ], - strip_prefix = "tools", luadeps = [ "lua/upb", "lua/upb/table", ], + strip_prefix = "tools", ) # Lua tests. ################################################################### lua_test( name = "lua/test_upb", - luamain = "tests/bindings/lua/test_upb.lua", luadeps = ["lua/upb"], + luamain = "tests/bindings/lua/test_upb.lua", ) lua_test( name = "lua/test_upb_pb", - luamain = "tests/bindings/lua/test_upb.pb.lua", luadeps = ["lua/upb/pb"], + luamain = "tests/bindings/lua/test_upb.pb.lua", ) # upb compiler ################################################################# @@ -337,14 +380,35 @@ lua_binary( luamain = "tools/upbc.lua", ) +# Test the CMake build ######################################################### + +make_shell_script( + name = "gen_run_cmake_build", + out = "run_cmake_build.sh", + contents = "mkdir build && cd build && cmake .. && make -j8" +) + +sh_test( + name = "cmake_build", + srcs = ["run_cmake_build.sh"], + data = glob([ + "CMakeLists.txt", + "google/**/*", + "upb/**/*", + "tests/**/*", + ]) + [ + "@bazel_tools//tools/bash/runfiles", + ], +) + # Generated files ############################################################## -exports_files(["staleness_test.py"]) +exports_files(["tools/staleness_test.py"]) py_library( name = "staleness_test_lib", testonly = 1, - srcs = ["staleness_test_lib.py"], + srcs = ["tools/staleness_test_lib.py"], ) genrule( @@ -363,51 +427,64 @@ genrule( proto_library( name = "upb_descriptor_proto", srcs = [ - "upb/descriptor/descriptor.proto" + "upb/descriptor/descriptor.proto", ], ) +py_binary( + name = "make_cmakelists", + srcs = ["tools/make_cmakelists.py"], +) + +genrule( + name = "gen_cmakelists", + outs = ["generated/CMakeLists.txt"], + srcs = ["BUILD", "WORKSPACE"], + tools = [":make_cmakelists"], + cmd = "$(location :make_cmakelists) $@" +) + genrule( name = "copy_upb_descriptor_pb", - outs = ["generated/upb/descriptor/descriptor.pb"], srcs = [":upb_descriptor_proto"], + outs = ["generated/upb/descriptor/descriptor.pb"], cmd = "cp $< $@", ) genrule( name = "generate_old_upbdefs", srcs = ["generated/upb/descriptor/descriptor.pb"], - tools = [":upbc"], outs = [ "generated/upb/descriptor/descriptor.upbdefs.h", "generated/upb/descriptor/descriptor.upbdefs.c", ], cmd = "UPBC=$$PWD/$(location :upbc); INFILE=$$PWD/$<; cd $(GENDIR)/generated && $$UPBC --generate-upbdefs $$INFILE", + tools = [":upbc"], ) proto_library( name = "google_descriptor_proto", srcs = [ - "google/protobuf/descriptor.proto" + "google/protobuf/descriptor.proto", ], ) genrule( name = "copy_google_descriptor_pb", - outs = ["generated/google/protobuf/descriptor.pb"], srcs = [":google_descriptor_proto"], + outs = ["generated/google/protobuf/descriptor.pb"], cmd = "cp $< $@", ) genrule( name = "generate_descriptor_c", srcs = ["generated/google/protobuf/descriptor.pb"], - tools = [":upbc"], outs = [ "generated/google/protobuf/descriptor.upb.h", "generated/google/protobuf/descriptor.upb.c", ], cmd = "UPBC=$$PWD/$(location :upbc); INFILE=$$PWD/$<; cd $(GENDIR)/generated && $$UPBC $$INFILE", + tools = [":upbc"], ) proto_library( @@ -425,35 +502,36 @@ genrule( genrule( name = "generated_json_test_proto_upbdefs", srcs = ["generated/tests/json/test.proto.pb"], - tools = [":upbc"], outs = [ "generated/tests/json/test.upbdefs.h", "generated/tests/json/test.upbdefs.c", ], cmd = "UPBC=$$PWD/$(location :upbc); INFILE=$$PWD/$<; cd $(GENDIR)/generated && $$UPBC --generate-upbdefs $$INFILE", + tools = [":upbc"], ) genrule( name = "generate_json_ragel", srcs = ["upb/json/parser.rl"], outs = ["generated/upb/json/parser.c"], - tools = ["@ragel//:ragel"], cmd = "$(location @ragel//:ragel) -C -o upb/json/parser.c $< && mv upb/json/parser.c $@", + tools = ["@ragel"], ) generated_file_staleness_test( name = "test_generated_files", outs = [ - "google/protobuf/descriptor.upb.h", + "CMakeLists.txt", "google/protobuf/descriptor.upb.c", - "upb/pb/compile_decoder_x64.h", - "upb/descriptor/descriptor.upbdefs.c", + "google/protobuf/descriptor.upb.h", + "tests/json/test.proto.pb", + "tests/json/test.upbdefs.c", + "tests/json/test.upbdefs.h", "upb/descriptor/descriptor.pb", + "upb/descriptor/descriptor.upbdefs.c", "upb/descriptor/descriptor.upbdefs.h", "upb/json/parser.c", - "tests/json/test.upbdefs.c", - "tests/json/test.upbdefs.h", - "tests/json/test.proto.pb", + "upb/pb/compile_decoder_x64.h", ], generated_pattern = "generated/%s", ) diff --git a/CMakeLists.txt b/CMakeLists.txt index ba572fa2ea..5c6d5fa032 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,3 +1,4 @@ +# This file was generated from BUILD using tools/make_cmakelists.py. cmake_minimum_required(VERSION 3.1) @@ -9,16 +10,14 @@ endif() cmake_minimum_required (VERSION 3.0) cmake_policy(SET CMP0048 NEW) -project (upb) + +project(upb) + # Prevent CMake from setting -rdynamic on Linux (!!). SET(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") SET(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") -# Options we define for users. -option(UPB_ENABLE_ASAN "Enable address sanitizer." OFF) -option(UPB_ENABLE_UBSAN "Enable undefined behavior sanitizer." OFF) - # Set default build type. if(NOT CMAKE_BUILD_TYPE) message(STATUS "Setting build type to 'RelWithDebInfo' as none was specified.") @@ -51,8 +50,6 @@ endif() include_directories(.) include_directories(${CMAKE_CURRENT_BINARY_DIR}) -set(CMAKE_CXX_FLAGS "-std=c++11 -W -Wall -Wno-sign-compare") -set(CMAKE_C_FLAGS "-std=c89 -W -Wall -Wno-sign-compare") if(APPLE) set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -undefined dynamic_lookup -flat_namespace") @@ -60,250 +57,132 @@ elseif(UNIX) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--build-id") endif() -FIND_PACKAGE(Lua) -if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/third_party/protobuf/Makefile.am") - set(PROTOBUF_FOUND TRUE) -endif() -find_program(RAGEL NAMES ragel) - -if(LUA_FOUND) - include_directories(${LUA_INCLUDE_DIR}) - - add_library(upb_c SHARED - upb/bindings/lua/upb.c - upb/bindings/lua/def.c - upb/bindings/lua/msg.c - ) - target_link_libraries(upb_c LINK_PRIVATE - upbpb_pic - upbdef_pic - upbhandlers_pic upb_pic ) - - add_library(table_c SHARED - upb/bindings/lua/upb/table.c - ) - target_link_libraries(table_c LINK_PRIVATE upb_c upb_pic) - - add_library(pb_c SHARED - upb/bindings/lua/upb/pb.c - ) - target_link_libraries(pb_c LINK_PRIVATE upb_c upbpb_pic) - - set_target_properties(upb_c - PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "lua" - PREFIX "" - SUFFIX ".so") - set_target_properties(table_c pb_c - PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "lua/upb" - PREFIX "" - SUFFIX ".so") - - add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/tools/upbc - DEPENDS ${CMAKE_SOURCE_DIR}/tools/upbc - ${CMAKE_SOURCE_DIR}/tools/upbc.lua - ${CMAKE_SOURCE_DIR}/tools/dump_cinit.lua - ${CMAKE_SOURCE_DIR}/tools/make_c_api.lua - ${CMAKE_SOURCE_DIR}/upb/bindings/lua/upb.lua - ${CMAKE_SOURCE_DIR}/upb/bindings/lua/upb/table.lua - ${CMAKE_SOURCE_DIR}/upb/bindings/lua/upb/pb.lua - upb_c - table_c - pb_c - COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_SOURCE_DIR}/tools/upbc - ${CMAKE_SOURCE_DIR}/tools/upbc.lua - ${CMAKE_SOURCE_DIR}/tools/dump_cinit.lua - ${CMAKE_SOURCE_DIR}/tools/make_c_api.lua - ${CMAKE_CURRENT_BINARY_DIR}/tools) - add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/lua/upb.lua - DEPENDS ${CMAKE_SOURCE_DIR}/upb/bindings/lua/upb.lua - COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_SOURCE_DIR}/upb/bindings/lua/upb.lua - ${CMAKE_CURRENT_BINARY_DIR}/lua) - add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/lua/upb/table.lua - ${CMAKE_CURRENT_BINARY_DIR}/lua/upb/pb.lua - DEPENDS ${CMAKE_SOURCE_DIR}/upb/bindings/lua/upb/table.lua - ${CMAKE_SOURCE_DIR}/upb/bindings/lua/upb/pb.lua - COMMAND ${CMAKE_COMMAND} -E copy - ${CMAKE_SOURCE_DIR}/upb/bindings/lua/upb/table.lua - ${CMAKE_SOURCE_DIR}/upb/bindings/lua/upb/pb.lua - ${CMAKE_CURRENT_BINARY_DIR}/lua/upb) - - add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/upb/pb/compile_decoder_x64.h - DEPENDS upb/pb/compile_decoder_x64.dasc - COMMAND - cd ${CMAKE_CURRENT_SOURCE_DIR} && - lua third_party/dynasm/dynasm.lua - -c upb/pb/compile_decoder_x64.dasc - > ${CMAKE_CURRENT_BINARY_DIR}/upb/pb/compile_decoder_x64.h - ) - - add_custom_target( - upbc ALL - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/tools/make_c_api.lua - ${CMAKE_CURRENT_SOURCE_DIR}/tools/upbc.lua - ${CMAKE_CURRENT_BINARY_DIR}/tools/upbc - ${CMAKE_CURRENT_BINARY_DIR}/lua/upb.lua - ${CMAKE_CURRENT_BINARY_DIR}/lua/upb/pb.lua - ) - - add_custom_target( - genfiles2 ALL - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/upb/pb/compile_decoder_x64.h - ) - - add_test( - NAME testlua - COMMAND lua ${CMAKE_SOURCE_DIR}/tests/bindings/lua/test_upb.lua - ${CMAKE_SOURCE_DIR}/tests/bindings/lua/test_upb.pb.lua - ) - set_property(TEST testlua PROPERTY ENVIRONMENT - LUA_PATH=${CMAKE_SOURCE_DIR}/third_party/lunit/?.lua$${CMAKE_SOURCE_DIR}/upb/bindings/lua/?.lua - LUA_CPATH=${CMAKE_BINARY_DIR}/upb/bindings/lua/?.so - ) - set_tests_properties(testlua - PROPERTIES - DEPENDS "upb_c;pb_c" - ) - enable_testing() -endif() - -if(LUA_FOUND AND PROTOBUF_FOUND) - set(protobuf_BUILD_TESTS OFF CACHE BOOL "enable tests for proto2" FORCE) - set(protobuf_BUILD_SHARED_LIBS OFF CACHE BOOL "enable shared libs for proto2" FORCE) - set(protobuf_BUILD_CONFORMANCE ON CACHE BOOL "Build conformance tests" FORCE) - add_subdirectory(third_party/protobuf/cmake) - - add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/conformance.upb.h - ${CMAKE_CURRENT_BINARY_DIR}/google/protobuf/test_messages_proto3.upb.h - ${CMAKE_CURRENT_BINARY_DIR}/google/protobuf/test_messages_proto3.upb.c - ${CMAKE_CURRENT_BINARY_DIR}/google/protobuf/any.upb.c - ${CMAKE_CURRENT_BINARY_DIR}/google/protobuf/duration.upb.c - ${CMAKE_CURRENT_BINARY_DIR}/google/protobuf/field_mask.upb.c - ${CMAKE_CURRENT_BINARY_DIR}/google/protobuf/struct.upb.c - ${CMAKE_CURRENT_BINARY_DIR}/google/protobuf/timestamp.upb.c - ${CMAKE_CURRENT_BINARY_DIR}/google/protobuf/wrappers.upb.c - ${CMAKE_CURRENT_BINARY_DIR}/conformance.upb.c - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/tools/upbc - ${CMAKE_CURRENT_SOURCE_DIR}/third_party/protobuf/conformance/conformance.proto - COMMAND protoc --include_imports - ${CMAKE_CURRENT_SOURCE_DIR}/third_party/protobuf/conformance/conformance.proto - ${CMAKE_CURRENT_SOURCE_DIR}/third_party/protobuf/src/google/protobuf/test_messages_proto3.proto - -I${CMAKE_CURRENT_SOURCE_DIR}/third_party/protobuf/conformance - -I${CMAKE_CURRENT_SOURCE_DIR}/third_party/protobuf/src - -o${CMAKE_CURRENT_BINARY_DIR}/conformance.pb && - tools/upbc ${CMAKE_CURRENT_BINARY_DIR}/conformance.pb - ) - - add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/google/protobuf/descriptor.upb.h - ${CMAKE_CURRENT_BINARY_DIR}/google/protobuf/descriptor.upb.c - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/tools/upbc - ${CMAKE_CURRENT_SOURCE_DIR}/google/protobuf/descriptor.proto - COMMAND protoc - ${CMAKE_CURRENT_SOURCE_DIR}/google/protobuf/descriptor.proto - -I${CMAKE_CURRENT_SOURCE_DIR} - -o${CMAKE_CURRENT_BINARY_DIR}/descriptor.pb && - tools/upbc ${CMAKE_CURRENT_BINARY_DIR}/descriptor.pb - ) - - add_custom_target( - genfiles ALL - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/conformance.upb.h - ${CMAKE_CURRENT_BINARY_DIR}/google/protobuf/descriptor.upb.h - ) - - add_executable(conformance_upb - tests/conformance_upb.c - ${CMAKE_CURRENT_BINARY_DIR}/conformance.upb.c - ${CMAKE_CURRENT_BINARY_DIR}/google/protobuf/test_messages_proto3.upb.c - ${CMAKE_CURRENT_BINARY_DIR}/google/protobuf/any.upb.c - ${CMAKE_CURRENT_BINARY_DIR}/google/protobuf/duration.upb.c - ${CMAKE_CURRENT_BINARY_DIR}/google/protobuf/field_mask.upb.c - ${CMAKE_CURRENT_BINARY_DIR}/google/protobuf/struct.upb.c - ${CMAKE_CURRENT_BINARY_DIR}/google/protobuf/timestamp.upb.c - ${CMAKE_CURRENT_BINARY_DIR}/google/protobuf/wrappers.upb.c - ) - target_link_libraries(conformance_upb LINK_PRIVATE - upb - ) - - add_custom_target( - conformance - DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/conformance_upb - ${CMAKE_CURRENT_BINARY_DIR}/third_party/protobuf/cmake/conformance_test_runner - COMMAND ${CMAKE_CURRENT_BINARY_DIR}/third_party/protobuf/cmake/conformance_test_runner ${CMAKE_CURRENT_BINARY_DIR}/conformance_upb - ) - -endif() - -if (RAGEL) - add_custom_command( - OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/upb/json/parser.c - DEPENDS upb/json/parser.rl - COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && RAGEL -C -o ${CMAKE_CURRENT_BINARY_DIR}/upb/json/parser.c upb/json/parser.rl - ) -endif() - -configure_file(tools/copy_genfiles.sh.in tools/copy_genfiles.sh) - -set(UPB_SRCS - upb/decode.c - upb/encode.c - upb/msg.c - upb/table.c - upb/upb.c -) - -set (UPBDEF_SRCS - upb/descriptor/descriptor.upbdefs.c - upb/descriptor/reader.c - upb/def.c - upb/msgfactory.c - upb/refcounted.c -) - -set(UPBHANDLERS_SRCS - upb/sink.c - upb/handlers.c -) - -set(UPBPB_SRCS +enable_testing() + +add_library(upb + google/protobuf/descriptor.upb.c + upb/decode.c + upb/def.c + upb/encode.c + upb/handlers.c + upb/handlers-inl.h + upb/msg.c + upb/msgfactory.c + upb/port_def.inc + upb/port_undef.inc + upb/refcounted.c + upb/sink.c + upb/structdefs.int.h + upb/structs.int.h + upb/table.c + upb/table.int.h + upb/upb.c + google/protobuf/descriptor.upb.h + upb/decode.h + upb/def.h + upb/encode.h + upb/handlers.h + upb/msg.h + upb/msgfactory.h + upb/refcounted.h + upb/sink.h + upb/upb.h) +add_library(upb_descriptor + upb/descriptor/descriptor.upbdefs.c + upb/descriptor/reader.c + upb/descriptor/descriptor.upbdefs.h + upb/descriptor/reader.h) +target_link_libraries(upb_descriptor + upb) +add_library(upb_pb upb/pb/compile_decoder.c upb/pb/decoder.c + upb/pb/decoder.int.h upb/pb/encoder.c upb/pb/glue.c upb/pb/textprinter.c upb/pb/varint.c -) - -set(UPBJSON_SRCS + upb/pb/varint.int.h + upb/pb/decoder.h + upb/pb/encoder.h + upb/pb/glue.h + upb/pb/textprinter.h) +target_link_libraries(upb_pb + upb + upb_descriptor) +add_library(upb_json upb/json/parser.c upb/json/printer.c -) - -add_library(upb ${UPB_SRCS}) -add_library(upbdef ${UPBDEF_SRCS}) -add_library(upbhandlers ${UPBHANDLERS_SRCS}) -add_library(upbpb ${UPBPB_SRCS}) -add_library(upbjson ${UPBJSON_SRCS}) - -add_library(upb_pic ${UPB_SRCS}) -add_library(upbdef_pic ${UPBDEF_SRCS}) -add_library(upbhandlers_pic ${UPBHANDLERS_SRCS}) -add_library(upbpb_pic ${UPBPB_SRCS}) -add_library(upbjson_pic ${UPBJSON_SRCS}) + upb/json/parser.h + upb/json/printer.h) +target_link_libraries(upb_json + upb) +add_library(upb_cc_bindings + upb/bindings/stdc++/string.h) +target_link_libraries(upb_cc_bindings + upb) +set_target_properties(upb_cc_bindings PROPERTIES LINKER_LANGUAGE CXX) +add_library(upb_test + tests/testmain.cc + tests/test_util.h + tests/upb_test.h) +add_executable(test_varint + tests/pb/test_varint.c) +add_test(NAME test_varint COMMAND test_varint) +target_link_libraries(test_varint + upb_pb + upb_test) +add_executable(test_def + tests/test_def.c) +add_test(NAME test_def COMMAND test_def) +target_link_libraries(test_def + upb_pb + upb_test) +add_executable(test_handlers + tests/test_handlers.c) +add_test(NAME test_handlers COMMAND test_handlers) +target_link_libraries(test_handlers + upb_pb + upb_test) +add_executable(test_decoder + tests/pb/test_decoder.cc) +add_test(NAME test_decoder COMMAND test_decoder) +target_link_libraries(test_decoder + upb_pb + upb_test) +add_executable(test_encoder + tests/pb/test_encoder.cc) +add_test(NAME test_encoder COMMAND test_encoder) +add_custom_command( + TARGET test_encoder POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_SOURCE_DIR}/upb/descriptor/descriptor.pb + ${CMAKE_CURRENT_BINARY_DIR}/upb/descriptor/descriptor.pb) +target_link_libraries(test_encoder + upb_cc_bindings + upb_pb + upb_test) +add_executable(test_cpp + tests/test_cpp.cc) +add_test(NAME test_cpp COMMAND test_cpp) +target_link_libraries(test_cpp + upb + upb_descriptor + upb_pb + upb_test) +add_executable(test_table + tests/test_table.cc) +add_test(NAME test_table COMMAND test_table) +target_link_libraries(test_table + upb + upb_test) +add_executable(test_json + tests/json/test.upbdefs.c + tests/json/test.upbdefs.h + tests/json/test_json.cc) +add_test(NAME test_json COMMAND test_json) +target_link_libraries(test_json + upb_json + upb_test) -set_property(TARGET upb_pic PROPERTY POSITION_INDEPENDENT_CODE ON) -set_property(TARGET upbdef_pic PROPERTY POSITION_INDEPENDENT_CODE ON) -set_property(TARGET upbhandlers_pic PROPERTY POSITION_INDEPENDENT_CODE ON) -set_property(TARGET upbpb_pic PROPERTY POSITION_INDEPENDENT_CODE ON) -set_property(TARGET upbjson_pic PROPERTY POSITION_INDEPENDENT_CODE ON) diff --git a/build_defs.bzl b/build_defs.bzl index 8df5660a64..0e58bbbca4 100644 --- a/build_defs.bzl +++ b/build_defs.bzl @@ -131,7 +131,7 @@ def generated_file_staleness_test(name, outs, generated_pattern): """ script_name = name + ".py" - script_src = "//:staleness_test.py" + script_src = "//:tools/staleness_test.py" # Filter out non-existing rules so Blaze doesn't error out before we even # run the test. diff --git a/tools/make_cmakelists.py b/tools/make_cmakelists.py new file mode 100755 index 0000000000..30e1b93dc6 --- /dev/null +++ b/tools/make_cmakelists.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python + +"""TODO(haberman): DO NOT SUBMIT without one-line documentation for make_cmakelists. + +TODO(haberman): DO NOT SUBMIT without a detailed description of make_cmakelists. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import sys +import textwrap + +def StripColons(deps): + return map(lambda x: x[1:], deps) + +def IsSourceFile(name): + return name.endswith(".c") or name.endswith(".cc") + +class BuildFileFunctions(object): + def __init__(self, converter): + self.converter = converter + + def _add_deps(self, kwargs): + if "deps" not in kwargs: + return + self.converter.toplevel += "target_link_libraries(%s\n %s)\n" % ( + kwargs["name"], + "\n ".join(StripColons(kwargs["deps"])) + ) + + def load(self, *args): + pass + + def cc_library(self, **kwargs): + if kwargs["name"] == "amalgamation": + return + files = kwargs.get("srcs", []) + kwargs.get("hdrs", []) + self.converter.toplevel += "add_library(%s\n %s)\n" % ( + kwargs["name"], + "\n ".join(files) + ) + self._add_deps(kwargs) + + # CMake wants to know if each library is C or C++. + # If there are only .h files, it can't infer. Assume C++. + if not filter(IsSourceFile, files): + line = "set_target_properties(%s PROPERTIES LINKER_LANGUAGE CXX)\n" + self.converter.toplevel += line % (kwargs["name"]) + + def cc_binary(self, **kwargs): + pass + + def cc_test(self, **kwargs): + self.converter.toplevel += "add_executable(%s\n %s)\n" % ( + kwargs["name"], + "\n ".join(kwargs["srcs"]) + ) + self.converter.toplevel += "add_test(NAME %s COMMAND %s)\n" % ( + kwargs["name"], + kwargs["name"], + ) + + if "data" in kwargs: + for data_dep in kwargs["data"]: + self.converter.toplevel += textwrap.dedent("""\ + add_custom_command( + TARGET %s POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_SOURCE_DIR}/%s + ${CMAKE_CURRENT_BINARY_DIR}/%s)\n""" % ( + kwargs["name"], data_dep, data_dep + )) + + self._add_deps(kwargs) + + def py_library(self, **kwargs): + pass + + def py_binary(self, **kwargs): + pass + + def lua_cclibrary(self, **kwargs): + pass + + def lua_library(self, **kwargs): + pass + + def lua_binary(self, **kwargs): + pass + + def lua_test(self, **kwargs): + pass + + def sh_test(self, **kwargs): + pass + + def make_shell_script(self, **kwargs): + pass + + def exports_files(self, files, **kwargs): + pass + + def proto_library(self, **kwargs): + pass + + def generated_file_staleness_test(self, **kwargs): + pass + + def upb_amalgamation(self, **kwargs): + pass + + def upb_proto_library(self, **kwargs): + pass + + def genrule(self, **kwargs): + pass + + def glob(*args): + return [] + + +class WorkspaceFileFunctions(object): + def __init__(self, converter): + self.converter = converter + + def load(self, *args): + pass + + def workspace(self, **kwargs): + self.converter.prelude += "project(%s)\n" % (kwargs["name"]) + + def http_archive(self, **kwargs): + pass + + def git_repository(self, **kwargs): + pass + + +class Converter(object): + def __init__(self): + self.prelude = "" + self.toplevel = "" + self.if_lua = "" + + def convert(self): + return self.template % { + "prelude": converter.prelude, + "toplevel": converter.toplevel, + } + + template = textwrap.dedent("""\ + # This file was generated from BUILD using tools/make_cmakelists.py. + + cmake_minimum_required(VERSION 3.1) + + if(${CMAKE_VERSION} VERSION_LESS 3.12) + cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) + else() + cmake_policy(VERSION 3.12) + endif() + + cmake_minimum_required (VERSION 3.0) + cmake_policy(SET CMP0048 NEW) + + %(prelude)s + + # Prevent CMake from setting -rdynamic on Linux (!!). + SET(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") + SET(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + + # Set default build type. + if(NOT CMAKE_BUILD_TYPE) + message(STATUS "Setting build type to 'RelWithDebInfo' as none was specified.") + set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING + "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." + FORCE) + endif() + + # When using Ninja, compiler output won't be colorized without this. + include(CheckCXXCompilerFlag) + CHECK_CXX_COMPILER_FLAG(-fdiagnostics-color=always SUPPORTS_COLOR_ALWAYS) + if(SUPPORTS_COLOR_ALWAYS) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always") + endif() + + # Implement ASAN/UBSAN options + if(UPB_ENABLE_ASAN) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=address") + endif() + + if(UPB_ENABLE_UBSAN) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=address") + endif() + + include_directories(.) + include_directories(${CMAKE_CURRENT_BINARY_DIR}) + + if(APPLE) + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -undefined dynamic_lookup -flat_namespace") + elseif(UNIX) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--build-id") + endif() + + enable_testing() + + %(toplevel)s + + """) + +data = {} +converter = Converter() + +def GetDict(obj): + ret = {} + for k in dir(obj): + if not k.startswith("_"): + ret[k] = getattr(obj, k); + return ret + +globs = GetDict(converter) + +execfile("WORKSPACE", GetDict(WorkspaceFileFunctions(converter))) +execfile("BUILD", GetDict(BuildFileFunctions(converter))) + +with open(sys.argv[1], "w") as f: + f.write(converter.convert()) diff --git a/staleness_test.py b/tools/staleness_test.py similarity index 94% rename from staleness_test.py rename to tools/staleness_test.py index 43176cf8f3..045cd1a319 100644 --- a/staleness_test.py +++ b/tools/staleness_test.py @@ -6,7 +6,7 @@ with the actual list of files before we actually run the script. from __future__ import absolute_import -import staleness_test_lib +from tools import staleness_test_lib import unittest import sys diff --git a/staleness_test_lib.py b/tools/staleness_test_lib.py similarity index 100% rename from staleness_test_lib.py rename to tools/staleness_test_lib.py diff --git a/tools/upbc b/tools/upbc deleted file mode 100755 index bd12250c79..0000000000 --- a/tools/upbc +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -BASE=`dirname "$0"` -export LUA_CPATH="$BASE/../lua/?.so" -export LUA_PATH="$BASE/?.lua;$BASE/../lua/?.lua" -lua $BASE/upbc.lua "$@" From 5f575995b56f2ae97d3b9f5dc1efef233def7419 Mon Sep 17 00:00:00 2001 From: Josh Haberman Date: Mon, 12 Nov 2018 21:21:20 -0800 Subject: [PATCH 09/16] Added upb compiler written in C++ as a normal protoc plugin. --- upbc/generator.cc | 540 +++++++++++++++++++++++++++++++++++++++++ upbc/generator.h | 12 + upbc/main.cc | 9 + upbc/message_layout.cc | 173 +++++++++++++ upbc/message_layout.h | 104 ++++++++ 5 files changed, 838 insertions(+) create mode 100644 upbc/generator.cc create mode 100644 upbc/generator.h create mode 100644 upbc/main.cc create mode 100644 upbc/message_layout.cc create mode 100644 upbc/message_layout.h diff --git a/upbc/generator.cc b/upbc/generator.cc new file mode 100644 index 0000000000..56e6276300 --- /dev/null +++ b/upbc/generator.cc @@ -0,0 +1,540 @@ + +#include +#include +#include + +#include "absl/strings/ascii.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/substitute.h" +#include "google/protobuf/compiler/code_generator.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/io/zero_copy_stream.h" + +#include "upbc/generator.h" +#include "upbc/message_layout.h" + +namespace protoc = ::google::protobuf::compiler; +namespace protobuf = ::google::protobuf; + +static std::string StripExtension(absl::string_view fname) { + size_t lastdot = fname.find_last_of("."); + if (lastdot == std::string::npos) { + return std::string(fname); + } + return std::string(fname.substr(0, lastdot)); +} + +static std::string HeaderFilename(std::string proto_filename) { + return StripExtension(proto_filename) + ".upb.h"; +} + +static std::string SourceFilename(std::string proto_filename) { + return StripExtension(proto_filename) + ".upb.c"; +} + +class Output { + public: + Output(protobuf::io::ZeroCopyOutputStream* stream) : stream_(stream) {} + ~Output() { stream_->BackUp(size_); } + + template + void operator()(absl::string_view format, const Arg&... arg) { + Write(absl::Substitute(format, arg...)); + } + + private: + void Write(absl::string_view data) { + while (!data.empty()) { + RefreshOutput(); + size_t to_write = std::min(data.size(), size_); + memcpy(ptr_, data.data(), to_write); + data.remove_prefix(to_write); + ptr_ += to_write; + size_ -= to_write; + } + } + + void RefreshOutput() { + while (size_ == 0) { + void *ptr; + int size; + if (!stream_->Next(&ptr, &size)) { + fprintf(stderr, "upbc: Failed to write to to output\n"); + abort(); + } + ptr_ = static_cast(ptr); + size_ = size; + } + } + + protobuf::io::ZeroCopyOutputStream* stream_; + char *ptr_ = nullptr; + size_t size_ = 0; +}; + +namespace upbc { + +class Generator : public protoc::CodeGenerator { + ~Generator() override {} + bool Generate(const protobuf::FileDescriptor* file, + const std::string& parameter, protoc::GeneratorContext* context, + std::string* error) const override; + +}; + +void AddMessages(const protobuf::Descriptor* message, + std::vector* messages) { + messages->push_back(message); + for (int i = 0; i < message->nested_type_count(); i++) { + AddMessages(message->nested_type(i), messages); + } +} + +void AddEnums(const protobuf::Descriptor* message, + std::vector* enums) { + for (int i = 0; i < message->enum_type_count(); i++) { + enums->push_back(message->enum_type(i)); + } + for (int i = 0; i < message->nested_type_count(); i++) { + AddEnums(message->nested_type(i), enums); + } +} + +template +void SortDefs(std::vector* defs) { + std::sort(defs->begin(), defs->end(), + [](T a, T b) { return a->full_name() < b->full_name(); }); +} + +std::vector SortedMessages( + const protobuf::FileDescriptor* file) { + std::vector messages; + for (int i = 0; i < file->message_type_count(); i++) { + AddMessages(file->message_type(i), &messages); + } + //SortDefs(&messages); + return messages; +} + +std::vector SortedEnums( + const protobuf::FileDescriptor* file) { + std::vector enums; + for (int i = 0; i < file->enum_type_count(); i++) { + enums.push_back(file->enum_type(i)); + } + for (int i = 0; i < file->message_type_count(); i++) { + AddEnums(file->message_type(i), &enums); + } + SortDefs(&enums); + return enums; +} + +std::vector FieldNumberOrder( + const protobuf::Descriptor* message) { + std::vector messages; + for (int i = 0; i < message->field_count(); i++) { + messages.push_back(message->field(i)); + } + std::sort(messages.begin(), messages.end(), + [](const protobuf::FieldDescriptor* a, + const protobuf::FieldDescriptor* b) { + return a->number() < b->number(); + }); + return messages; +} + +std::vector SortedSubmessages( + const protobuf::Descriptor* message) { + std::vector ret; + for (int i = 0; i < message->field_count(); i++) { + if (message->field(i)->cpp_type() == + protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + ret.push_back(message->field(i)); + } + } + std::sort(ret.begin(), ret.end(), + [](const protobuf::FieldDescriptor* a, + const protobuf::FieldDescriptor* b) { + return a->message_type()->full_name() < + b->message_type()->full_name(); + }); + return ret; +} + +std::string ToCIdent(absl::string_view str) { + return absl::StrReplaceAll(str, {{".", "_"}, {"/", "_"}}); +} + +std::string ToPreproc(absl::string_view str) { + return absl::AsciiStrToUpper(ToCIdent(str)); +} + +std::string EnumValueSymbol(const protobuf::EnumValueDescriptor* value) { + return ToCIdent(value->full_name()); +} + +std::string GetSizeInit(const MessageLayout::Size& size) { + return absl::Substitute("UPB_SIZE($0, $1)", size.size32, size.size64); +} + +std::string CTypeInternal(const protobuf::FieldDescriptor* field, + bool is_const) { + std::string maybe_const = is_const ? "const " : ""; + if (field->label() == protobuf::FieldDescriptor::LABEL_REPEATED) { + return maybe_const + "upb_array*"; + } + + switch (field->cpp_type()) { + case protobuf::FieldDescriptor::CPPTYPE_MESSAGE: { + std::string maybe_struct = + field->file() != field->message_type()->file() ? "struct " : ""; + return maybe_const + maybe_struct + + ToCIdent(field->message_type()->full_name()) + "*"; + } + case protobuf::FieldDescriptor::CPPTYPE_ENUM: + return ToCIdent(field->enum_type()->full_name()); + case protobuf::FieldDescriptor::CPPTYPE_BOOL: + return "bool"; + case protobuf::FieldDescriptor::CPPTYPE_FLOAT: + return "float"; + case protobuf::FieldDescriptor::CPPTYPE_INT32: + return "int32_t"; + case protobuf::FieldDescriptor::CPPTYPE_UINT32: + return "uint32_t"; + case protobuf::FieldDescriptor::CPPTYPE_DOUBLE: + return "double"; + case protobuf::FieldDescriptor::CPPTYPE_INT64: + return "int64_t"; + case protobuf::FieldDescriptor::CPPTYPE_UINT64: + return "uint64_t"; + case protobuf::FieldDescriptor::CPPTYPE_STRING: + return "upb_stringview"; + default: + fprintf(stderr, "Unexpected type"); + abort(); + } +} + +std::string FieldDefault(const protobuf::FieldDescriptor* field) { + switch (field->cpp_type()) { + case protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + return "NULL"; + case protobuf::FieldDescriptor::CPPTYPE_STRING: + return absl::Substitute("upb_stringview_make(\"$0\", strlen(\"$0\"))", + absl::CEscape(field->default_value_string())); + case protobuf::FieldDescriptor::CPPTYPE_INT32: + return absl::StrCat(field->default_value_int32()); + case protobuf::FieldDescriptor::CPPTYPE_INT64: + return absl::StrCat(field->default_value_int64()); + case protobuf::FieldDescriptor::CPPTYPE_UINT32: + return absl::StrCat(field->default_value_uint32()); + case protobuf::FieldDescriptor::CPPTYPE_UINT64: + return absl::StrCat(field->default_value_uint64()); + case protobuf::FieldDescriptor::CPPTYPE_FLOAT: + return absl::StrCat(field->default_value_float()); + case protobuf::FieldDescriptor::CPPTYPE_DOUBLE: + return absl::StrCat(field->default_value_double()); + case protobuf::FieldDescriptor::CPPTYPE_BOOL: + return field->default_value_bool() ? "true" : "false"; + case protobuf::FieldDescriptor::CPPTYPE_ENUM: + return EnumValueSymbol(field->default_value_enum()); + } + ABSL_ASSERT(false); + return "XXX"; +} + +std::string CType(const protobuf::FieldDescriptor* field) { + return CTypeInternal(field, false); +} + +std::string CTypeConst(const protobuf::FieldDescriptor* field) { + return CTypeInternal(field, true); +} + +void DumpEnumValues(const protobuf::EnumDescriptor* desc, Output& output) { + std::vector values; + for (int i = 0; i < desc->value_count(); i++) { + values.push_back(desc->value(i)); + } + std::sort(values.begin(), values.end(), + [](const protobuf::EnumValueDescriptor* a, + const protobuf::EnumValueDescriptor* b) { + return a->number() < b->number(); + }); + + for (size_t i = 0; i < values.size(); i++) { + auto value = values[i]; + output(" $0 = $1", EnumValueSymbol(value), value->number()); + if (i != values.size() - 1) { + output(","); + } + output("\n"); + } +} + +void EmitFileWarning(const protobuf::FileDescriptor* file, Output& output) { + output( + "/* This file was generated by upbc (the upb compiler) from the input\n" + " * file:\n" + " *\n" + " * $0\n" + " *\n" + " * Do not edit -- your changes will be discarded when the file is\n" + " * regenerated. */\n\n", + file->name()); +} + +void GenerateMessageInHeader(const protobuf::Descriptor* message, Output& output) { + MessageLayout layout(message); + + output("/* $0 */\n\n", message->full_name()); + std::string msgname = ToCIdent(message->full_name()); + output( + "extern const upb_msglayout $0_msginit;\n" + "UPB_INLINE $0 *$0_new(upb_arena *arena) {\n" + " return upb_msg_new(&$0_msginit, arena);\n" + "}\n" + "UPB_INLINE $0 *$0_parsenew(upb_stringview buf, upb_arena *arena) {\n" + " $0 *ret = $0_new(arena);\n" + " return (ret && upb_decode(buf, ret, &$0_msginit)) ? ret : NULL;\n" + "}\n" + "UPB_INLINE char *$0_serialize(const $0 *msg, upb_arena *arena, size_t " + "*len) {\n" + " return upb_encode(msg, &$0_msginit, arena, len);\n" + "}\n" + "\n", + msgname); + + for (int i = 0; i < message->oneof_decl_count(); i++) { + const protobuf::OneofDescriptor* oneof = message->oneof_decl(i); + std::string fullname = ToCIdent(oneof->full_name()); + output("typedef enum {\n"); + for (int i = 0; i < oneof->field_count(); i++) { + const protobuf::FieldDescriptor* field = oneof->field(i); + output(" $0_$1 = $2,\n", fullname, field->name(), field->number()); + } + output( + " $0_NOT_SET = 0,\n" + "} $0_oneofcases;\n", + fullname); + output( + "UPB_INLINE $0_oneofcases $1_$2_case(const $1* msg) { " + "return UPB_FIELD_AT(msg, int, $3); }\n" + "\n", + fullname, msgname, oneof->name(), + GetSizeInit(layout.GetOneofCaseOffset(oneof))); + } + + for (auto field : FieldNumberOrder(message)) { + output("UPB_INLINE $0 $1_$2(const $1 *msg) {", CTypeConst(field), msgname, + field->name()); + if (field->containing_oneof()) { + output(" return UPB_READ_ONEOF(msg, $0, $1, $2, $3, $4); }\n", + CTypeConst(field), GetSizeInit(layout.GetFieldOffset(field)), + GetSizeInit(layout.GetOneofCaseOffset(field->containing_oneof())), + field->number(), FieldDefault(field)); + } else { + output(" return UPB_FIELD_AT(msg, $0, $1); }\n", CTypeConst(field), + GetSizeInit(layout.GetFieldOffset(field))); + } + } + + output("\n"); + + for (auto field : FieldNumberOrder(message)) { + output("UPB_INLINE void $0_set_$1($0 *msg, $2 value) { ", msgname, + field->name(), CType(field)); + if (field->containing_oneof()) { + output("UPB_WRITE_ONEOF(msg, $0, $1, value, $2, $3); }\n", CType(field), + GetSizeInit(layout.GetFieldOffset(field)), + GetSizeInit(layout.GetOneofCaseOffset(field->containing_oneof())), + field->number()); + } else { + output("UPB_FIELD_AT(msg, $0, $1) = value; }\n", CType(field), + GetSizeInit(layout.GetFieldOffset(field))); + } + } + + output("\n\n"); +} + +void WriteHeader(const protobuf::FileDescriptor* file, Output& output) { + EmitFileWarning(file, output); + output( + "#ifndef $0_UPB_H_\n" + "#define $0_UPB_H_\n\n" + "#include \"upb/msg.h\"\n\n" + "#include \"upb/decode.h\"\n" + "#include \"upb/encode.h\"\n" + "#include \"upb/port_def.inc\"\n" + "UPB_BEGIN_EXTERN_C\n\n", + ToPreproc(file->name())); + + // Forward-declare types defined in this file. + for (auto message : SortedMessages(file)) { + output("struct $0;\n", ToCIdent(message->full_name())); + } + for (auto message : SortedMessages(file)) { + output("typedef struct $0 $0;\n", ToCIdent(message->full_name())); + }; + + // Forward-declare types not in this file, but used as submessages. + std::set forward_names; + for (auto message : SortedMessages(file)) { + for (int i = 0; i < message->field_count(); i++) { + const protobuf::FieldDescriptor* field = message->field(i); + if (field->cpp_type() == protobuf::FieldDescriptor::CPPTYPE_MESSAGE && + field->file() != message->file()) { + forward_names.insert(ToCIdent(field->message_type()->full_name())); + } + } + } + for (const auto& name : forward_names) { + output("struct $0;\n", name); + } + + output( + "\n" + "/* Enums */\n\n"); + for (auto enumdesc : SortedEnums(file)) { + output("typedef enum {\n"); + DumpEnumValues(enumdesc, output); + output("} $0;\n\n", ToCIdent(enumdesc->full_name())); + } + + for (auto message : SortedMessages(file)) { + GenerateMessageInHeader(message, output); + } + + output( + "UPB_END_EXTERN_C\n" + "\n" + "#include \"upb/port_undef.inc\"\n" + "\n" + "#endif /* $0_UPB_H_ */\n", + ToPreproc(file->name())); +} + +void WriteSource(const protobuf::FileDescriptor* file, Output& output) { + EmitFileWarning(file, output); + + output( + "#include \n" + "#include \"upb/msg.h\"\n" + "#include \"$0\"\n", + HeaderFilename(file->name())); + + for (int i = 0; i < file->dependency_count(); i++) { + output("#include \"$0\"\n", HeaderFilename(file->dependency(i)->name())); + } + + output( + "\n" + "#include \"upb/port_def.inc\"\n" + "\n"); + + + for (auto message : SortedMessages(file)) { + std::string msgname = ToCIdent(message->full_name()); + std::string fields_array_ref = "NULL"; + std::string submsgs_array_ref = "NULL"; + std::string oneofs_array_ref = "NULL"; + std::unordered_map submsg_indexes; + MessageLayout layout(message); + std::vector sorted_submsgs = + SortedSubmessages(message); + + if (!sorted_submsgs.empty()) { + // TODO(haberman): could save a little bit of space by only generating a + // "submsgs" array for every strongly-connected component. + std::string submsgs_array_name = msgname + "_submsgs"; + submsgs_array_ref = "&" + submsgs_array_name + "[0]"; + output("static const upb_msglayout *const $0[$1] = {\n", + submsgs_array_name, sorted_submsgs.size()); + + int i = 0; + for (auto submsg : sorted_submsgs) { + if (submsg_indexes.find(submsg->message_type()) != + submsg_indexes.end()) { + continue; + } + output(" &$0_msginit,\n", + ToCIdent(submsg->message_type()->full_name())); + submsg_indexes[submsg->message_type()] = i++; + } + + output("};\n\n"); + } + + std::vector field_number_order = + FieldNumberOrder(message); + if (!field_number_order.empty()) { + std::string fields_array_name = msgname + "__fields"; + fields_array_ref = "&" + fields_array_name + "[0]"; + output("static const upb_msglayout_field $0[$1] = {\n", + fields_array_name, field_number_order.size()); + for (auto field : field_number_order) { + int submsg_index = 0; + std::string presence = "0"; + + if (field->cpp_type() == protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + submsg_index = submsg_indexes[field->message_type()]; + } + + if (MessageLayout::HasHasbit(field)) { + presence = absl::StrCat(layout.GetHasbitIndex(field) + 1); + } else if (field->containing_oneof()) { + MessageLayout::Size case_offset = + layout.GetOneofCaseOffset(field->containing_oneof()); + + // Our encoding that distinguishes oneofs from presence-having fields. + case_offset.size32 = -case_offset.size32 - 1; + case_offset.size64 = -case_offset.size64 - 1; + presence = GetSizeInit(case_offset); + } + + output(" {$0, $1, $2, $3, $4, $5},\n", + field->number(), + GetSizeInit(layout.GetFieldOffset(field)), + presence, + submsg_index, + field->type(), + field->label()); + } + output("};\n\n"); + } + + output("const upb_msglayout $0_msginit = {\n", msgname); + output(" $0,\n", submsgs_array_ref); + output(" $0,\n", fields_array_ref); + output(" $0, $1, $2,\n", GetSizeInit(layout.message_size()), + field_number_order.size(), + "false" // TODO: extendable + ); + + output("};\n\n"); + } + + output("#include \"upb/port_undef.inc\"\n"); + output("\n"); +} + +bool Generator::Generate(const protobuf::FileDescriptor* file, + const std::string& parameter, + protoc::GeneratorContext* context, + std::string* error) const { + Output h_output(context->Open(HeaderFilename(file->name()))); + WriteHeader(file, h_output); + + Output c_output(context->Open(SourceFilename(file->name()))); + WriteSource(file, c_output); + + return true; +} + +std::unique_ptr GetGenerator() { + return std::unique_ptr( + new Generator()); +} + +} // namespace upbc diff --git a/upbc/generator.h b/upbc/generator.h new file mode 100644 index 0000000000..ed6cedc6c7 --- /dev/null +++ b/upbc/generator.h @@ -0,0 +1,12 @@ + +#ifndef UPBC_GENERATOR_H_ +#define UPBC_GENERATOR_H_ + +#include +#include + +namespace upbc { +std::unique_ptr GetGenerator(); +} + +#endif // UPBC_GENERATOR_H_ diff --git a/upbc/main.cc b/upbc/main.cc new file mode 100644 index 0000000000..a9682a9c1a --- /dev/null +++ b/upbc/main.cc @@ -0,0 +1,9 @@ + +#include + +#include "upbc/generator.h" + +int main(int argc, char** argv) { + return google::protobuf::compiler::PluginMain(argc, argv, + upbc::GetGenerator().get()); +} diff --git a/upbc/message_layout.cc b/upbc/message_layout.cc new file mode 100644 index 0000000000..b6614f0f66 --- /dev/null +++ b/upbc/message_layout.cc @@ -0,0 +1,173 @@ + +#include "upbc/message_layout.h" + +namespace upbc { + +namespace protobuf = ::google::protobuf; + +static int64_t DivRoundUp(int64_t a, int64_t b) { + ABSL_ASSERT(a >= 0); + ABSL_ASSERT(b > 0); + return (a + b - 1) / b; +} + +MessageLayout::Size MessageLayout::Place( + MessageLayout::SizeAndAlign size_and_align) { + Size offset = size_; + offset.AlignUp(size_and_align.align); + size_ = offset; + size_.Add(size_and_align.size); + //maxalign_.MaxFrom(size_and_align.align); + maxalign_.MaxFrom(size_and_align.size); + return offset; +} + +bool MessageLayout::HasHasbit(const protobuf::FieldDescriptor* field) { + return field->file()->syntax() == protobuf::FileDescriptor::SYNTAX_PROTO2 && + field->label() != protobuf::FieldDescriptor::LABEL_REPEATED && + !field->containing_oneof(); +} + +MessageLayout::SizeAndAlign MessageLayout::SizeOf( + const protobuf::FieldDescriptor* field) { + if (field->label() == protobuf::FieldDescriptor::LABEL_REPEATED || + field->cpp_type() == protobuf::FieldDescriptor::CPPTYPE_MESSAGE) { + return {{4, 8}, {4, 8}}; + } + + switch (field->cpp_type()) { + case protobuf::FieldDescriptor::CPPTYPE_STRING: + // upb_stringview + // return {{8, 16}, {4, 8}}; + return {{8, 16}, {8, 16}}; + case protobuf::FieldDescriptor::CPPTYPE_BOOL: + return {{1, 1}, {1, 1}}; + case protobuf::FieldDescriptor::CPPTYPE_FLOAT: + case protobuf::FieldDescriptor::CPPTYPE_INT32: + case protobuf::FieldDescriptor::CPPTYPE_UINT32: + return {{4, 4}, {4, 4}}; + default: + return {{8, 8}, {8, 8}}; + } +} + +int64_t MessageLayout::FieldLayoutRank(const protobuf::FieldDescriptor* field) { + // Order: + // 1, 2, 3. primitive fields (8, 4, 1 byte) + // 4. string fields + // 5. submessage fields + // 6. repeated fields + // + // This has the following nice properties: + // + // 1. padding alignment is (nearly) minimized. + // 2. fields that might have defaults (1-4) are segregated + // from fields that are always zero-initialized (5-7). + // + // We skip oneof fields, because they are emitted in a separate pass. + int64_t rank; + if (field->containing_oneof()) { + fprintf(stderr, "shouldn't have oneofs here.\n"); + abort(); + } else if (field->label() == protobuf::FieldDescriptor::LABEL_REPEATED) { + rank = 6; + } else { + switch (field->cpp_type()) { + case protobuf::FieldDescriptor::CPPTYPE_MESSAGE: + rank = 5; + break; + case protobuf::FieldDescriptor::CPPTYPE_STRING: + rank = 4; + break; + case protobuf::FieldDescriptor::CPPTYPE_BOOL: + rank = 3; + break; + case protobuf::FieldDescriptor::CPPTYPE_FLOAT: + case protobuf::FieldDescriptor::CPPTYPE_INT32: + case protobuf::FieldDescriptor::CPPTYPE_UINT32: + rank = 2; + break; + default: + rank = 1; + break; + } + } + + // Break ties with field number. + return (rank << 29) | field->number(); +} + +void MessageLayout::ComputeLayout(const protobuf::Descriptor* descriptor) { + size_ = Size{0, 0}; + maxalign_ = Size{0, 0}; + PlaceNonOneofFields(descriptor); + PlaceOneofFields(descriptor); + + // Align overall size up to max size. + size_.AlignUp(maxalign_); +} + +void MessageLayout::PlaceNonOneofFields( + const protobuf::Descriptor* descriptor) { + std::vector field_order; + for (int i = 0; i < descriptor->field_count(); i++) { + const protobuf::FieldDescriptor* field = descriptor->field(i); + if (!field->containing_oneof()) { + field_order.push_back(descriptor->field(i)); + } + } + std::sort(field_order.begin(), field_order.end(), + [](const protobuf::FieldDescriptor* a, + const protobuf::FieldDescriptor* b) { + return FieldLayoutRank(a) < FieldLayoutRank(b); + }); + + // Place/count hasbits. + int hasbit_count = 0; + for (auto field : field_order) { + if (HasHasbit(field)) { + hasbit_indexes_[field] = hasbit_count++; + } + } + + // Place hasbits at the beginning. + int64_t hasbit_bytes = DivRoundUp(hasbit_count, 8); + Place(SizeAndAlign{{hasbit_bytes, hasbit_bytes}, {1, 1}}); + + // Place non-oneof fields. + for (auto field : field_order) { + field_offsets_[field] = Place(SizeOf(field)); + } +} + +void MessageLayout::PlaceOneofFields(const protobuf::Descriptor* descriptor) { + std::vector oneof_order; + for (int i = 0; i < descriptor->oneof_decl_count(); i++) { + oneof_order.push_back(descriptor->oneof_decl(i)); + } + std::sort(oneof_order.begin(), oneof_order.end(), + [](const protobuf::OneofDescriptor* a, + const protobuf::OneofDescriptor* b) { + return a->full_name() < b->full_name(); + }); + + for (auto oneof : oneof_order) { + SizeAndAlign oneof_maxsize{{0, 0}, {0, 0}}; + // Calculate max size. + for (int i = 0; i < oneof->field_count(); i++) { + oneof_maxsize.MaxFrom(SizeOf(oneof->field(i))); + } + + // Place discriminator enum and data. + Size data = Place(oneof_maxsize); + Size discriminator = Place(SizeAndAlign{{4, 4}, {4, 4}}); + + oneof_case_offsets_[oneof] = discriminator; + + for (int i = 0; i < oneof->field_count(); i++) { + field_offsets_[oneof->field(i)] = data; + } + } +} + +} // namespace upbc diff --git a/upbc/message_layout.h b/upbc/message_layout.h new file mode 100644 index 0000000000..bdcc3360ed --- /dev/null +++ b/upbc/message_layout.h @@ -0,0 +1,104 @@ + +#ifndef UPBC_MESSAGE_LAYOUT_H +#define UPBC_MESSAGE_LAYOUT_H + +#include +#include "absl/base/macros.h" +#include "google/protobuf/descriptor.h" + +namespace upbc { + +class MessageLayout { + public: + struct Size { + void Add(const Size& other) { + size32 += other.size32; + size64 += other.size64; + } + + void MaxFrom(const Size& other) { + size32 = std::max(size32, other.size32); + size64 = std::max(size64, other.size64); + } + + void AlignUp(const Size& align) { + size32 = Align(size32, align.size32); + size64 = Align(size64, align.size64); + } + + int64_t size32; + int64_t size64; + }; + + struct SizeAndAlign { + Size size; + Size align; + + void MaxFrom(const SizeAndAlign& other) { + size.MaxFrom(other.size); + align.MaxFrom(other.align); + } + }; + + MessageLayout(const google::protobuf::Descriptor* descriptor) { + ComputeLayout(descriptor); + } + + Size GetFieldOffset(const google::protobuf::FieldDescriptor* field) const { + return GetMapValue(field_offsets_, field); + } + + Size GetOneofCaseOffset( + const google::protobuf::OneofDescriptor* oneof) const { + return GetMapValue(oneof_case_offsets_, oneof); + } + + int GetHasbitIndex(const google::protobuf::FieldDescriptor* field) const { + return GetMapValue(hasbit_indexes_, field); + } + + Size message_size() const { return size_; } + + static bool HasHasbit(const google::protobuf::FieldDescriptor* field); + + private: + void ComputeLayout(const google::protobuf::Descriptor* descriptor); + void PlaceNonOneofFields(const google::protobuf::Descriptor* descriptor); + void PlaceOneofFields(const google::protobuf::Descriptor* descriptor); + Size Place(SizeAndAlign size_and_align); + + template + static V GetMapValue(const std::unordered_map& map, K key) { + auto iter = map.find(key); + if (iter == map.end()) { + fprintf(stderr, "No value for field.\n"); + abort(); + } + return iter->second; + } + + static bool IsPowerOfTwo(size_t val) { + return (val & (val - 1)) == 0; + } + + static size_t Align(size_t val, size_t align) { + ABSL_ASSERT(IsPowerOfTwo(align)); + return (val + align - 1) & ~(align - 1); + } + + static SizeAndAlign SizeOf(const google::protobuf::FieldDescriptor* field); + static int64_t FieldLayoutRank(const google::protobuf::FieldDescriptor* field); + + std::unordered_map + field_offsets_; + std::unordered_map + hasbit_indexes_; + std::unordered_map + oneof_case_offsets_; + Size maxalign_; + Size size_; +}; + +} // namespace upbc + +#endif // UPBC_MESSAGE_LAYOUT_H From 9ce4a77f61c134bbed28bfd5be5cd7dc0e80f5e3 Mon Sep 17 00:00:00 2001 From: Josh Haberman Date: Wed, 14 Nov 2018 16:46:19 -0800 Subject: [PATCH 10/16] Updates to build system for upb C++ compiler. --- BUILD | 42 +++++++++++++++++++++++++++++++--------- WORKSPACE | 6 ++++++ build_defs.bzl | 28 +++++++++++++-------------- tools/make_cmakelists.py | 2 +- 4 files changed, 54 insertions(+), 24 deletions(-) diff --git a/BUILD b/BUILD index 458463e089..6503ed0b67 100644 --- a/BUILD +++ b/BUILD @@ -249,7 +249,7 @@ cc_test( upb_proto_library( name = "conformance_proto_upb", - upbc = ":upbc", + upbc = ":protoc-gen-upb", deps = [ "@com_google_protobuf//:conformance_proto", "@com_google_protobuf//:test_messages_proto3_proto", @@ -373,13 +373,33 @@ lua_test( # upb compiler ################################################################# lua_binary( - name = "upbc", + name = "lua_upbc", luadeps = [ "lua/upbc_lib", ], luamain = "tools/upbc.lua", ) +cc_library( + name = "upbc_generator", + hdrs = ["upbc/generator.h"], + srcs = ["upbc/generator.cc", "upbc/message_layout.h", "upbc/message_layout.cc"], + deps = [ + "@com_google_protobuf//:protobuf", + "@com_google_protobuf//:protoc_lib", + "@absl//absl/strings", + ], +) + +cc_binary( + name = "protoc-gen-upb", + srcs = ["upbc/main.cc"], + deps = [ + ":upbc_generator", + "@com_google_protobuf//:protoc_lib", + ], +) + # Test the CMake build ######################################################### make_shell_script( @@ -394,6 +414,7 @@ sh_test( data = glob([ "CMakeLists.txt", "google/**/*", + "upbc/**/*", "upb/**/*", "tests/**/*", ]) + [ @@ -458,8 +479,8 @@ genrule( "generated/upb/descriptor/descriptor.upbdefs.h", "generated/upb/descriptor/descriptor.upbdefs.c", ], - cmd = "UPBC=$$PWD/$(location :upbc); INFILE=$$PWD/$<; cd $(GENDIR)/generated && $$UPBC --generate-upbdefs $$INFILE", - tools = [":upbc"], + cmd = "UPBC=$$PWD/$(location :lua_upbc); INFILE=$$PWD/$<; cd $(GENDIR)/generated && $$UPBC --generate-upbdefs $$INFILE", + tools = [":lua_upbc"], ) proto_library( @@ -478,13 +499,16 @@ genrule( genrule( name = "generate_descriptor_c", - srcs = ["generated/google/protobuf/descriptor.pb"], + srcs = ["google/protobuf/descriptor.proto"], outs = [ "generated/google/protobuf/descriptor.upb.h", "generated/google/protobuf/descriptor.upb.c", ], - cmd = "UPBC=$$PWD/$(location :upbc); INFILE=$$PWD/$<; cd $(GENDIR)/generated && $$UPBC $$INFILE", - tools = [":upbc"], + cmd = "$(location @com_google_protobuf//:protoc) $< --upb_out=$(GENDIR)/generated --plugin=protoc-gen-upb=$(location :protoc-gen-upb)", + tools = [ + "@com_google_protobuf//:protoc", + ":protoc-gen-upb" + ], ) proto_library( @@ -506,8 +530,8 @@ genrule( "generated/tests/json/test.upbdefs.h", "generated/tests/json/test.upbdefs.c", ], - cmd = "UPBC=$$PWD/$(location :upbc); INFILE=$$PWD/$<; cd $(GENDIR)/generated && $$UPBC --generate-upbdefs $$INFILE", - tools = [":upbc"], + cmd = "UPBC=$$PWD/$(location :lua_upbc); INFILE=$$PWD/$<; cd $(GENDIR)/generated && $$UPBC --generate-upbdefs $$INFILE", + tools = [":lua_upbc"], ) genrule( diff --git a/WORKSPACE b/WORKSPACE index b4d19c50a8..8d208525c3 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -23,6 +23,12 @@ git_repository( #tag = "conformance-build-tag", ) +git_repository( + name = "absl", + commit = "070f6e47b33a2909d039e620c873204f78809492", + remote = "https://github.com/abseil/abseil-cpp.git", +) + http_archive( name = "ragel", sha256 = "5f156edb65d20b856d638dd9ee2dfb43285914d9aa2b6ec779dac0270cd56c3f", diff --git a/build_defs.bzl b/build_defs.bzl index 0e58bbbca4..c1dd4223d5 100644 --- a/build_defs.bzl +++ b/build_defs.bzl @@ -218,31 +218,26 @@ def _remove_up(string): return _remove_suffix(string, ".proto") def _upb_proto_library_srcs_impl(ctx): - descriptors = [] + sources = [] outs = [] + include_dirs = {} for dep in ctx.attr.deps: if hasattr(dep, 'proto'): - for desc in dep.proto.transitive_descriptor_sets: - descriptors.append(desc) for src in dep.proto.transitive_sources: + sources.append(src) + include_dirs[_remove_suffix(src.path, _remove_up(src.short_path) + "." + src.extension)] = True outs.append(ctx.actions.declare_file(_remove_up(src.short_path) + ".upb.h")) outs.append(ctx.actions.declare_file(_remove_up(src.short_path) + ".upb.c")) outdir = _remove_suffix(outs[-1].path, _remove_up(src.short_path) + ".upb.c") - concatenated = ctx.actions.declare_file(ctx.label.name + "_concatenated_descriptor.bin") - descriptor_paths = [d.path for d in descriptors] + source_paths = [d.path for d in sources] + include_args = ["-I" + root for root in include_dirs.keys()] - ctx.actions.run_shell( - inputs = descriptors, - outputs = [concatenated], - progress_message = "Concatenating descriptors", - command = "cat %s > %s" % (" ".join(descriptor_paths), concatenated.path), - ) ctx.actions.run( - inputs = [concatenated], + inputs = [ctx.executable.upbc] + sources, outputs = outs, - executable = ctx.executable.upbc, - arguments = ["--outdir", outdir, concatenated.path], + executable = ctx.executable.protoc, + arguments = ["--upb_out", outdir, "--plugin=protoc-gen-upb=" + ctx.executable.upbc.path] + include_args + source_paths, progress_message = "Generating upb protos", ) @@ -255,6 +250,11 @@ _upb_proto_library_srcs = rule( executable = True, cfg = "host", ), + "protoc": attr.label( + executable = True, + cfg = "host", + default = "@com_google_protobuf//:protoc", + ), "deps": attr.label_list(), } ) diff --git a/tools/make_cmakelists.py b/tools/make_cmakelists.py index 30e1b93dc6..d7996d8748 100755 --- a/tools/make_cmakelists.py +++ b/tools/make_cmakelists.py @@ -34,7 +34,7 @@ class BuildFileFunctions(object): pass def cc_library(self, **kwargs): - if kwargs["name"] == "amalgamation": + if kwargs["name"] == "amalgamation" or kwargs["name"] == "upbc_generator": return files = kwargs.get("srcs", []) + kwargs.get("hdrs", []) self.converter.toplevel += "add_library(%s\n %s)\n" % ( From 0811b6fead2e2f728608a00bed714e7d4fbb5e47 Mon Sep 17 00:00:00 2001 From: Vishal Powar Date: Tue, 20 Nov 2018 11:36:38 -0800 Subject: [PATCH 11/16] Explicit cast when creating a new struct --- upbc/generator.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upbc/generator.cc b/upbc/generator.cc index 56e6276300..0f7d14da07 100644 --- a/upbc/generator.cc +++ b/upbc/generator.cc @@ -292,7 +292,7 @@ void GenerateMessageInHeader(const protobuf::Descriptor* message, Output& output output( "extern const upb_msglayout $0_msginit;\n" "UPB_INLINE $0 *$0_new(upb_arena *arena) {\n" - " return upb_msg_new(&$0_msginit, arena);\n" + " return ($0 *)upb_msg_new(&$0_msginit, arena);\n" "}\n" "UPB_INLINE $0 *$0_parsenew(upb_stringview buf, upb_arena *arena) {\n" " $0 *ret = $0_new(arena);\n" From e2ad6bd1467f8b708715b4572693c0776d4bd919 Mon Sep 17 00:00:00 2001 From: Vishal Powar Date: Tue, 20 Nov 2018 12:21:50 -0800 Subject: [PATCH 12/16] Fix tests for generated code --- google/protobuf/descriptor.upb.h | 54 ++++++++++++++++---------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/google/protobuf/descriptor.upb.h b/google/protobuf/descriptor.upb.h index 7afa8c5fa8..6289095153 100644 --- a/google/protobuf/descriptor.upb.h +++ b/google/protobuf/descriptor.upb.h @@ -128,7 +128,7 @@ typedef enum { extern const upb_msglayout google_protobuf_FileDescriptorSet_msginit; UPB_INLINE google_protobuf_FileDescriptorSet *google_protobuf_FileDescriptorSet_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_FileDescriptorSet_msginit, arena); + return (google_protobuf_FileDescriptorSet *)upb_msg_new(&google_protobuf_FileDescriptorSet_msginit, arena); } UPB_INLINE google_protobuf_FileDescriptorSet *google_protobuf_FileDescriptorSet_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_FileDescriptorSet *ret = google_protobuf_FileDescriptorSet_new(arena); @@ -147,7 +147,7 @@ UPB_INLINE void google_protobuf_FileDescriptorSet_set_file(google_protobuf_FileD extern const upb_msglayout google_protobuf_FileDescriptorProto_msginit; UPB_INLINE google_protobuf_FileDescriptorProto *google_protobuf_FileDescriptorProto_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_FileDescriptorProto_msginit, arena); + return (google_protobuf_FileDescriptorProto *)upb_msg_new(&google_protobuf_FileDescriptorProto_msginit, arena); } UPB_INLINE google_protobuf_FileDescriptorProto *google_protobuf_FileDescriptorProto_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_FileDescriptorProto *ret = google_protobuf_FileDescriptorProto_new(arena); @@ -188,7 +188,7 @@ UPB_INLINE void google_protobuf_FileDescriptorProto_set_syntax(google_protobuf_F extern const upb_msglayout google_protobuf_DescriptorProto_msginit; UPB_INLINE google_protobuf_DescriptorProto *google_protobuf_DescriptorProto_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_DescriptorProto_msginit, arena); + return (google_protobuf_DescriptorProto *)upb_msg_new(&google_protobuf_DescriptorProto_msginit, arena); } UPB_INLINE google_protobuf_DescriptorProto *google_protobuf_DescriptorProto_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_DescriptorProto *ret = google_protobuf_DescriptorProto_new(arena); @@ -225,7 +225,7 @@ UPB_INLINE void google_protobuf_DescriptorProto_set_reserved_name(google_protobu extern const upb_msglayout google_protobuf_DescriptorProto_ExtensionRange_msginit; UPB_INLINE google_protobuf_DescriptorProto_ExtensionRange *google_protobuf_DescriptorProto_ExtensionRange_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_DescriptorProto_ExtensionRange_msginit, arena); + return (google_protobuf_DescriptorProto_ExtensionRange *)upb_msg_new(&google_protobuf_DescriptorProto_ExtensionRange_msginit, arena); } UPB_INLINE google_protobuf_DescriptorProto_ExtensionRange *google_protobuf_DescriptorProto_ExtensionRange_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_DescriptorProto_ExtensionRange *ret = google_protobuf_DescriptorProto_ExtensionRange_new(arena); @@ -248,7 +248,7 @@ UPB_INLINE void google_protobuf_DescriptorProto_ExtensionRange_set_options(googl extern const upb_msglayout google_protobuf_DescriptorProto_ReservedRange_msginit; UPB_INLINE google_protobuf_DescriptorProto_ReservedRange *google_protobuf_DescriptorProto_ReservedRange_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_DescriptorProto_ReservedRange_msginit, arena); + return (google_protobuf_DescriptorProto_ReservedRange *)upb_msg_new(&google_protobuf_DescriptorProto_ReservedRange_msginit, arena); } UPB_INLINE google_protobuf_DescriptorProto_ReservedRange *google_protobuf_DescriptorProto_ReservedRange_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_DescriptorProto_ReservedRange *ret = google_protobuf_DescriptorProto_ReservedRange_new(arena); @@ -269,7 +269,7 @@ UPB_INLINE void google_protobuf_DescriptorProto_ReservedRange_set_end(google_pro extern const upb_msglayout google_protobuf_ExtensionRangeOptions_msginit; UPB_INLINE google_protobuf_ExtensionRangeOptions *google_protobuf_ExtensionRangeOptions_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_ExtensionRangeOptions_msginit, arena); + return (google_protobuf_ExtensionRangeOptions *)upb_msg_new(&google_protobuf_ExtensionRangeOptions_msginit, arena); } UPB_INLINE google_protobuf_ExtensionRangeOptions *google_protobuf_ExtensionRangeOptions_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_ExtensionRangeOptions *ret = google_protobuf_ExtensionRangeOptions_new(arena); @@ -288,7 +288,7 @@ UPB_INLINE void google_protobuf_ExtensionRangeOptions_set_uninterpreted_option(g extern const upb_msglayout google_protobuf_FieldDescriptorProto_msginit; UPB_INLINE google_protobuf_FieldDescriptorProto *google_protobuf_FieldDescriptorProto_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_FieldDescriptorProto_msginit, arena); + return (google_protobuf_FieldDescriptorProto *)upb_msg_new(&google_protobuf_FieldDescriptorProto_msginit, arena); } UPB_INLINE google_protobuf_FieldDescriptorProto *google_protobuf_FieldDescriptorProto_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_FieldDescriptorProto *ret = google_protobuf_FieldDescriptorProto_new(arena); @@ -325,7 +325,7 @@ UPB_INLINE void google_protobuf_FieldDescriptorProto_set_json_name(google_protob extern const upb_msglayout google_protobuf_OneofDescriptorProto_msginit; UPB_INLINE google_protobuf_OneofDescriptorProto *google_protobuf_OneofDescriptorProto_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_OneofDescriptorProto_msginit, arena); + return (google_protobuf_OneofDescriptorProto *)upb_msg_new(&google_protobuf_OneofDescriptorProto_msginit, arena); } UPB_INLINE google_protobuf_OneofDescriptorProto *google_protobuf_OneofDescriptorProto_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_OneofDescriptorProto *ret = google_protobuf_OneofDescriptorProto_new(arena); @@ -346,7 +346,7 @@ UPB_INLINE void google_protobuf_OneofDescriptorProto_set_options(google_protobuf extern const upb_msglayout google_protobuf_EnumDescriptorProto_msginit; UPB_INLINE google_protobuf_EnumDescriptorProto *google_protobuf_EnumDescriptorProto_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_EnumDescriptorProto_msginit, arena); + return (google_protobuf_EnumDescriptorProto *)upb_msg_new(&google_protobuf_EnumDescriptorProto_msginit, arena); } UPB_INLINE google_protobuf_EnumDescriptorProto *google_protobuf_EnumDescriptorProto_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_EnumDescriptorProto *ret = google_protobuf_EnumDescriptorProto_new(arena); @@ -373,7 +373,7 @@ UPB_INLINE void google_protobuf_EnumDescriptorProto_set_reserved_name(google_pro extern const upb_msglayout google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit; UPB_INLINE google_protobuf_EnumDescriptorProto_EnumReservedRange *google_protobuf_EnumDescriptorProto_EnumReservedRange_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit, arena); + return (google_protobuf_EnumDescriptorProto_EnumReservedRange *)upb_msg_new(&google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit, arena); } UPB_INLINE google_protobuf_EnumDescriptorProto_EnumReservedRange *google_protobuf_EnumDescriptorProto_EnumReservedRange_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_EnumDescriptorProto_EnumReservedRange *ret = google_protobuf_EnumDescriptorProto_EnumReservedRange_new(arena); @@ -394,7 +394,7 @@ UPB_INLINE void google_protobuf_EnumDescriptorProto_EnumReservedRange_set_end(go extern const upb_msglayout google_protobuf_EnumValueDescriptorProto_msginit; UPB_INLINE google_protobuf_EnumValueDescriptorProto *google_protobuf_EnumValueDescriptorProto_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_EnumValueDescriptorProto_msginit, arena); + return (google_protobuf_EnumValueDescriptorProto *)upb_msg_new(&google_protobuf_EnumValueDescriptorProto_msginit, arena); } UPB_INLINE google_protobuf_EnumValueDescriptorProto *google_protobuf_EnumValueDescriptorProto_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_EnumValueDescriptorProto *ret = google_protobuf_EnumValueDescriptorProto_new(arena); @@ -417,7 +417,7 @@ UPB_INLINE void google_protobuf_EnumValueDescriptorProto_set_options(google_prot extern const upb_msglayout google_protobuf_ServiceDescriptorProto_msginit; UPB_INLINE google_protobuf_ServiceDescriptorProto *google_protobuf_ServiceDescriptorProto_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_ServiceDescriptorProto_msginit, arena); + return (google_protobuf_ServiceDescriptorProto *)upb_msg_new(&google_protobuf_ServiceDescriptorProto_msginit, arena); } UPB_INLINE google_protobuf_ServiceDescriptorProto *google_protobuf_ServiceDescriptorProto_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_ServiceDescriptorProto *ret = google_protobuf_ServiceDescriptorProto_new(arena); @@ -440,7 +440,7 @@ UPB_INLINE void google_protobuf_ServiceDescriptorProto_set_options(google_protob extern const upb_msglayout google_protobuf_MethodDescriptorProto_msginit; UPB_INLINE google_protobuf_MethodDescriptorProto *google_protobuf_MethodDescriptorProto_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_MethodDescriptorProto_msginit, arena); + return (google_protobuf_MethodDescriptorProto *)upb_msg_new(&google_protobuf_MethodDescriptorProto_msginit, arena); } UPB_INLINE google_protobuf_MethodDescriptorProto *google_protobuf_MethodDescriptorProto_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_MethodDescriptorProto *ret = google_protobuf_MethodDescriptorProto_new(arena); @@ -469,7 +469,7 @@ UPB_INLINE void google_protobuf_MethodDescriptorProto_set_server_streaming(googl extern const upb_msglayout google_protobuf_FileOptions_msginit; UPB_INLINE google_protobuf_FileOptions *google_protobuf_FileOptions_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_FileOptions_msginit, arena); + return (google_protobuf_FileOptions *)upb_msg_new(&google_protobuf_FileOptions_msginit, arena); } UPB_INLINE google_protobuf_FileOptions *google_protobuf_FileOptions_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_FileOptions *ret = google_protobuf_FileOptions_new(arena); @@ -524,7 +524,7 @@ UPB_INLINE void google_protobuf_FileOptions_set_uninterpreted_option(google_prot extern const upb_msglayout google_protobuf_MessageOptions_msginit; UPB_INLINE google_protobuf_MessageOptions *google_protobuf_MessageOptions_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_MessageOptions_msginit, arena); + return (google_protobuf_MessageOptions *)upb_msg_new(&google_protobuf_MessageOptions_msginit, arena); } UPB_INLINE google_protobuf_MessageOptions *google_protobuf_MessageOptions_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_MessageOptions *ret = google_protobuf_MessageOptions_new(arena); @@ -551,7 +551,7 @@ UPB_INLINE void google_protobuf_MessageOptions_set_uninterpreted_option(google_p extern const upb_msglayout google_protobuf_FieldOptions_msginit; UPB_INLINE google_protobuf_FieldOptions *google_protobuf_FieldOptions_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_FieldOptions_msginit, arena); + return (google_protobuf_FieldOptions *)upb_msg_new(&google_protobuf_FieldOptions_msginit, arena); } UPB_INLINE google_protobuf_FieldOptions *google_protobuf_FieldOptions_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_FieldOptions *ret = google_protobuf_FieldOptions_new(arena); @@ -582,7 +582,7 @@ UPB_INLINE void google_protobuf_FieldOptions_set_uninterpreted_option(google_pro extern const upb_msglayout google_protobuf_OneofOptions_msginit; UPB_INLINE google_protobuf_OneofOptions *google_protobuf_OneofOptions_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_OneofOptions_msginit, arena); + return (google_protobuf_OneofOptions *)upb_msg_new(&google_protobuf_OneofOptions_msginit, arena); } UPB_INLINE google_protobuf_OneofOptions *google_protobuf_OneofOptions_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_OneofOptions *ret = google_protobuf_OneofOptions_new(arena); @@ -601,7 +601,7 @@ UPB_INLINE void google_protobuf_OneofOptions_set_uninterpreted_option(google_pro extern const upb_msglayout google_protobuf_EnumOptions_msginit; UPB_INLINE google_protobuf_EnumOptions *google_protobuf_EnumOptions_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_EnumOptions_msginit, arena); + return (google_protobuf_EnumOptions *)upb_msg_new(&google_protobuf_EnumOptions_msginit, arena); } UPB_INLINE google_protobuf_EnumOptions *google_protobuf_EnumOptions_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_EnumOptions *ret = google_protobuf_EnumOptions_new(arena); @@ -624,7 +624,7 @@ UPB_INLINE void google_protobuf_EnumOptions_set_uninterpreted_option(google_prot extern const upb_msglayout google_protobuf_EnumValueOptions_msginit; UPB_INLINE google_protobuf_EnumValueOptions *google_protobuf_EnumValueOptions_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_EnumValueOptions_msginit, arena); + return (google_protobuf_EnumValueOptions *)upb_msg_new(&google_protobuf_EnumValueOptions_msginit, arena); } UPB_INLINE google_protobuf_EnumValueOptions *google_protobuf_EnumValueOptions_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_EnumValueOptions *ret = google_protobuf_EnumValueOptions_new(arena); @@ -645,7 +645,7 @@ UPB_INLINE void google_protobuf_EnumValueOptions_set_uninterpreted_option(google extern const upb_msglayout google_protobuf_ServiceOptions_msginit; UPB_INLINE google_protobuf_ServiceOptions *google_protobuf_ServiceOptions_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_ServiceOptions_msginit, arena); + return (google_protobuf_ServiceOptions *)upb_msg_new(&google_protobuf_ServiceOptions_msginit, arena); } UPB_INLINE google_protobuf_ServiceOptions *google_protobuf_ServiceOptions_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_ServiceOptions *ret = google_protobuf_ServiceOptions_new(arena); @@ -666,7 +666,7 @@ UPB_INLINE void google_protobuf_ServiceOptions_set_uninterpreted_option(google_p extern const upb_msglayout google_protobuf_MethodOptions_msginit; UPB_INLINE google_protobuf_MethodOptions *google_protobuf_MethodOptions_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_MethodOptions_msginit, arena); + return (google_protobuf_MethodOptions *)upb_msg_new(&google_protobuf_MethodOptions_msginit, arena); } UPB_INLINE google_protobuf_MethodOptions *google_protobuf_MethodOptions_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_MethodOptions *ret = google_protobuf_MethodOptions_new(arena); @@ -689,7 +689,7 @@ UPB_INLINE void google_protobuf_MethodOptions_set_uninterpreted_option(google_pr extern const upb_msglayout google_protobuf_UninterpretedOption_msginit; UPB_INLINE google_protobuf_UninterpretedOption *google_protobuf_UninterpretedOption_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena); + return (google_protobuf_UninterpretedOption *)upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena); } UPB_INLINE google_protobuf_UninterpretedOption *google_protobuf_UninterpretedOption_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_UninterpretedOption *ret = google_protobuf_UninterpretedOption_new(arena); @@ -720,7 +720,7 @@ UPB_INLINE void google_protobuf_UninterpretedOption_set_aggregate_value(google_p extern const upb_msglayout google_protobuf_UninterpretedOption_NamePart_msginit; UPB_INLINE google_protobuf_UninterpretedOption_NamePart *google_protobuf_UninterpretedOption_NamePart_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_UninterpretedOption_NamePart_msginit, arena); + return (google_protobuf_UninterpretedOption_NamePart *)upb_msg_new(&google_protobuf_UninterpretedOption_NamePart_msginit, arena); } UPB_INLINE google_protobuf_UninterpretedOption_NamePart *google_protobuf_UninterpretedOption_NamePart_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_UninterpretedOption_NamePart *ret = google_protobuf_UninterpretedOption_NamePart_new(arena); @@ -741,7 +741,7 @@ UPB_INLINE void google_protobuf_UninterpretedOption_NamePart_set_is_extension(go extern const upb_msglayout google_protobuf_SourceCodeInfo_msginit; UPB_INLINE google_protobuf_SourceCodeInfo *google_protobuf_SourceCodeInfo_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_SourceCodeInfo_msginit, arena); + return (google_protobuf_SourceCodeInfo *)upb_msg_new(&google_protobuf_SourceCodeInfo_msginit, arena); } UPB_INLINE google_protobuf_SourceCodeInfo *google_protobuf_SourceCodeInfo_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_SourceCodeInfo *ret = google_protobuf_SourceCodeInfo_new(arena); @@ -760,7 +760,7 @@ UPB_INLINE void google_protobuf_SourceCodeInfo_set_location(google_protobuf_Sour extern const upb_msglayout google_protobuf_SourceCodeInfo_Location_msginit; UPB_INLINE google_protobuf_SourceCodeInfo_Location *google_protobuf_SourceCodeInfo_Location_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_SourceCodeInfo_Location_msginit, arena); + return (google_protobuf_SourceCodeInfo_Location *)upb_msg_new(&google_protobuf_SourceCodeInfo_Location_msginit, arena); } UPB_INLINE google_protobuf_SourceCodeInfo_Location *google_protobuf_SourceCodeInfo_Location_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_SourceCodeInfo_Location *ret = google_protobuf_SourceCodeInfo_Location_new(arena); @@ -787,7 +787,7 @@ UPB_INLINE void google_protobuf_SourceCodeInfo_Location_set_leading_detached_com extern const upb_msglayout google_protobuf_GeneratedCodeInfo_msginit; UPB_INLINE google_protobuf_GeneratedCodeInfo *google_protobuf_GeneratedCodeInfo_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_GeneratedCodeInfo_msginit, arena); + return (google_protobuf_GeneratedCodeInfo *)upb_msg_new(&google_protobuf_GeneratedCodeInfo_msginit, arena); } UPB_INLINE google_protobuf_GeneratedCodeInfo *google_protobuf_GeneratedCodeInfo_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_GeneratedCodeInfo *ret = google_protobuf_GeneratedCodeInfo_new(arena); @@ -806,7 +806,7 @@ UPB_INLINE void google_protobuf_GeneratedCodeInfo_set_annotation(google_protobuf extern const upb_msglayout google_protobuf_GeneratedCodeInfo_Annotation_msginit; UPB_INLINE google_protobuf_GeneratedCodeInfo_Annotation *google_protobuf_GeneratedCodeInfo_Annotation_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_GeneratedCodeInfo_Annotation_msginit, arena); + return (google_protobuf_GeneratedCodeInfo_Annotation *)upb_msg_new(&google_protobuf_GeneratedCodeInfo_Annotation_msginit, arena); } UPB_INLINE google_protobuf_GeneratedCodeInfo_Annotation *google_protobuf_GeneratedCodeInfo_Annotation_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_GeneratedCodeInfo_Annotation *ret = google_protobuf_GeneratedCodeInfo_Annotation_new(arena); From f6c33307c4a708a07994d7ee01ffa1794575861d Mon Sep 17 00:00:00 2001 From: Vishal Powar Date: Tue, 20 Nov 2018 12:21:50 -0800 Subject: [PATCH 13/16] Fix tests for generated code --- google/protobuf/descriptor.upb.h | 54 ++++++++++++++++---------------- tools/make_c_api.lua | 2 +- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/google/protobuf/descriptor.upb.h b/google/protobuf/descriptor.upb.h index 7afa8c5fa8..6289095153 100644 --- a/google/protobuf/descriptor.upb.h +++ b/google/protobuf/descriptor.upb.h @@ -128,7 +128,7 @@ typedef enum { extern const upb_msglayout google_protobuf_FileDescriptorSet_msginit; UPB_INLINE google_protobuf_FileDescriptorSet *google_protobuf_FileDescriptorSet_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_FileDescriptorSet_msginit, arena); + return (google_protobuf_FileDescriptorSet *)upb_msg_new(&google_protobuf_FileDescriptorSet_msginit, arena); } UPB_INLINE google_protobuf_FileDescriptorSet *google_protobuf_FileDescriptorSet_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_FileDescriptorSet *ret = google_protobuf_FileDescriptorSet_new(arena); @@ -147,7 +147,7 @@ UPB_INLINE void google_protobuf_FileDescriptorSet_set_file(google_protobuf_FileD extern const upb_msglayout google_protobuf_FileDescriptorProto_msginit; UPB_INLINE google_protobuf_FileDescriptorProto *google_protobuf_FileDescriptorProto_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_FileDescriptorProto_msginit, arena); + return (google_protobuf_FileDescriptorProto *)upb_msg_new(&google_protobuf_FileDescriptorProto_msginit, arena); } UPB_INLINE google_protobuf_FileDescriptorProto *google_protobuf_FileDescriptorProto_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_FileDescriptorProto *ret = google_protobuf_FileDescriptorProto_new(arena); @@ -188,7 +188,7 @@ UPB_INLINE void google_protobuf_FileDescriptorProto_set_syntax(google_protobuf_F extern const upb_msglayout google_protobuf_DescriptorProto_msginit; UPB_INLINE google_protobuf_DescriptorProto *google_protobuf_DescriptorProto_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_DescriptorProto_msginit, arena); + return (google_protobuf_DescriptorProto *)upb_msg_new(&google_protobuf_DescriptorProto_msginit, arena); } UPB_INLINE google_protobuf_DescriptorProto *google_protobuf_DescriptorProto_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_DescriptorProto *ret = google_protobuf_DescriptorProto_new(arena); @@ -225,7 +225,7 @@ UPB_INLINE void google_protobuf_DescriptorProto_set_reserved_name(google_protobu extern const upb_msglayout google_protobuf_DescriptorProto_ExtensionRange_msginit; UPB_INLINE google_protobuf_DescriptorProto_ExtensionRange *google_protobuf_DescriptorProto_ExtensionRange_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_DescriptorProto_ExtensionRange_msginit, arena); + return (google_protobuf_DescriptorProto_ExtensionRange *)upb_msg_new(&google_protobuf_DescriptorProto_ExtensionRange_msginit, arena); } UPB_INLINE google_protobuf_DescriptorProto_ExtensionRange *google_protobuf_DescriptorProto_ExtensionRange_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_DescriptorProto_ExtensionRange *ret = google_protobuf_DescriptorProto_ExtensionRange_new(arena); @@ -248,7 +248,7 @@ UPB_INLINE void google_protobuf_DescriptorProto_ExtensionRange_set_options(googl extern const upb_msglayout google_protobuf_DescriptorProto_ReservedRange_msginit; UPB_INLINE google_protobuf_DescriptorProto_ReservedRange *google_protobuf_DescriptorProto_ReservedRange_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_DescriptorProto_ReservedRange_msginit, arena); + return (google_protobuf_DescriptorProto_ReservedRange *)upb_msg_new(&google_protobuf_DescriptorProto_ReservedRange_msginit, arena); } UPB_INLINE google_protobuf_DescriptorProto_ReservedRange *google_protobuf_DescriptorProto_ReservedRange_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_DescriptorProto_ReservedRange *ret = google_protobuf_DescriptorProto_ReservedRange_new(arena); @@ -269,7 +269,7 @@ UPB_INLINE void google_protobuf_DescriptorProto_ReservedRange_set_end(google_pro extern const upb_msglayout google_protobuf_ExtensionRangeOptions_msginit; UPB_INLINE google_protobuf_ExtensionRangeOptions *google_protobuf_ExtensionRangeOptions_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_ExtensionRangeOptions_msginit, arena); + return (google_protobuf_ExtensionRangeOptions *)upb_msg_new(&google_protobuf_ExtensionRangeOptions_msginit, arena); } UPB_INLINE google_protobuf_ExtensionRangeOptions *google_protobuf_ExtensionRangeOptions_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_ExtensionRangeOptions *ret = google_protobuf_ExtensionRangeOptions_new(arena); @@ -288,7 +288,7 @@ UPB_INLINE void google_protobuf_ExtensionRangeOptions_set_uninterpreted_option(g extern const upb_msglayout google_protobuf_FieldDescriptorProto_msginit; UPB_INLINE google_protobuf_FieldDescriptorProto *google_protobuf_FieldDescriptorProto_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_FieldDescriptorProto_msginit, arena); + return (google_protobuf_FieldDescriptorProto *)upb_msg_new(&google_protobuf_FieldDescriptorProto_msginit, arena); } UPB_INLINE google_protobuf_FieldDescriptorProto *google_protobuf_FieldDescriptorProto_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_FieldDescriptorProto *ret = google_protobuf_FieldDescriptorProto_new(arena); @@ -325,7 +325,7 @@ UPB_INLINE void google_protobuf_FieldDescriptorProto_set_json_name(google_protob extern const upb_msglayout google_protobuf_OneofDescriptorProto_msginit; UPB_INLINE google_protobuf_OneofDescriptorProto *google_protobuf_OneofDescriptorProto_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_OneofDescriptorProto_msginit, arena); + return (google_protobuf_OneofDescriptorProto *)upb_msg_new(&google_protobuf_OneofDescriptorProto_msginit, arena); } UPB_INLINE google_protobuf_OneofDescriptorProto *google_protobuf_OneofDescriptorProto_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_OneofDescriptorProto *ret = google_protobuf_OneofDescriptorProto_new(arena); @@ -346,7 +346,7 @@ UPB_INLINE void google_protobuf_OneofDescriptorProto_set_options(google_protobuf extern const upb_msglayout google_protobuf_EnumDescriptorProto_msginit; UPB_INLINE google_protobuf_EnumDescriptorProto *google_protobuf_EnumDescriptorProto_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_EnumDescriptorProto_msginit, arena); + return (google_protobuf_EnumDescriptorProto *)upb_msg_new(&google_protobuf_EnumDescriptorProto_msginit, arena); } UPB_INLINE google_protobuf_EnumDescriptorProto *google_protobuf_EnumDescriptorProto_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_EnumDescriptorProto *ret = google_protobuf_EnumDescriptorProto_new(arena); @@ -373,7 +373,7 @@ UPB_INLINE void google_protobuf_EnumDescriptorProto_set_reserved_name(google_pro extern const upb_msglayout google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit; UPB_INLINE google_protobuf_EnumDescriptorProto_EnumReservedRange *google_protobuf_EnumDescriptorProto_EnumReservedRange_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit, arena); + return (google_protobuf_EnumDescriptorProto_EnumReservedRange *)upb_msg_new(&google_protobuf_EnumDescriptorProto_EnumReservedRange_msginit, arena); } UPB_INLINE google_protobuf_EnumDescriptorProto_EnumReservedRange *google_protobuf_EnumDescriptorProto_EnumReservedRange_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_EnumDescriptorProto_EnumReservedRange *ret = google_protobuf_EnumDescriptorProto_EnumReservedRange_new(arena); @@ -394,7 +394,7 @@ UPB_INLINE void google_protobuf_EnumDescriptorProto_EnumReservedRange_set_end(go extern const upb_msglayout google_protobuf_EnumValueDescriptorProto_msginit; UPB_INLINE google_protobuf_EnumValueDescriptorProto *google_protobuf_EnumValueDescriptorProto_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_EnumValueDescriptorProto_msginit, arena); + return (google_protobuf_EnumValueDescriptorProto *)upb_msg_new(&google_protobuf_EnumValueDescriptorProto_msginit, arena); } UPB_INLINE google_protobuf_EnumValueDescriptorProto *google_protobuf_EnumValueDescriptorProto_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_EnumValueDescriptorProto *ret = google_protobuf_EnumValueDescriptorProto_new(arena); @@ -417,7 +417,7 @@ UPB_INLINE void google_protobuf_EnumValueDescriptorProto_set_options(google_prot extern const upb_msglayout google_protobuf_ServiceDescriptorProto_msginit; UPB_INLINE google_protobuf_ServiceDescriptorProto *google_protobuf_ServiceDescriptorProto_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_ServiceDescriptorProto_msginit, arena); + return (google_protobuf_ServiceDescriptorProto *)upb_msg_new(&google_protobuf_ServiceDescriptorProto_msginit, arena); } UPB_INLINE google_protobuf_ServiceDescriptorProto *google_protobuf_ServiceDescriptorProto_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_ServiceDescriptorProto *ret = google_protobuf_ServiceDescriptorProto_new(arena); @@ -440,7 +440,7 @@ UPB_INLINE void google_protobuf_ServiceDescriptorProto_set_options(google_protob extern const upb_msglayout google_protobuf_MethodDescriptorProto_msginit; UPB_INLINE google_protobuf_MethodDescriptorProto *google_protobuf_MethodDescriptorProto_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_MethodDescriptorProto_msginit, arena); + return (google_protobuf_MethodDescriptorProto *)upb_msg_new(&google_protobuf_MethodDescriptorProto_msginit, arena); } UPB_INLINE google_protobuf_MethodDescriptorProto *google_protobuf_MethodDescriptorProto_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_MethodDescriptorProto *ret = google_protobuf_MethodDescriptorProto_new(arena); @@ -469,7 +469,7 @@ UPB_INLINE void google_protobuf_MethodDescriptorProto_set_server_streaming(googl extern const upb_msglayout google_protobuf_FileOptions_msginit; UPB_INLINE google_protobuf_FileOptions *google_protobuf_FileOptions_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_FileOptions_msginit, arena); + return (google_protobuf_FileOptions *)upb_msg_new(&google_protobuf_FileOptions_msginit, arena); } UPB_INLINE google_protobuf_FileOptions *google_protobuf_FileOptions_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_FileOptions *ret = google_protobuf_FileOptions_new(arena); @@ -524,7 +524,7 @@ UPB_INLINE void google_protobuf_FileOptions_set_uninterpreted_option(google_prot extern const upb_msglayout google_protobuf_MessageOptions_msginit; UPB_INLINE google_protobuf_MessageOptions *google_protobuf_MessageOptions_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_MessageOptions_msginit, arena); + return (google_protobuf_MessageOptions *)upb_msg_new(&google_protobuf_MessageOptions_msginit, arena); } UPB_INLINE google_protobuf_MessageOptions *google_protobuf_MessageOptions_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_MessageOptions *ret = google_protobuf_MessageOptions_new(arena); @@ -551,7 +551,7 @@ UPB_INLINE void google_protobuf_MessageOptions_set_uninterpreted_option(google_p extern const upb_msglayout google_protobuf_FieldOptions_msginit; UPB_INLINE google_protobuf_FieldOptions *google_protobuf_FieldOptions_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_FieldOptions_msginit, arena); + return (google_protobuf_FieldOptions *)upb_msg_new(&google_protobuf_FieldOptions_msginit, arena); } UPB_INLINE google_protobuf_FieldOptions *google_protobuf_FieldOptions_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_FieldOptions *ret = google_protobuf_FieldOptions_new(arena); @@ -582,7 +582,7 @@ UPB_INLINE void google_protobuf_FieldOptions_set_uninterpreted_option(google_pro extern const upb_msglayout google_protobuf_OneofOptions_msginit; UPB_INLINE google_protobuf_OneofOptions *google_protobuf_OneofOptions_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_OneofOptions_msginit, arena); + return (google_protobuf_OneofOptions *)upb_msg_new(&google_protobuf_OneofOptions_msginit, arena); } UPB_INLINE google_protobuf_OneofOptions *google_protobuf_OneofOptions_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_OneofOptions *ret = google_protobuf_OneofOptions_new(arena); @@ -601,7 +601,7 @@ UPB_INLINE void google_protobuf_OneofOptions_set_uninterpreted_option(google_pro extern const upb_msglayout google_protobuf_EnumOptions_msginit; UPB_INLINE google_protobuf_EnumOptions *google_protobuf_EnumOptions_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_EnumOptions_msginit, arena); + return (google_protobuf_EnumOptions *)upb_msg_new(&google_protobuf_EnumOptions_msginit, arena); } UPB_INLINE google_protobuf_EnumOptions *google_protobuf_EnumOptions_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_EnumOptions *ret = google_protobuf_EnumOptions_new(arena); @@ -624,7 +624,7 @@ UPB_INLINE void google_protobuf_EnumOptions_set_uninterpreted_option(google_prot extern const upb_msglayout google_protobuf_EnumValueOptions_msginit; UPB_INLINE google_protobuf_EnumValueOptions *google_protobuf_EnumValueOptions_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_EnumValueOptions_msginit, arena); + return (google_protobuf_EnumValueOptions *)upb_msg_new(&google_protobuf_EnumValueOptions_msginit, arena); } UPB_INLINE google_protobuf_EnumValueOptions *google_protobuf_EnumValueOptions_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_EnumValueOptions *ret = google_protobuf_EnumValueOptions_new(arena); @@ -645,7 +645,7 @@ UPB_INLINE void google_protobuf_EnumValueOptions_set_uninterpreted_option(google extern const upb_msglayout google_protobuf_ServiceOptions_msginit; UPB_INLINE google_protobuf_ServiceOptions *google_protobuf_ServiceOptions_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_ServiceOptions_msginit, arena); + return (google_protobuf_ServiceOptions *)upb_msg_new(&google_protobuf_ServiceOptions_msginit, arena); } UPB_INLINE google_protobuf_ServiceOptions *google_protobuf_ServiceOptions_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_ServiceOptions *ret = google_protobuf_ServiceOptions_new(arena); @@ -666,7 +666,7 @@ UPB_INLINE void google_protobuf_ServiceOptions_set_uninterpreted_option(google_p extern const upb_msglayout google_protobuf_MethodOptions_msginit; UPB_INLINE google_protobuf_MethodOptions *google_protobuf_MethodOptions_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_MethodOptions_msginit, arena); + return (google_protobuf_MethodOptions *)upb_msg_new(&google_protobuf_MethodOptions_msginit, arena); } UPB_INLINE google_protobuf_MethodOptions *google_protobuf_MethodOptions_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_MethodOptions *ret = google_protobuf_MethodOptions_new(arena); @@ -689,7 +689,7 @@ UPB_INLINE void google_protobuf_MethodOptions_set_uninterpreted_option(google_pr extern const upb_msglayout google_protobuf_UninterpretedOption_msginit; UPB_INLINE google_protobuf_UninterpretedOption *google_protobuf_UninterpretedOption_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena); + return (google_protobuf_UninterpretedOption *)upb_msg_new(&google_protobuf_UninterpretedOption_msginit, arena); } UPB_INLINE google_protobuf_UninterpretedOption *google_protobuf_UninterpretedOption_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_UninterpretedOption *ret = google_protobuf_UninterpretedOption_new(arena); @@ -720,7 +720,7 @@ UPB_INLINE void google_protobuf_UninterpretedOption_set_aggregate_value(google_p extern const upb_msglayout google_protobuf_UninterpretedOption_NamePart_msginit; UPB_INLINE google_protobuf_UninterpretedOption_NamePart *google_protobuf_UninterpretedOption_NamePart_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_UninterpretedOption_NamePart_msginit, arena); + return (google_protobuf_UninterpretedOption_NamePart *)upb_msg_new(&google_protobuf_UninterpretedOption_NamePart_msginit, arena); } UPB_INLINE google_protobuf_UninterpretedOption_NamePart *google_protobuf_UninterpretedOption_NamePart_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_UninterpretedOption_NamePart *ret = google_protobuf_UninterpretedOption_NamePart_new(arena); @@ -741,7 +741,7 @@ UPB_INLINE void google_protobuf_UninterpretedOption_NamePart_set_is_extension(go extern const upb_msglayout google_protobuf_SourceCodeInfo_msginit; UPB_INLINE google_protobuf_SourceCodeInfo *google_protobuf_SourceCodeInfo_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_SourceCodeInfo_msginit, arena); + return (google_protobuf_SourceCodeInfo *)upb_msg_new(&google_protobuf_SourceCodeInfo_msginit, arena); } UPB_INLINE google_protobuf_SourceCodeInfo *google_protobuf_SourceCodeInfo_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_SourceCodeInfo *ret = google_protobuf_SourceCodeInfo_new(arena); @@ -760,7 +760,7 @@ UPB_INLINE void google_protobuf_SourceCodeInfo_set_location(google_protobuf_Sour extern const upb_msglayout google_protobuf_SourceCodeInfo_Location_msginit; UPB_INLINE google_protobuf_SourceCodeInfo_Location *google_protobuf_SourceCodeInfo_Location_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_SourceCodeInfo_Location_msginit, arena); + return (google_protobuf_SourceCodeInfo_Location *)upb_msg_new(&google_protobuf_SourceCodeInfo_Location_msginit, arena); } UPB_INLINE google_protobuf_SourceCodeInfo_Location *google_protobuf_SourceCodeInfo_Location_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_SourceCodeInfo_Location *ret = google_protobuf_SourceCodeInfo_Location_new(arena); @@ -787,7 +787,7 @@ UPB_INLINE void google_protobuf_SourceCodeInfo_Location_set_leading_detached_com extern const upb_msglayout google_protobuf_GeneratedCodeInfo_msginit; UPB_INLINE google_protobuf_GeneratedCodeInfo *google_protobuf_GeneratedCodeInfo_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_GeneratedCodeInfo_msginit, arena); + return (google_protobuf_GeneratedCodeInfo *)upb_msg_new(&google_protobuf_GeneratedCodeInfo_msginit, arena); } UPB_INLINE google_protobuf_GeneratedCodeInfo *google_protobuf_GeneratedCodeInfo_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_GeneratedCodeInfo *ret = google_protobuf_GeneratedCodeInfo_new(arena); @@ -806,7 +806,7 @@ UPB_INLINE void google_protobuf_GeneratedCodeInfo_set_annotation(google_protobuf extern const upb_msglayout google_protobuf_GeneratedCodeInfo_Annotation_msginit; UPB_INLINE google_protobuf_GeneratedCodeInfo_Annotation *google_protobuf_GeneratedCodeInfo_Annotation_new(upb_arena *arena) { - return upb_msg_new(&google_protobuf_GeneratedCodeInfo_Annotation_msginit, arena); + return (google_protobuf_GeneratedCodeInfo_Annotation *)upb_msg_new(&google_protobuf_GeneratedCodeInfo_Annotation_msginit, arena); } UPB_INLINE google_protobuf_GeneratedCodeInfo_Annotation *google_protobuf_GeneratedCodeInfo_Annotation_parsenew(upb_stringview buf, upb_arena *arena) { google_protobuf_GeneratedCodeInfo_Annotation *ret = google_protobuf_GeneratedCodeInfo_Annotation_new(arena); diff --git a/tools/make_c_api.lua b/tools/make_c_api.lua index bf79b2677d..d9c11b206e 100644 --- a/tools/make_c_api.lua +++ b/tools/make_c_api.lua @@ -380,7 +380,7 @@ local function write_h_file(filedef, append) local msgname = to_cident(msg:full_name()) append('extern const upb_msglayout %s_msginit;\n', msgname) append('UPB_INLINE %s *%s_new(upb_arena *arena) {\n', msgname, msgname) - append(' return upb_msg_new(&%s_msginit, arena);\n', msgname) + append(' return (%s *)upb_msg_new(&%s_msginit, arena);\n', msgname, msgname) append('}\n') append('UPB_INLINE %s *%s_parsenew(upb_stringview buf, upb_arena *arena) {\n', msgname, msgname) From 7031f776663596a68736c7db5c68c8eeca6d3f07 Mon Sep 17 00:00:00 2001 From: Vishal Powar Date: Wed, 28 Nov 2018 15:32:16 -0800 Subject: [PATCH 14/16] Add visibility for use in other projects. --- BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/BUILD b/BUILD index 6503ed0b67..fb3465787e 100644 --- a/BUILD +++ b/BUILD @@ -50,6 +50,7 @@ cc_library( "-pedantic", "-Wno-long-long", ], + visibility = ["//visibility:public"], ) cc_library( From 9e44a757edb7ba7155469d3305da32b3dc11c756 Mon Sep 17 00:00:00 2001 From: Paul Yang Date: Sat, 1 Dec 2018 01:24:58 -0800 Subject: [PATCH 15/16] Fix Bazel Kokoro build (#136) --- BUILD | 1 + build_defs.bzl | 1 + 2 files changed, 2 insertions(+) diff --git a/BUILD b/BUILD index fb3465787e..97380a4257 100644 --- a/BUILD +++ b/BUILD @@ -266,6 +266,7 @@ cc_binary( ":conformance_proto_upb", ":upb", ], + copts = ["-Ibazel-out/k8-fastbuild/bin"], ) make_shell_script( diff --git a/build_defs.bzl b/build_defs.bzl index c1dd4223d5..5bbbc4fd28 100644 --- a/build_defs.bzl +++ b/build_defs.bzl @@ -270,4 +270,5 @@ def upb_proto_library(name, deps, upbc): name = name, srcs = [":" + srcs_rule], deps = [":upb"], + copts = ["-Ibazel-out/k8-fastbuild/bin"], ) From 35fa3df8ecc3f451af0512e70a03f89ee407c85c Mon Sep 17 00:00:00 2001 From: Joshua Haberman Date: Sat, 1 Dec 2018 02:33:16 -0800 Subject: [PATCH 16/16] Fixes for macOS build. Also run tests in CMake test. --- BUILD | 2 +- CMakeLists.txt | 6 ++---- build_defs.bzl | 2 +- tools/make_cmakelists.py | 30 ++++++++++++++++++------------ 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/BUILD b/BUILD index 97380a4257..a0eadb3918 100644 --- a/BUILD +++ b/BUILD @@ -407,7 +407,7 @@ cc_binary( make_shell_script( name = "gen_run_cmake_build", out = "run_cmake_build.sh", - contents = "mkdir build && cd build && cmake .. && make -j8" + contents = "mkdir build && cd build && cmake .. && make -j8 && make test" ) sh_test( diff --git a/CMakeLists.txt b/CMakeLists.txt index 5c6d5fa032..412cdcd09d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -117,11 +117,9 @@ add_library(upb_json upb/json/printer.h) target_link_libraries(upb_json upb) -add_library(upb_cc_bindings - upb/bindings/stdc++/string.h) -target_link_libraries(upb_cc_bindings +add_library(upb_cc_bindings INTERFACE) +target_link_libraries(upb_cc_bindings INTERFACE upb) -set_target_properties(upb_cc_bindings PROPERTIES LINKER_LANGUAGE CXX) add_library(upb_test tests/testmain.cc tests/test_util.h diff --git a/build_defs.bzl b/build_defs.bzl index 5bbbc4fd28..362691a878 100644 --- a/build_defs.bzl +++ b/build_defs.bzl @@ -147,7 +147,7 @@ def generated_file_staleness_test(name, outs, generated_pattern): srcs = [script_src], testonly = 1, cmd = "cat $(location " + script_src + ") > $@; " + - "sed -i 's|INSERT_FILE_LIST_HERE|" + "\\n ".join(file_list) + "|' $@", + "sed -i '' -e 's|INSERT_FILE_LIST_HERE|" + "\\\n ".join(file_list) + "|' $@", ) native.py_test( diff --git a/tools/make_cmakelists.py b/tools/make_cmakelists.py index d7996d8748..b8f46b9029 100755 --- a/tools/make_cmakelists.py +++ b/tools/make_cmakelists.py @@ -22,11 +22,12 @@ class BuildFileFunctions(object): def __init__(self, converter): self.converter = converter - def _add_deps(self, kwargs): + def _add_deps(self, kwargs, keyword=""): if "deps" not in kwargs: return - self.converter.toplevel += "target_link_libraries(%s\n %s)\n" % ( + self.converter.toplevel += "target_link_libraries(%s%s\n %s)\n" % ( kwargs["name"], + keyword, "\n ".join(StripColons(kwargs["deps"])) ) @@ -37,17 +38,22 @@ class BuildFileFunctions(object): if kwargs["name"] == "amalgamation" or kwargs["name"] == "upbc_generator": return files = kwargs.get("srcs", []) + kwargs.get("hdrs", []) - self.converter.toplevel += "add_library(%s\n %s)\n" % ( - kwargs["name"], - "\n ".join(files) - ) - self._add_deps(kwargs) - # CMake wants to know if each library is C or C++. - # If there are only .h files, it can't infer. Assume C++. - if not filter(IsSourceFile, files): - line = "set_target_properties(%s PROPERTIES LINKER_LANGUAGE CXX)\n" - self.converter.toplevel += line % (kwargs["name"]) + if filter(IsSourceFile, files): + # Has sources, make this a normal library. + self.converter.toplevel += "add_library(%s\n %s)\n" % ( + kwargs["name"], + "\n ".join(files) + ) + self._add_deps(kwargs) + else: + # Header-only library, have to do a couple things differently. + # For some info, see: + # http://mariobadr.com/creating-a-header-only-library-with-cmake.html + self.converter.toplevel += "add_library(%s INTERFACE)\n" % ( + kwargs["name"] + ) + self._add_deps(kwargs, " INTERFACE") def cc_binary(self, **kwargs): pass