diff --git a/BUILD b/BUILD index c336aee2a0..96a4b677d4 100644 --- a/BUILD +++ b/BUILD @@ -1,9 +1,7 @@ load( "//bazel:build_defs.bzl", - "generated_file_staleness_test", - "licenses", # copybara:strip_for_google3 - "make_shell_script", - "upb_amalgamation", + "UPB_DEFAULT_COPTS", + "upb_amalgamation", # copybara:strip_for_google3 ) load( "//bazel:upb_proto_library.bzl", @@ -16,10 +14,7 @@ load( "@rules_proto//proto:defs.bzl", "proto_library", ) -load( - "//:upb/bindings/lua/lua_proto_library.bzl", - "lua_proto_library", -) + # copybara:strip_end licenses(["notice"]) # BSD (Google-authored w/ possible external contributions) @@ -29,23 +24,6 @@ exports_files([ "build_defs", ]) -CPPOPTS = [ - # copybara:strip_for_google3_begin - "-Wextra", - # "-Wshorten-64-to-32", # not in GCC (and my Kokoro images doesn't have Clang) - "-Werror", - "-Wno-long-long", - # copybara:strip_end -] - -COPTS = CPPOPTS + [ - # copybara:strip_for_google3_begin - "-pedantic", - "-Werror=pedantic", - "-Wstrict-prototypes", - # copybara:strip_end -] - config_setting( name = "darwin", values = {"cpu": "darwin"}, @@ -57,11 +35,6 @@ config_setting( constraint_values = ["@bazel_tools//platforms:windows"], ) -config_setting( - name = "fuzz", - values = {"define": "fuzz=true"}, -) - # Public C/C++ libraries ####################################################### cc_library( @@ -73,6 +46,7 @@ cc_library( "upb/port_def.inc", "upb/port_undef.inc", ], + visibility = ["//tests:__pkg__"], ) cc_library( @@ -93,10 +67,7 @@ cc_library( "upb/upb.h", "upb/upb.hpp", ], - copts = select({ - ":windows": [], - "//conditions:default": COPTS, - }), + copts = UPB_DEFAULT_COPTS, visibility = ["//visibility:public"], deps = [":port"], ) @@ -114,10 +85,7 @@ cc_library( "upb/port_def.inc", "upb/port_undef.inc", ], - copts = select({ - ":windows": [], - "//conditions:default": COPTS, - }), + copts = UPB_DEFAULT_COPTS, visibility = ["//visibility:public"], deps = [ ":table", @@ -149,10 +117,7 @@ cc_library( "upb/def.hpp", "upb/reflection.h", ], - copts = select({ - ":windows": [], - "//conditions:default": COPTS, - }), + copts = UPB_DEFAULT_COPTS, visibility = ["//visibility:public"], deps = [ ":descriptor_upb_proto", @@ -170,10 +135,7 @@ cc_library( hdrs = [ "upb/text_encode.h", ], - copts = select({ - ":windows": [], - "//conditions:default": COPTS, - }), + copts = UPB_DEFAULT_COPTS, visibility = ["//visibility:public"], deps = [ ":port", @@ -191,10 +153,8 @@ cc_library( "upb/json_decode.h", "upb/json_encode.h", ], - copts = select({ - ":windows": [], - "//conditions:default": COPTS, - }), + copts = UPB_DEFAULT_COPTS, + visibility = ["//tests:__pkg__"], deps = [ ":port", ":reflection", @@ -207,6 +167,7 @@ cc_library( cc_library( name = "table", hdrs = ["upb/table.int.h"], + visibility = ["//tests:__pkg__"], deps = [ ":port", ":upb", @@ -226,10 +187,8 @@ cc_library( "upb/handlers.h", "upb/sink.h", ], - copts = select({ - ":windows": [], - "//conditions:default": COPTS, - }), + copts = UPB_DEFAULT_COPTS, + visibility = ["//tests:__pkg__"], deps = [ ":port", ":reflection", @@ -254,10 +213,8 @@ cc_library( "upb/pb/encoder.h", "upb/pb/textprinter.h", ], - copts = select({ - ":windows": [], - "//conditions:default": COPTS, - }), + copts = UPB_DEFAULT_COPTS, + visibility = ["//tests:__pkg__"], deps = [ ":descriptor_upb_proto", ":handlers", @@ -279,430 +236,22 @@ cc_library( "upb/json/parser.h", "upb/json/printer.h", ], - copts = select({ - ":windows": [], - "//conditions:default": COPTS, - }), - deps = [ - ":upb", - ":upb_pb", - ], -) -# copybara:strip_end - -cc_library( - name = "upb_cc_bindings", - hdrs = [ - "upb/bindings/stdc++/string.h", - ], - deps = [ - ":descriptor_upb_proto", - ":handlers", - ":port", - ":upb", - ], -) - -# upb compiler ################################################################# - -cc_library( - name = "upbc_generator", - srcs = [ - "upbc/generator.cc", - "upbc/message_layout.cc", - "upbc/message_layout.h", - ], - hdrs = ["upbc/generator.h"], - copts = select({ - ":windows": [], - "//conditions:default": CPPOPTS, - }), - deps = [ - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/strings", - "@com_google_protobuf//:protobuf", - "@com_google_protobuf//:protoc_lib", - ], -) - -cc_binary( - name = "protoc-gen-upb", - srcs = ["upbc/main.cc"], - copts = select({ - ":windows": [], - "//conditions:default": CPPOPTS, - }), - visibility = ["//visibility:public"], - deps = [ - ":upbc_generator", - "@com_google_protobuf//:protoc_lib", - ], -) - -# We strip the tests and remaining rules from google3 until the upb_proto_library() -# and upb_proto_reflection_library() rules are fixed. - -# C/C++ tests ################################################################## - -proto_library( - name = "benchmark_descriptor_proto", - srcs = ["tests/descriptor.proto"], -) - -upb_proto_library( - name = "benchmark_descriptor_upb_proto", - deps = [":benchmark_descriptor_proto"], -) - -upb_proto_reflection_library( - name = "benchmark_descriptor_upb_proto_reflection", - deps = [":benchmark_descriptor_proto"], -) - -cc_proto_library( - name = "benchmark_descriptor_cc_proto", - deps = [":benchmark_descriptor_proto"], -) - -proto_library( - name = "benchmark_descriptor_sv_proto", - srcs = ["tests/descriptor_sv.proto"], -) - -cc_proto_library( - name = "benchmark_descriptor_sv_cc_proto", - deps = [":benchmark_descriptor_sv_proto"], -) - -cc_binary( - name = "benchmark", - testonly = 1, - srcs = ["tests/benchmark.cc"], - deps = [ - ":benchmark_descriptor_cc_proto", - ":benchmark_descriptor_sv_cc_proto", - ":benchmark_descriptor_upb_proto", - ":benchmark_descriptor_upb_proto_reflection", - ":descriptor_upb_proto", - ":reflection", - "@com_github_google_benchmark//:benchmark_main", - "@com_google_protobuf//:protobuf", - ], -) - -cc_library( - name = "upb_test", - testonly = 1, - srcs = [ - "tests/testmain.cc", - ], - hdrs = [ - "tests/test_util.h", - "tests/upb_test.h", - ], - copts = select({ - ":windows": [], - "//conditions:default": CPPOPTS, - }), - deps = [ - ":handlers", - ":port", - ":upb", - ], -) - -cc_test( - name = "test_varint", - srcs = [ - "tests/pb/test_varint.c", - "upb/pb/varint.int.h", - ], - copts = select({ - ":windows": [], - "//conditions:default": COPTS, - }), - deps = [ - ":port", - ":upb", - ":upb_pb", - ":upb_test", - ], -) - -proto_library( - name = "test_proto", - testonly = 1, - srcs = ["tests/test.proto"], -) - -upb_proto_library( - name = "test_upb_proto", - testonly = 1, - deps = [":test_proto"], -) - -cc_test( - name = "test_generated_code", - srcs = ["tests/test_generated_code.c"], - deps = [ - ":test_messages_proto3_proto_upb", - ":empty_upbdefs_proto", - ":test_upb_proto", - ":upb_test", - ], -) - -proto_library( - name = "empty_proto", - srcs = ["tests/empty.proto"], -) - -upb_proto_reflection_library( - name = "empty_upbdefs_proto", - testonly = 1, - deps = [":empty_proto"], -) - -upb_proto_library( - name = "test_messages_proto3_proto_upb", - testonly = 1, - deps = ["@com_google_protobuf//:test_messages_proto3_proto"], -) - -proto_library( - name = "test_decoder_proto", - srcs = [ - "tests/pb/test_decoder.proto", - ], -) - -upb_proto_reflection_library( - name = "test_decoder_upb_proto", - deps = [":test_decoder_proto"], -) - -cc_test( - name = "test_decoder", - srcs = [ - "tests/pb/test_decoder.cc", - "upb/pb/varint.int.h", - ], - copts = select({ - ":windows": [], - "//conditions:default": CPPOPTS, - }), - deps = [ - ":handlers", - ":port", - ":test_decoder_upb_proto", - ":upb", - ":upb_pb", - ":upb_test", - ], -) - -proto_library( - name = "test_cpp_proto", - srcs = [ - "tests/test_cpp.proto", - ], -) - -upb_proto_reflection_library( - name = "test_cpp_upb_proto", - deps = ["test_cpp_proto"], -) - -cc_test( - name = "test_cpp", - srcs = ["tests/test_cpp.cc"], - copts = select({ - ":windows": [], - "//conditions:default": CPPOPTS, - }), - deps = [ - ":handlers", - ":port", - ":reflection", - ":test_cpp_upb_proto", - ":upb", - ":upb_pb", - ":upb_test", - ], -) - -cc_test( - name = "test_table", - srcs = ["tests/test_table.cc"], - copts = select({ - ":windows": [], - "//conditions:default": CPPOPTS, - }), - deps = [ - ":port", - ":table", - ":upb", - ":upb_test", - ], -) - -# OSS-Fuzz test -cc_binary( - name = "file_descriptor_parsenew_fuzzer", - testonly = 1, - srcs = ["tests/file_descriptor_parsenew_fuzzer.cc"], - copts = select({ - ":windows": [], - "//conditions:default": CPPOPTS, - }) + select({ - "//conditions:default": [], - ":fuzz": ["-fsanitize=fuzzer,address"], - }), - defines = select({ - "//conditions:default": [], - ":fuzz": ["HAVE_FUZZER"], - }), + copts = UPB_DEFAULT_COPTS, + visibility = ["//tests:__pkg__"], deps = [ - ":descriptor_upb_proto", ":upb", - ], -) - -# copybara:strip_for_google3_begin -cc_test( - name = "test_encoder", - srcs = ["tests/pb/test_encoder.cc"], - copts = select({ - ":windows": [], - "//conditions:default": CPPOPTS, - }), - deps = [ - ":descriptor_upb_proto", - ":descriptor_upb_proto_reflection", - ":upb", - ":upb_cc_bindings", ":upb_pb", - ":upb_test", - ], -) - -proto_library( - name = "test_json_enum_from_separate", - srcs = ["tests/json/enum_from_separate_file.proto"], - deps = [":test_json_proto"], -) - -proto_library( - name = "test_json_proto", - srcs = ["tests/json/test.proto"], -) - -upb_proto_reflection_library( - name = "test_json_upb_proto_reflection", - deps = ["test_json_proto"], -) - -upb_proto_library( - name = "test_json_enum_from_separate_upb_proto", - deps = [":test_json_enum_from_separate"], -) - -upb_proto_library( - name = "test_json_upb_proto", - deps = [":test_json_proto"], -) - -cc_test( - name = "test_json", - srcs = [ - "tests/json/test_json.cc", - ], - copts = select({ - ":windows": [], - "//conditions:default": CPPOPTS, - }), - deps = [ - ":test_json_upb_proto", - ":test_json_upb_proto_reflection", - ":upb_json", - ":upb_test", - ], -) -# copybara:strip_end - -upb_proto_library( - name = "conformance_proto_upb", - testonly = 1, - deps = ["@com_google_protobuf//:conformance_proto"], -) - -upb_proto_reflection_library( - name = "conformance_proto_upbdefs", - testonly = 1, - deps = ["@com_google_protobuf//:conformance_proto"], -) - -upb_proto_reflection_library( - name = "test_messages_proto2_upbdefs", - testonly = 1, - deps = ["@com_google_protobuf//:test_messages_proto2_proto"], -) - -upb_proto_reflection_library( - name = "test_messages_proto3_upbdefs", - testonly = 1, - deps = ["@com_google_protobuf//:test_messages_proto3_proto"], -) - -cc_binary( - name = "conformance_upb", - testonly = 1, - srcs = [ - "tests/conformance_upb.c", - ], - data = [ - "tests/conformance_upb_failures.txt", - ], - copts = select({ - ":windows": [], - "//conditions:default": COPTS, - }) + ["-Ibazel-out/k8-fastbuild/bin"], - deps = [ - ":port", - ":conformance_proto_upb", - ":conformance_proto_upbdefs", - ":json", - ":reflection", - ":test_messages_proto2_upbdefs", - ":test_messages_proto3_upbdefs", - ":textformat", - ":upb", ], ) -make_shell_script( - name = "gen_test_conformance_upb", - out = "test_conformance_upb.sh", - contents = "external/com_google_protobuf/conformance_test_runner " + - " --enforce_recommended " + - " --failure_list ./tests/conformance_upb_failures.txt" + - " ./conformance_upb", -) - -sh_test( - name = "test_conformance_upb", - srcs = ["test_conformance_upb.sh"], - data = [ - "tests/conformance_upb_failures.txt", - ":conformance_upb", - "@com_google_protobuf//:conformance_test_runner", - ], - deps = ["@bazel_tools//tools/bash/runfiles"], +genrule( + name = "generate_json_ragel", + srcs = ["//:upb/json/parser.rl"], + outs = ["upb/json/parser.c"], + cmd = "$(location @ragel//:ragelc) -C -o upb/json/parser.c $< && mv upb/json/parser.c $@", + tools = ["@ragel//:ragelc"], ) -# copybara:strip_for_google3_begin - # Amalgamation ################################################################# py_binary( @@ -732,15 +281,11 @@ cc_library( name = "amalgamation", srcs = ["upb.c"], hdrs = ["upb.h"], - copts = select({ - ":windows": [], - "//conditions:default": COPTS, - }), + copts = UPB_DEFAULT_COPTS, ) upb_amalgamation( name = "gen_php_amalgamation", - prefix = "php-", outs = [ "php-upb.c", "php-upb.h", @@ -754,21 +299,18 @@ upb_amalgamation( ":port", ":json", ], + prefix = "php-", ) cc_library( name = "php_amalgamation", srcs = ["php-upb.c"], hdrs = ["php-upb.h"], - copts = select({ - ":windows": [], - "//conditions:default": COPTS, - }), + copts = UPB_DEFAULT_COPTS, ) upb_amalgamation( name = "gen_ruby_amalgamation", - prefix = "ruby-", outs = [ "ruby-upb.c", "ruby-upb.h", @@ -781,102 +323,37 @@ upb_amalgamation( ":port", ":json", ], + prefix = "ruby-", ) cc_library( name = "ruby_amalgamation", srcs = ["ruby-upb.c"], hdrs = ["ruby-upb.h"], - copts = select({ - ":windows": [], - "//conditions:default": COPTS, - }), + copts = UPB_DEFAULT_COPTS, ) -# Lua ########################################################################## - -cc_library( - name = "lupb", - srcs = [ - "upb/bindings/lua/def.c", - "upb/bindings/lua/msg.c", - "upb/bindings/lua/upb.c", - ], - hdrs = [ - "upb/bindings/lua/upb.h", - ], - deps = [ - ":reflection", - ":textformat", - ":upb", - "@lua//:liblua", +exports_files( + [ + "upb/json/parser.rl", + "BUILD", + "WORKSPACE", ], + visibility = ["//cmake:__pkg__"], ) -cc_test( - name = "test_lua", - srcs = ["tests/bindings/lua/main.c"], - data = [ - "tests/bindings/lua/test_upb.lua", +exports_files( + [ "third_party/lunit/console.lua", "third_party/lunit/lunit.lua", - "upb/bindings/lua/upb.lua", - ":descriptor_proto_lua", - ":test_messages_proto3_proto_lua", - ":test_messages_proto2_proto_lua", - ":test_proto_lua", - "@com_google_protobuf//:conformance_proto", - "@com_google_protobuf//:descriptor_proto", - ], - linkstatic = 1, - deps = [ - ":lupb", - "@lua//:liblua", - ], -) - -cc_binary( - name = "protoc-gen-lua", - srcs = ["upb/bindings/lua/upbc.cc"], - copts = select({ - ":windows": [], - "//conditions:default": CPPOPTS, - }), - visibility = ["//visibility:public"], - deps = [ - "@com_google_absl//absl/strings", - "@com_google_protobuf//:protoc_lib", ], + visibility = ["//tests/bindings/lua:__pkg__"], ) -lua_proto_library( - name = "test_proto_lua", - testonly = 1, - deps = [":test_proto"], -) - -lua_proto_library( - name = "descriptor_proto_lua", - deps = ["@com_google_protobuf//:descriptor_proto"], -) - -lua_proto_library( - name = "test_messages_proto3_proto_lua", - testonly = 1, - deps = ["@com_google_protobuf//:test_messages_proto3_proto"], -) - -lua_proto_library( - name = "test_messages_proto2_proto_lua", - testonly = 1, - deps = ["@com_google_protobuf//:test_messages_proto2_proto"], -) - -# Test the CMake build ######################################################### - filegroup( name = "cmake_files", srcs = glob([ + "upb/json/parser.c", "CMakeLists.txt", "generated_for_cmake/**/*", "google/**/*", @@ -884,82 +361,7 @@ filegroup( "upb/**/*", "tests/**/*", ]), -) - -make_shell_script( - name = "gen_run_cmake_build", - out = "run_cmake_build.sh", - contents = "find . && mkdir build && cd build && cmake .. && make -j8 && make test", -) - -sh_test( - name = "cmake_build", - srcs = ["run_cmake_build.sh"], - data = [":cmake_files"], - deps = ["@bazel_tools//tools/bash/runfiles"], -) - -# Generated files ############################################################## - -exports_files(["tools/staleness_test.py"]) - -py_library( - name = "staleness_test_lib", - testonly = 1, - srcs = ["tools/staleness_test_lib.py"], -) - -py_binary( - name = "make_cmakelists", - srcs = ["tools/make_cmakelists.py"], -) - -genrule( - name = "gen_cmakelists", - srcs = [ - "BUILD", - "WORKSPACE", - ":cmake_files", - ], - outs = ["generated-in/CMakeLists.txt"], - cmd = "$(location :make_cmakelists) $@", - tools = [":make_cmakelists"], -) - -genrule( - name = "generate_json_ragel", - srcs = ["upb/json/parser.rl"], - outs = ["upb/json/parser.c"], - cmd = "$(location @ragel//:ragelc) -C -o upb/json/parser.c $< && mv upb/json/parser.c $@", - tools = ["@ragel//:ragelc"], -) - -genrule( - name = "copy_json_ragel", - srcs = ["upb/json/parser.c"], - outs = ["generated-in/generated_for_cmake/upb/json/parser.c"], - cmd = "cp $< $@", -) - -genrule( - name = "copy_protos", - srcs = [":descriptor_upb_proto"], - outs = [ - "generated-in/generated_for_cmake/google/protobuf/descriptor.upb.c", - "generated-in/generated_for_cmake/google/protobuf/descriptor.upb.h", - ], - cmd = "cp $(SRCS) $(@D)/generated-in/generated_for_cmake/google/protobuf", -) - -generated_file_staleness_test( - name = "test_generated_files", - outs = [ - "CMakeLists.txt", - "generated_for_cmake/google/protobuf/descriptor.upb.c", - "generated_for_cmake/google/protobuf/descriptor.upb.h", - "generated_for_cmake/upb/json/parser.c", - ], - generated_pattern = "generated-in/%s", + visibility = ["//cmake:__pkg__"], ) # copybara:strip_end diff --git a/bazel/build_defs.bzl b/bazel/build_defs.bzl index 121ae270c9..3555a03e8e 100644 --- a/bazel/build_defs.bzl +++ b/bazel/build_defs.bzl @@ -2,6 +2,29 @@ load(":upb_proto_library.bzl", "GeneratedSrcsInfo") +UPB_DEFAULT_CPPOPTS = select({ + "//:windows": [], + "//conditions:default": [ + # copybara:strip_for_google3_begin + "-Wextra", + # "-Wshorten-64-to-32", # not in GCC (and my Kokoro images doesn't have Clang) + "-Werror", + "-Wno-long-long", + # copybara:strip_end + ], +}) + +UPB_DEFAULT_COPTS = select({ + "//:windows": [], + "//conditions:default": [ + # copybara:strip_for_google3_begin + "-pedantic", + "-Werror=pedantic", + "-Wstrict-prototypes", + # copybara:strip_end + ], +}) + def _librule(name): return name + "_lib" @@ -58,50 +81,6 @@ def make_shell_script(name, contents, out): cmd = "(cat <<'HEREDOC'\n%s\nHEREDOC\n) > $@" % contents, ) -def generated_file_staleness_test(name, outs, generated_pattern): - """Tests that checked-in file(s) match the contents of generated file(s). - - The resulting test will verify that all output files exist and have the - correct contents. If the test fails, it can be invoked with --fix to - bring the checked-in files up to date. - - Args: - name: Name of the rule. - outs: the checked-in files that are copied from generated files. - generated_pattern: the pattern for transforming each "out" file into a - generated file. For example, if generated_pattern="generated/%s" then - a file foo.txt will look for generated file generated/foo.txt. - """ - - script_name = name + ".py" - script_src = "//:tools/staleness_test.py" - - # Filter out non-existing rules so Blaze doesn't error out before we even - # run the test. - existing_outs = native.glob(include = outs) - - # The file list contains a few extra bits of information at the end. - # These get unpacked by the Config class in staleness_test_lib.py. - file_list = outs + [generated_pattern, native.package_name() or ".", name] - - native.genrule( - name = name + "_makescript", - outs = [script_name], - srcs = [script_src], - testonly = 1, - cmd = "cat $(location " + script_src + ") > $@; " + - "sed -i.bak -e 's|INSERT_FILE_LIST_HERE|" + "\\\n ".join(file_list) + "|' $@", - ) - - native.py_test( - name = name, - srcs = [script_name], - data = existing_outs + [generated_pattern % file for file in outs], - deps = [ - "//:staleness_test_lib", - ], - ) - # upb_amalgamation() rule, with file_list aspect. SrcList = provider( @@ -156,7 +135,3 @@ upb_amalgamation = rule( }, implementation = _upb_amalgamation, ) - -def licenses(*args): - # No-op (for Google-internal usage). - pass diff --git a/bazel/lua.BUILD b/bazel/lua.BUILD index 7be0b594e5..113c71f9ae 100644 --- a/bazel/lua.BUILD +++ b/bazel/lua.BUILD @@ -4,7 +4,6 @@ package( cc_library( name = "liblua_headers", - defines = ["LUA_USE_LINUX"], hdrs = [ "src/lauxlib.h", "src/lua.h", @@ -12,6 +11,7 @@ cc_library( "src/luaconf.h", "src/lualib.h", ], + defines = ["LUA_USE_LINUX"], includes = ["src"], ) @@ -72,7 +72,6 @@ cc_library( "src/lzio.c", "src/lzio.h", ], - defines = ["LUA_USE_LINUX"], hdrs = [ "src/lauxlib.h", "src/lua.h", @@ -80,6 +79,7 @@ cc_library( "src/luaconf.h", "src/lualib.h", ], + defines = ["LUA_USE_LINUX"], includes = ["src"], linkopts = [ "-lm", @@ -92,11 +92,11 @@ cc_binary( srcs = [ "src/lua.c", ], - deps = [ - ":liblua", - ], linkopts = [ "-lreadline", "-rdynamic", ], + deps = [ + ":liblua", + ], ) diff --git a/bazel/ragel.BUILD b/bazel/ragel.BUILD index 5e3b24913d..5916bea198 100644 --- a/bazel/ragel.BUILD +++ b/bazel/ragel.BUILD @@ -1,4 +1,3 @@ - package( default_visibility = ["//visibility:public"], ) @@ -158,7 +157,10 @@ cc_binary( "aapl/avlimelkey.h", "aapl/avltree.h", ], - includes = ["ragel", "aapl"], + includes = [ + "aapl", + "ragel", + ], ) config_h_contents = """ diff --git a/bazel/upb_proto_library.bzl b/bazel/upb_proto_library.bzl index 7efd560b3e..c09cb3d127 100644 --- a/bazel/upb_proto_library.bzl +++ b/bazel/upb_proto_library.bzl @@ -225,7 +225,7 @@ _upb_proto_library_aspect = aspect( "_upbc": attr.label( executable = True, cfg = "host", - default = "//:protoc-gen-upb", + default = "//upbc:protoc-gen-upb", ), "_protoc": attr.label( executable = True, @@ -270,7 +270,7 @@ _upb_proto_reflection_library_aspect = aspect( "_upbc": attr.label( executable = True, cfg = "host", - default = "//:protoc-gen-upb", + default = "//upbc:protoc-gen-upb", ), "_protoc": attr.label( executable = True, diff --git a/benchmarks/BUILD b/benchmarks/BUILD new file mode 100644 index 0000000000..87315a34e9 --- /dev/null +++ b/benchmarks/BUILD @@ -0,0 +1,53 @@ +load( + "//bazel:upb_proto_library.bzl", + "upb_proto_library", + "upb_proto_reflection_library", +) + +licenses(["notice"]) + +proto_library( + name = "benchmark_descriptor_proto", + srcs = ["descriptor.proto"], +) + +upb_proto_library( + name = "benchmark_descriptor_upb_proto", + deps = [":benchmark_descriptor_proto"], +) + +upb_proto_reflection_library( + name = "benchmark_descriptor_upb_proto_reflection", + deps = [":benchmark_descriptor_proto"], +) + +cc_proto_library( + name = "benchmark_descriptor_cc_proto", + deps = [":benchmark_descriptor_proto"], +) + +proto_library( + name = "benchmark_descriptor_sv_proto", + srcs = ["descriptor_sv.proto"], +) + +cc_proto_library( + name = "benchmark_descriptor_sv_cc_proto", + deps = [":benchmark_descriptor_sv_proto"], +) + +cc_binary( + name = "benchmark", + testonly = 1, + srcs = ["benchmark.cc"], + deps = [ + ":benchmark_descriptor_cc_proto", + ":benchmark_descriptor_sv_cc_proto", + ":benchmark_descriptor_upb_proto", + ":benchmark_descriptor_upb_proto_reflection", + "//:descriptor_upb_proto", + "//:reflection", + "@com_github_google_benchmark//:benchmark_main", + "@com_google_protobuf//:protobuf", + ], +) diff --git a/tests/benchmark.cc b/benchmarks/benchmark.cc similarity index 96% rename from tests/benchmark.cc rename to benchmarks/benchmark.cc index c608572764..7f6b5f0f77 100644 --- a/tests/benchmark.cc +++ b/benchmarks/benchmark.cc @@ -3,10 +3,10 @@ #include // For benchmarks of parsing speed. -#include "tests/descriptor.pb.h" -#include "tests/descriptor.upb.h" -#include "tests/descriptor.upbdefs.h" -#include "tests/descriptor_sv.pb.h" +#include "benchmarks/descriptor.pb.h" +#include "benchmarks/descriptor.upb.h" +#include "benchmarks/descriptor.upbdefs.h" +#include "benchmarks/descriptor_sv.pb.h" // For for benchmarks of building descriptors. #include "google/protobuf/descriptor.upb.h" @@ -14,7 +14,7 @@ #include "upb/def.hpp" -upb_strview descriptor = tests_descriptor_proto_upbdefinit.descriptor; +upb_strview descriptor = benchmarks_descriptor_proto_upbdefinit.descriptor; namespace protobuf = ::google::protobuf; /* A buffer big enough to parse descriptor.proto without going to heap. */ diff --git a/benchmark.py b/benchmarks/compare.py similarity index 100% rename from benchmark.py rename to benchmarks/compare.py diff --git a/tests/descriptor.proto b/benchmarks/descriptor.proto similarity index 100% rename from tests/descriptor.proto rename to benchmarks/descriptor.proto diff --git a/tests/descriptor_sv.proto b/benchmarks/descriptor_sv.proto similarity index 100% rename from tests/descriptor_sv.proto rename to benchmarks/descriptor_sv.proto diff --git a/cmake/BUILD b/cmake/BUILD new file mode 100644 index 0000000000..76601b0e0f --- /dev/null +++ b/cmake/BUILD @@ -0,0 +1,89 @@ +load( + ":build_defs.bzl", + "generated_file_staleness_test", +) +load( + "//bazel:build_defs.bzl", + "make_shell_script", +) + +licenses(["notice"]) + +exports_files(["staleness_test.py"]) + +py_library( + name = "staleness_test_lib", + testonly = 1, + srcs = ["staleness_test_lib.py"], +) + +py_binary( + name = "make_cmakelists", + srcs = ["make_cmakelists.py"], +) + +genrule( + name = "gen_cmakelists", + srcs = [ + "//:BUILD", + "//:WORKSPACE", + "//:cmake_files", + ":cmake_files", + ], + outs = ["generated-in/CMakeLists.txt"], + cmd = "$(location :make_cmakelists) $@", + tools = [":make_cmakelists"], +) + +genrule( + name = "copy_json_ragel", + srcs = ["upb/json/parser.c"], + outs = ["generated-in/upb/json/parser.c"], + cmd = "cp $< $@", +) + +genrule( + name = "copy_protos", + srcs = ["//:descriptor_upb_proto"], + outs = [ + "generated-in/google/protobuf/descriptor.upb.c", + "generated-in/google/protobuf/descriptor.upb.h", + ], + cmd = "cp $(SRCS) $(@D)/generated-in/google/protobuf", +) + +generated_file_staleness_test( + name = "test_generated_files", + outs = [ + "CMakeLists.txt", + "google/protobuf/descriptor.upb.c", + "google/protobuf/descriptor.upb.h", + "upb/json/parser.c", + ], + generated_pattern = "generated-in/%s", +) + +# Test the CMake build ######################################################### + +filegroup( + name = "cmake_files", + srcs = glob([ + "**/*", + ]), +) + +make_shell_script( + name = "gen_run_cmake_build", + out = "run_cmake_build.sh", + contents = "find . && mkdir build && cd build && cmake ../cmake && make -j8 && make test", +) + +sh_test( + name = "cmake_build", + srcs = ["run_cmake_build.sh"], + data = [ + ":cmake_files", + "//:cmake_files", + ], + deps = ["@bazel_tools//tools/bash/runfiles"], +) diff --git a/CMakeLists.txt b/cmake/CMakeLists.txt similarity index 72% rename from CMakeLists.txt rename to cmake/CMakeLists.txt index 67eeb2c698..487e8190f3 100644 --- a/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -48,8 +48,8 @@ if(UPB_ENABLE_UBSAN) set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=address") endif() -include_directories(.) -include_directories(generated_for_cmake) +include_directories(..) +include_directories(../cmake) include_directories(${CMAKE_CURRENT_BINARY_DIR}) if(APPLE) @@ -61,20 +61,20 @@ endif() enable_testing() add_library(port - upb/port.c) + ../upb/port.c) add_library(upb - upb/decode.c - upb/encode.c - upb/msg.c - upb/msg.h - upb/table.c - upb/table.int.h - upb/upb.c - upb/upb.int.h - upb/decode.h - upb/encode.h - upb/upb.h - upb/upb.hpp) + ../upb/decode.c + ../upb/encode.c + ../upb/msg.c + ../upb/msg.h + ../upb/table.c + ../upb/table.int.h + ../upb/upb.c + ../upb/upb.int.h + ../upb/decode.h + ../upb/encode.h + ../upb/upb.h + ../upb/upb.hpp) target_link_libraries(upb port) add_library(generated_code_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me INTERFACE) @@ -82,28 +82,28 @@ target_link_libraries(generated_code_support__only_for_generated_code_do_not_use table upb) add_library(reflection - upb/def.c - upb/msg.h - upb/reflection.c - upb/def.h - upb/def.hpp - upb/reflection.h) + ../upb/def.c + ../upb/msg.h + ../upb/reflection.c + ../upb/def.h + ../upb/def.hpp + ../upb/reflection.h) target_link_libraries(reflection descriptor_upb_proto port table upb) add_library(textformat - upb/text_encode.c - upb/text_encode.h) + ../upb/text_encode.c + ../upb/text_encode.h) target_link_libraries(textformat port reflection) add_library(json - upb/json_decode.c - upb/json_encode.c - upb/json_decode.h - upb/json_encode.h) + ../upb/json_decode.c + ../upb/json_encode.c + ../upb/json_decode.h + ../upb/json_encode.h) target_link_libraries(json port reflection @@ -113,27 +113,27 @@ target_link_libraries(table INTERFACE port upb) add_library(handlers - upb/handlers.c - upb/handlers-inl.h - upb/sink.c - upb/handlers.h - upb/sink.h) + ../upb/handlers.c + ../upb/handlers-inl.h + ../upb/sink.c + ../upb/handlers.h + ../upb/sink.h) target_link_libraries(handlers port reflection table upb) add_library(upb_pb - upb/pb/compile_decoder.c - upb/pb/decoder.c - upb/pb/decoder.int.h - upb/pb/encoder.c - upb/pb/textprinter.c - upb/pb/varint.c - upb/pb/varint.int.h - upb/pb/decoder.h - upb/pb/encoder.h - upb/pb/textprinter.h) + ../upb/pb/compile_decoder.c + ../upb/pb/decoder.c + ../upb/pb/decoder.int.h + ../upb/pb/encoder.c + ../upb/pb/textprinter.c + ../upb/pb/varint.c + ../upb/pb/varint.int.h + ../upb/pb/decoder.h + ../upb/pb/encoder.h + ../upb/pb/textprinter.h) target_link_libraries(upb_pb descriptor_upb_proto handlers @@ -142,26 +142,12 @@ target_link_libraries(upb_pb table upb) add_library(upb_json - generated_for_cmake/upb/json/parser.c - upb/json/printer.c - upb/json/parser.h - upb/json/printer.h) + ../cmake/upb/json/parser.c + ../upb/json/printer.c + ../upb/json/parser.h + ../upb/json/printer.h) target_link_libraries(upb_json upb upb_pb) -add_library(upb_cc_bindings INTERFACE) -target_link_libraries(upb_cc_bindings INTERFACE - descriptor_upb_proto - handlers - port - upb) -add_library(upb_test - tests/testmain.cc - tests/test_util.h - tests/upb_test.h) -target_link_libraries(upb_test - handlers - port - upb) diff --git a/cmake/README.md b/cmake/README.md new file mode 100644 index 0000000000..211a054ab0 --- /dev/null +++ b/cmake/README.md @@ -0,0 +1,23 @@ + +# upb CMake build (EXPERIMENTAL) + +upb's CMake support is experimental. The core library builds successfully +under CMake, and this is verified by the Bazel tests in this directory. +However there is no support for building the upb compiler or for generating +.upb.c/upb.h files. This means upb's CMake support is incomplete at best, +unless your application is intended to be purely reflective. + +If you find this CMake setup useful in its current state, please consider +filing an issue so we know. If you have suggestions for how it could be +more useful (and particularly if you can contribute some code for it) +please feel free to file an issue for that too. Do keep in mind that upb +does not currently provide any ABI stability, so we want to avoid providing +a shared library. + +The CMakeLists.txt is generated from the Bazel BUILD files using the Python +scripts in this directory. We want to avoid having two separate sources of +truth that both need to be updated when a file is added or removed. + +This directory also contains some generated files that would be created +on the fly during a Bazel build. These are automaticaly kept in sync by +the Bazel test `//cmake:test_generated_files`. diff --git a/cmake/build_defs.bzl b/cmake/build_defs.bzl new file mode 100644 index 0000000000..83f2f7aa75 --- /dev/null +++ b/cmake/build_defs.bzl @@ -0,0 +1,44 @@ + +def generated_file_staleness_test(name, outs, generated_pattern): + """Tests that checked-in file(s) match the contents of generated file(s). + + The resulting test will verify that all output files exist and have the + correct contents. If the test fails, it can be invoked with --fix to + bring the checked-in files up to date. + + Args: + name: Name of the rule. + outs: the checked-in files that are copied from generated files. + generated_pattern: the pattern for transforming each "out" file into a + generated file. For example, if generated_pattern="generated/%s" then + a file foo.txt will look for generated file generated/foo.txt. + """ + + script_name = name + ".py" + script_src = ":staleness_test.py" + + # Filter out non-existing rules so Blaze doesn't error out before we even + # run the test. + existing_outs = native.glob(include = outs) + + # The file list contains a few extra bits of information at the end. + # These get unpacked by the Config class in staleness_test_lib.py. + file_list = outs + [generated_pattern, native.package_name() or ".", name] + + native.genrule( + name = name + "_makescript", + outs = [script_name], + srcs = [script_src], + testonly = 1, + cmd = "cat $(location " + script_src + ") > $@; " + + "sed -i.bak -e 's|INSERT_FILE_LIST_HERE|" + "\\\n ".join(file_list) + "|' $@", + ) + + native.py_test( + name = name, + srcs = [script_name], + data = existing_outs + [generated_pattern % file for file in outs], + deps = [ + ":staleness_test_lib", + ], + ) diff --git a/generated_for_cmake/google/protobuf/descriptor.upb.c b/cmake/google/protobuf/descriptor.upb.c similarity index 100% rename from generated_for_cmake/google/protobuf/descriptor.upb.c rename to cmake/google/protobuf/descriptor.upb.c diff --git a/generated_for_cmake/google/protobuf/descriptor.upb.h b/cmake/google/protobuf/descriptor.upb.h similarity index 100% rename from generated_for_cmake/google/protobuf/descriptor.upb.h rename to cmake/google/protobuf/descriptor.upb.h diff --git a/tools/make_cmakelists.py b/cmake/make_cmakelists.py similarity index 96% rename from tools/make_cmakelists.py rename to cmake/make_cmakelists.py index 15c0cf6925..3582d0543a 100755 --- a/tools/make_cmakelists.py +++ b/cmake/make_cmakelists.py @@ -46,9 +46,9 @@ class BuildFileFunctions(object): found_files = [] for file in files: if os.path.isfile(file): - found_files.append(file) - elif os.path.isfile("generated_for_cmake/" + file): - found_files.append("generated_for_cmake/" + file) + found_files.append("../" + file) + elif os.path.isfile("cmake/" + file): + found_files.append("../cmake/" + file) else: print("Warning: no such file: " + file) @@ -242,8 +242,8 @@ class Converter(object): set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=address") endif() - include_directories(.) - include_directories(generated_for_cmake) + include_directories(..) + include_directories(../cmake) include_directories(${CMAKE_CURRENT_BINARY_DIR}) if(APPLE) @@ -263,6 +263,7 @@ converter = Converter() def GetDict(obj): ret = {} + ret["UPB_DEFAULT_COPTS"] = [] # HACK for k in dir(obj): if not k.startswith("_"): ret[k] = getattr(obj, k); diff --git a/tools/staleness_test.py b/cmake/staleness_test.py similarity index 94% rename from tools/staleness_test.py rename to cmake/staleness_test.py index 045cd1a319..585730715e 100644 --- a/tools/staleness_test.py +++ b/cmake/staleness_test.py @@ -6,7 +6,7 @@ with the actual list of files before we actually run the script. from __future__ import absolute_import -from tools import staleness_test_lib +from cmake import staleness_test_lib import unittest import sys diff --git a/tools/staleness_test_lib.py b/cmake/staleness_test_lib.py similarity index 93% rename from tools/staleness_test_lib.py rename to cmake/staleness_test_lib.py index 6d5c3d3068..cdfcc0d17a 100644 --- a/tools/staleness_test_lib.py +++ b/cmake/staleness_test_lib.py @@ -7,6 +7,7 @@ generated_file_staleness_test() rules. from __future__ import absolute_import from __future__ import print_function +import sys import os from shutil import copyfile @@ -47,13 +48,13 @@ def _GetFilePairs(config): ret = [] - has_bazel_genfiles = os.path.exists("bazel-genfiles") + has_bazel_genfiles = os.path.exists("bazel-bin") for filename in config.file_list: target = os.path.join(config.package_name, filename) generated = os.path.join(config.package_name, config.pattern % filename) if has_bazel_genfiles: - generated = os.path.join("bazel-genfiles", generated) + generated = os.path.join("bazel-bin", generated) # Generated files should always exist. Blaze should guarantee this before # we are run. @@ -61,6 +62,7 @@ def _GetFilePairs(config): print("Generated file '%s' does not exist." % generated) print("Please run this command to generate it:") print(" bazel build %s:%s" % (config.package_name, config.target_name)) + sys.exit(1) ret.append(_FilePair(target, generated)) return ret @@ -87,10 +89,9 @@ def _GetMissingAndStaleFiles(file_pairs): missing_files.append(pair) continue - generated = open(pair.generated).read() - target = open(pair.target).read() - if generated != target: - stale_files.append(pair) + with open(pair.generated) as g, open(pair.target) as t: + if g.read() != t.read(): + stale_files.append(pair) return missing_files, stale_files diff --git a/generated_for_cmake/upb/json/parser.c b/cmake/upb/json/parser.c similarity index 100% rename from generated_for_cmake/upb/json/parser.c rename to cmake/upb/json/parser.c diff --git a/examples/bazel/BUILD b/examples/bazel/BUILD index 9fb5c1f4a6..550115b21d 100644 --- a/examples/bazel/BUILD +++ b/examples/bazel/BUILD @@ -1,6 +1,8 @@ load("@rules_proto//proto:defs.bzl", "proto_library") load("@upb//bazel:upb_proto_library.bzl", "upb_proto_library") +licenses(["notice"]) + proto_library( name = "foo_proto", srcs = ["foo.proto"], diff --git a/examples/bazel/test_binary.c b/examples/bazel/test_binary.c index 78f367a370..98617cf020 100644 --- a/examples/bazel/test_binary.c +++ b/examples/bazel/test_binary.c @@ -1,7 +1,7 @@ #include -#include "foo.upb.h" +#include "examples/bazel/foo.upb.h" int main() { upb_arena *arena = upb_arena_new(); diff --git a/kokoro/ubuntu/build.sh b/kokoro/ubuntu/build.sh index 2174e8aa57..1b1089097e 100644 --- a/kokoro/ubuntu/build.sh +++ b/kokoro/ubuntu/build.sh @@ -19,12 +19,12 @@ which bazel bazel version cd $(dirname $0)/../.. -bazel test --test_output=errors :all +bazel test --test_output=errors ... if [[ $(uname) = "Linux" ]]; then # Verify the ASAN build. Have to exclude test_conformance_upb as protobuf # currently leaks memory in the conformance test runner. - bazel test --copt=-fsanitize=address --linkopt=-fsanitize=address --test_output=errors :all + bazel test --copt=-fsanitize=address --linkopt=-fsanitize=address --test_output=errors ... # Verify the UBSan build. Have to exclude Lua as the version we are using # fails some UBSan tests. @@ -35,5 +35,5 @@ if [[ $(uname) = "Linux" ]]; then fi if which valgrind; then - bazel test --run_under='valgrind --leak-check=full --error-exitcode=1' :all -- -:test_conformance_upb -:cmake_build + bazel test --run_under='valgrind --leak-check=full --error-exitcode=1' ... -- -tests:test_conformance_upb -cmake:cmake_build fi diff --git a/tests/BUILD b/tests/BUILD new file mode 100644 index 0000000000..998023e96d --- /dev/null +++ b/tests/BUILD @@ -0,0 +1,282 @@ +load( + "//bazel:build_defs.bzl", + "UPB_DEFAULT_COPTS", + "UPB_DEFAULT_CPPOPTS", + "make_shell_script", +) +load( + "//bazel:upb_proto_library.bzl", + "upb_proto_library", + "upb_proto_reflection_library", +) + +licenses(["notice"]) + +config_setting( + name = "fuzz", + values = {"define": "fuzz=true"}, +) + +cc_library( + name = "upb_test", + testonly = 1, + srcs = [ + "testmain.cc", + ], + hdrs = [ + "test_util.h", + "upb_test.h", + ], + copts = UPB_DEFAULT_CPPOPTS, + deps = [ + "//:handlers", + "//:port", + "//:upb", + ], +) + +proto_library( + name = "test_proto", + testonly = 1, + srcs = ["test.proto"], +) + +upb_proto_library( + name = "test_upb_proto", + testonly = 1, + deps = [":test_proto"], +) + +cc_test( + name = "test_generated_code", + srcs = ["test_generated_code.c"], + deps = [ + ":empty_upbdefs_proto", + ":test_messages_proto3_proto_upb", + ":test_upb_proto", + ":upb_test", + ], +) + +proto_library( + name = "empty_proto", + srcs = ["empty.proto"], +) + +upb_proto_reflection_library( + name = "empty_upbdefs_proto", + testonly = 1, + deps = [":empty_proto"], +) + +upb_proto_library( + name = "test_messages_proto3_proto_upb", + testonly = 1, + deps = ["@com_google_protobuf//:test_messages_proto3_proto"], +) + +proto_library( + name = "test_decoder_proto", + srcs = [ + "pb/test_decoder.proto", + ], +) + +upb_proto_reflection_library( + name = "test_decoder_upb_proto", + deps = [":test_decoder_proto"], +) + +cc_test( + name = "test_decoder", + srcs = ["pb/test_decoder.cc"], + copts = UPB_DEFAULT_CPPOPTS, + deps = [ + ":test_decoder_upb_proto", + ":upb_test", + "//:handlers", + "//:port", + "//:upb", + "//:upb_pb", + ], +) + +proto_library( + name = "test_cpp_proto", + srcs = [ + "test_cpp.proto", + ], +) + +upb_proto_reflection_library( + name = "test_cpp_upb_proto", + deps = ["test_cpp_proto"], +) + +cc_test( + name = "test_cpp", + srcs = ["test_cpp.cc"], + copts = UPB_DEFAULT_CPPOPTS, + deps = [ + ":test_cpp_upb_proto", + ":upb_test", + "//:handlers", + "//:port", + "//:reflection", + "//:upb", + "//:upb_pb", + ], +) + +cc_test( + name = "test_table", + srcs = ["test_table.cc"], + copts = UPB_DEFAULT_CPPOPTS, + deps = [ + ":upb_test", + "//:port", + "//:table", + "//:upb", + ], +) + +# OSS-Fuzz test +cc_binary( + name = "file_descriptor_parsenew_fuzzer", + testonly = 1, + srcs = ["file_descriptor_parsenew_fuzzer.cc"], + copts = UPB_DEFAULT_CPPOPTS + select({ + "//conditions:default": [], + ":fuzz": ["-fsanitize=fuzzer,address"], + }), + defines = select({ + "//conditions:default": [], + ":fuzz": ["HAVE_FUZZER"], + }), + deps = [ + "//:descriptor_upb_proto", + "//:upb", + ], +) + +# copybara:strip_for_google3_begin +cc_test( + name = "test_encoder", + srcs = ["pb/test_encoder.cc"], + copts = UPB_DEFAULT_CPPOPTS, + deps = [ + ":upb_test", + "//:descriptor_upb_proto", + "//:descriptor_upb_proto_reflection", + "//:upb", + "//:upb_pb", + ], +) + +proto_library( + name = "test_json_enum_from_separate", + srcs = ["json/enum_from_separate_file.proto"], + deps = [":test_json_proto"], +) + +proto_library( + name = "test_json_proto", + srcs = ["json/test.proto"], +) + +upb_proto_reflection_library( + name = "test_json_upb_proto_reflection", + deps = ["test_json_proto"], +) + +upb_proto_library( + name = "test_json_enum_from_separate_upb_proto", + deps = [":test_json_enum_from_separate"], +) + +upb_proto_library( + name = "test_json_upb_proto", + deps = [":test_json_proto"], +) + +cc_test( + name = "test_json", + srcs = [ + "json/test_json.cc", + ], + copts = UPB_DEFAULT_CPPOPTS, + deps = [ + ":test_json_upb_proto", + ":test_json_upb_proto_reflection", + ":upb_test", + "//:upb_json", + ], +) +# copybara:strip_end + +upb_proto_library( + name = "conformance_proto_upb", + testonly = 1, + deps = ["@com_google_protobuf//:conformance_proto"], +) + +upb_proto_reflection_library( + name = "conformance_proto_upbdefs", + testonly = 1, + deps = ["@com_google_protobuf//:conformance_proto"], +) + +upb_proto_reflection_library( + name = "test_messages_proto2_upbdefs", + testonly = 1, + deps = ["@com_google_protobuf//:test_messages_proto2_proto"], +) + +upb_proto_reflection_library( + name = "test_messages_proto3_upbdefs", + testonly = 1, + deps = ["@com_google_protobuf//:test_messages_proto3_proto"], +) + +cc_binary( + name = "conformance_upb", + testonly = 1, + srcs = [ + "conformance_upb.c", + ], + copts = UPB_DEFAULT_COPTS, + data = [ + "conformance_upb_failures.txt", + ], + deps = [ + ":conformance_proto_upb", + ":conformance_proto_upbdefs", + ":test_messages_proto2_upbdefs", + ":test_messages_proto3_upbdefs", + "//:json", + "//:port", + "//:reflection", + "//:textformat", + "//:upb", + ], +) + +make_shell_script( + name = "gen_test_conformance_upb", + out = "test_conformance_upb.sh", + contents = "external/com_google_protobuf/conformance_test_runner " + + " --enforce_recommended " + + " --failure_list ./tests/conformance_upb_failures.txt" + + " ./tests/conformance_upb", +) + +sh_test( + name = "test_conformance_upb", + srcs = ["test_conformance_upb.sh"], + data = [ + "conformance_upb_failures.txt", + ":conformance_upb", + "@com_google_protobuf//:conformance_test_runner", + ], + deps = ["@bazel_tools//tools/bash/runfiles"], +) diff --git a/tests/bindings/lua/BUILD b/tests/bindings/lua/BUILD new file mode 100644 index 0000000000..ba4918d415 --- /dev/null +++ b/tests/bindings/lua/BUILD @@ -0,0 +1,57 @@ +load( + "//upb/bindings/lua:lua_proto_library.bzl", + "lua_proto_library", +) + +licenses(["notice"]) + +cc_test( + name = "test_lua", + srcs = ["main.c"], + data = [ + "test_upb.lua", + ":descriptor_proto_lua", + ":test_messages_proto2_proto_lua", + ":test_messages_proto3_proto_lua", + ":test_proto_lua", + "//:third_party/lunit/console.lua", + "//:third_party/lunit/lunit.lua", + "//upb/bindings/lua:upb.lua", + "@com_google_protobuf//:conformance_proto", + "@com_google_protobuf//:descriptor_proto", + ], + linkstatic = 1, + deps = [ + "//upb/bindings/lua:lupb", + "@lua//:liblua", + ], +) + +proto_library( + name = "test_proto", + testonly = 1, + srcs = ["test.proto"], +) + +lua_proto_library( + name = "test_proto_lua", + testonly = 1, + deps = [":test_proto"], +) + +lua_proto_library( + name = "descriptor_proto_lua", + deps = ["@com_google_protobuf//:descriptor_proto"], +) + +lua_proto_library( + name = "test_messages_proto3_proto_lua", + testonly = 1, + deps = ["@com_google_protobuf//:test_messages_proto3_proto"], +) + +lua_proto_library( + name = "test_messages_proto2_proto_lua", + testonly = 1, + deps = ["@com_google_protobuf//:test_messages_proto2_proto"], +) diff --git a/tests/bindings/lua/test.proto b/tests/bindings/lua/test.proto new file mode 100644 index 0000000000..c4b7e9c3db --- /dev/null +++ b/tests/bindings/lua/test.proto @@ -0,0 +1,8 @@ + +syntax = "proto2"; + +package upb_test; + +message MapTest { + map map_string_double = 1; +} diff --git a/tests/bindings/lua/test_upb.lua b/tests/bindings/lua/test_upb.lua index b44c595dd1..345828d500 100644 --- a/tests/bindings/lua/test_upb.lua +++ b/tests/bindings/lua/test_upb.lua @@ -1,7 +1,7 @@ local upb = require "lupb" local lunit = require "lunit" -local upb_test = require "tests.test_pb" +local upb_test = require "tests.bindings.lua.test_pb" local test_messages_proto3 = require "google.protobuf.test_messages_proto3_pb" local test_messages_proto2 = require "google.protobuf.test_messages_proto2_pb" local descriptor = require "google.protobuf.descriptor_pb" diff --git a/tests/pb/test_decoder.cc b/tests/pb/test_decoder.cc index 14fd72abbc..32eec748e7 100644 --- a/tests/pb/test_decoder.cc +++ b/tests/pb/test_decoder.cc @@ -43,7 +43,6 @@ #else // AMALGAMATED #include "upb/handlers.h" #include "upb/pb/decoder.h" -#include "upb/pb/varint.int.h" #include "upb/upb.h" #endif // !AMALGAMATED @@ -147,6 +146,29 @@ void PrintBinary(const string& str) { } } +#define UPB_PB_VARINT_MAX_LEN 10 + +static size_t upb_vencode64(uint64_t val, char *buf) { + size_t i; + if (val == 0) { buf[0] = 0; return 1; } + i = 0; + while (val) { + uint8_t byte = val & 0x7fU; + val >>= 7; + if (val) byte |= 0x80U; + buf[i++] = byte; + } + return i; +} + +static uint32_t upb_zzenc_32(int32_t n) { + return ((uint32_t)n << 1) ^ (n >> 31); +} + +static uint64_t upb_zzenc_64(int64_t n) { + return ((uint64_t)n << 1) ^ (n >> 63); +} + /* Routines for building arbitrary protos *************************************/ const string empty; diff --git a/tests/pb/test_encoder.cc b/tests/pb/test_encoder.cc index aeca1b39ca..b358ef592c 100644 --- a/tests/pb/test_encoder.cc +++ b/tests/pb/test_encoder.cc @@ -5,12 +5,65 @@ #include "google/protobuf/descriptor.upbdefs.h" #include "tests/test_util.h" #include "tests/upb_test.h" -#include "upb/bindings/stdc++/string.h" #include "upb/pb/decoder.h" #include "upb/pb/encoder.h" #include "upb/port_def.inc" #include "upb/upb.hpp" +template +class FillStringHandler { + public: + static void SetHandler(upb_byteshandler* handler) { + upb_byteshandler_setstartstr(handler, &FillStringHandler::StartString, + NULL); + upb_byteshandler_setstring(handler, &FillStringHandler::StringBuf, NULL); + } + + private: + // TODO(haberman): add UpbBind/UpbMakeHandler support to BytesHandler so these + // can be prettier callbacks. + static void* StartString(void *c, const void *hd, size_t size) { + UPB_UNUSED(hd); + UPB_UNUSED(size); + + T* str = static_cast(c); + str->clear(); + return c; + } + + static size_t StringBuf(void* c, const void* hd, const char* buf, size_t n, + const upb_bufhandle* h) { + UPB_UNUSED(hd); + UPB_UNUSED(h); + + T* str = static_cast(c); + try { + str->append(buf, n); + return n; + } catch (const std::exception&) { + return 0; + } + } +}; + +class StringSink { + public: + template + explicit StringSink(T* target) { + // TODO(haberman): we need to avoid rebuilding a new handler every time, + // but with class globals disallowed for google3 C++ this is tricky. + upb_byteshandler_init(&handler_); + FillStringHandler::SetHandler(&handler_); + input_.Reset(&handler_, target); + } + + upb::BytesSink input() { return input_; } + + private: + upb_byteshandler handler_; + upb::BytesSink input_; +}; + void test_pb_roundtrip() { std::string input( google_protobuf_descriptor_proto_upbdefinit.descriptor.data, @@ -29,7 +82,7 @@ void test_pb_roundtrip() { const upb::pb::DecoderMethodPtr method = decoder_cache.Get(md); std::string output; - upb::StringSink string_sink(&output); + StringSink string_sink(&output); upb::pb::EncoderPtr encoder = upb::pb::EncoderPtr::Create(&arena, encoder_handlers, string_sink.input()); upb::pb::DecoderPtr decoder = diff --git a/tests/pb/test_varint.c b/tests/pb/test_varint.c deleted file mode 100644 index 45d8e47dcd..0000000000 --- a/tests/pb/test_varint.c +++ /dev/null @@ -1,126 +0,0 @@ - -#include -#include "upb/pb/varint.int.h" -#include "tests/upb_test.h" - -#include "upb/port_def.inc" - -/* Test that we can round-trip from int->varint->int. */ -static void test_varint_for_num(upb_decoderet (*decoder)(const char*), - uint64_t num) { - char buf[16]; - size_t bytes; - upb_decoderet r; - - memset(buf, 0xff, sizeof(buf)); - bytes = upb_vencode64(num, buf); - - if (num <= UINT32_MAX) { - uint64_t encoded = upb_vencode32((uint32_t)num); - char buf2[16]; - upb_decoderet r; - - memset(buf2, 0, sizeof(buf2)); - memcpy(&buf2, &encoded, 8); -#ifdef UPB_BIG_ENDIAN - char swap[8]; - swap[0] = buf2[7]; - swap[1] = buf2[6]; - swap[2] = buf2[5]; - swap[3] = buf2[4]; - swap[4] = buf2[3]; - swap[5] = buf2[2]; - swap[6] = buf2[1]; - swap[7] = buf2[0]; - buf2[0] = swap[0]; - buf2[1] = swap[1]; - buf2[2] = swap[2]; - buf2[3] = swap[3]; - buf2[4] = swap[4]; - buf2[5] = swap[5]; - buf2[6] = swap[6]; - buf2[7] = swap[7]; -#endif - r = decoder(buf2); - ASSERT(r.val == num); - ASSERT(r.p == buf2 + upb_value_size(encoded)); - ASSERT(upb_zzenc_32(upb_zzdec_32((uint32_t)num)) == num); - } - - r = decoder(buf); - ASSERT(r.val == num); - ASSERT(r.p == buf + bytes); - ASSERT(upb_zzenc_64(upb_zzdec_64(num)) == num); -} - -/* Making up for the lack of 64-bit constants in C89. */ -static uint64_t make_u64(uint32_t high, uint32_t low) { - uint64_t ret = high; - ret = (ret << 32) | low; - return ret; -} - -static void test_varint_decoder(upb_decoderet (*decoder)(const char*)) { -#define TEST(bytes, expected_val) {\ - size_t n = sizeof(bytes) - 1; /* for NULL */ \ - char buf[UPB_PB_VARINT_MAX_LEN]; \ - upb_decoderet r; \ - memset(buf, 0xff, sizeof(buf)); \ - memcpy(buf, bytes, n); \ - r = decoder(buf); \ - ASSERT(r.val == expected_val); \ - ASSERT(r.p == buf + n); \ - } - - uint64_t num; - - char twelvebyte[16] = {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 1}; - const char *twelvebyte_buf = twelvebyte; - /* A varint that terminates before hitting the end of the provided buffer, - * but in too many bytes (11 instead of 10). */ - upb_decoderet r = decoder(twelvebyte_buf); - ASSERT(r.p == NULL); - - TEST("\x00", 0UL); - TEST("\x01", 1UL); - TEST("\x81\x14", 0xa01UL); - TEST("\x81\x03", 0x181UL); - TEST("\x81\x83\x07", 0x1c181UL); - TEST("\x81\x83\x87\x0f", 0x1e1c181UL); - TEST("\x81\x83\x87\x8f\x1f", make_u64(0x1, 0xf1e1c181UL)); - TEST("\x81\x83\x87\x8f\x9f\x3f", make_u64(0x1f9, 0xf1e1c181UL)); - TEST("\x81\x83\x87\x8f\x9f\xbf\x7f", make_u64(0x1fdf9, 0xf1e1c181UL)); - TEST("\x81\x83\x87\x8f\x9f\xbf\xff\x01", make_u64(0x3fdf9, 0xf1e1c181UL)); - TEST("\x81\x83\x87\x8f\x9f\xbf\xff\x81\x03", - make_u64(0x303fdf9, 0xf1e1c181UL)); - TEST("\x81\x83\x87\x8f\x9f\xbf\xff\x81\x83\x07", - make_u64(0x8303fdf9, 0xf1e1c181UL)); -#undef TEST - - for (num = 5; num * 1.5 < UINT64_MAX; num *= 1.5) { - test_varint_for_num(decoder, num); - } - test_varint_for_num(decoder, 0); -} - - -#define TEST_VARINT_DECODER(decoder) \ - /* Create non-inline versions for convenient inspection of assembly language \ - * output. */ \ - upb_decoderet _upb_vdecode_ ## decoder(const char *p) { \ - return upb_vdecode_ ## decoder(p); \ - } \ - void test_ ## decoder(void) { \ - test_varint_decoder(&_upb_vdecode_ ## decoder); \ - } \ - -TEST_VARINT_DECODER(check2_branch32) -TEST_VARINT_DECODER(check2_branch64) - -int run_tests(int argc, char *argv[]) { - UPB_UNUSED(argc); - UPB_UNUSED(argv); - test_check2_branch32(); - test_check2_branch64(); - return 0; -} diff --git a/upb/bindings/lua/BUILD b/upb/bindings/lua/BUILD new file mode 100644 index 0000000000..cf1aa76371 --- /dev/null +++ b/upb/bindings/lua/BUILD @@ -0,0 +1,40 @@ +load( + "//bazel:build_defs.bzl", + "UPB_DEFAULT_COPTS", + "UPB_DEFAULT_CPPOPTS", +) + +licenses(["notice"]) + +cc_library( + name = "lupb", + srcs = [ + "def.c", + "msg.c", + "upb.c", + ], + hdrs = [ + "upb.h", + ], + copts = UPB_DEFAULT_COPTS, + visibility = ["//visibility:public"], + deps = [ + "//:reflection", + "//:textformat", + "//:upb", + "@lua//:liblua", + ], +) + +cc_binary( + name = "protoc-gen-lua", + srcs = ["upbc.cc"], + copts = UPB_DEFAULT_CPPOPTS, + visibility = ["//visibility:public"], + deps = [ + "@com_google_absl//absl/strings", + "@com_google_protobuf//:protoc_lib", + ], +) + +exports_files(["upb.lua"]) diff --git a/upb/bindings/lua/def.c b/upb/bindings/lua/def.c index 0aa366d593..3bf1d9654f 100644 --- a/upb/bindings/lua/def.c +++ b/upb/bindings/lua/def.c @@ -56,8 +56,8 @@ static void lupb_wrapper_pushwrapper(lua_State *L, int narg, const void *def, * wrapper for field |f| of this msgdef. */ void lupb_msgdef_pushsubmsgdef(lua_State *L, const upb_fielddef *f) { - assert(luaL_testudata(L, -1, LUPB_MSGDEF)); const upb_msgdef *m = upb_fielddef_msgsubdef(f); + assert(m); assert(upb_fielddef_containingtype(f) == lupb_msgdef_check(L, -1)); lupb_wrapper_pushwrapper(L, -1, m, LUPB_MSGDEF); lua_replace(L, -2); /* Replace msgdef with submsgdef. */ @@ -251,8 +251,8 @@ static int lupb_oneofiter_next(lua_State *L) { } static int lupb_oneofdef_fields(lua_State *L) { - lupb_oneofdef_check(L, 1); int *index = lua_newuserdata(L, sizeof(int)); + lupb_oneofdef_check(L, 1); *index = 0; /* Closure upvalues are: oneofdef, index. */ @@ -408,8 +408,8 @@ static int lupb_msgfielditer_next(lua_State *L) { } static int lupb_msgdef_fields(lua_State *L) { - lupb_msgdef_check(L, 1); int *index = lua_newuserdata(L, sizeof(int)); + lupb_msgdef_check(L, 1); *index = 0; /* Closure upvalues are: msgdef, index. */ @@ -441,8 +441,8 @@ static int lupb_msgoneofiter_next(lua_State *L) { } static int lupb_msgdef_oneofs(lua_State *L) { - lupb_msgdef_check(L, 1); int *index = lua_newuserdata(L, sizeof(int)); + lupb_msgdef_check(L, 1); *index = 0; /* Closure upvalues are: msgdef, index. */ @@ -750,7 +750,7 @@ static int lupb_symtab_addfile(lua_State *L) { size_t len; upb_symtab *s = lupb_symtab_check(L, 1); const char *str = luaL_checklstring(L, 2, &len); - upb_arena *arena = lupb_arena_pushnew(L);; + upb_arena *arena = lupb_arena_pushnew(L); const google_protobuf_FileDescriptorProto *file; const upb_filedef *file_def; upb_status status; @@ -776,7 +776,7 @@ static int lupb_symtab_addset(lua_State *L) { google_protobuf_FileDescriptorSet *set; upb_symtab *s = lupb_symtab_check(L, 1); const char *str = luaL_checklstring(L, 2, &len); - upb_arena *arena = lupb_arena_pushnew(L);; + upb_arena *arena = lupb_arena_pushnew(L); upb_status status; upb_status_clear(&status); diff --git a/upb/bindings/lua/lua_proto_library.bzl b/upb/bindings/lua/lua_proto_library.bzl index f21655140a..29b1caa07a 100644 --- a/upb/bindings/lua/lua_proto_library.bzl +++ b/upb/bindings/lua/lua_proto_library.bzl @@ -82,7 +82,7 @@ _lua_proto_library_aspect = aspect( "_upbc": attr.label( executable = True, cfg = "host", - default = "//:protoc-gen-lua", + default = "//upb/bindings/lua:protoc-gen-lua", ), "_protoc": attr.label( executable = True, diff --git a/upb/bindings/lua/msg.c b/upb/bindings/lua/msg.c index df9f009d76..6a2ecb36c7 100644 --- a/upb/bindings/lua/msg.c +++ b/upb/bindings/lua/msg.c @@ -564,8 +564,8 @@ static int lupb_mapiter_next(lua_State *L) { * pairs(map) */ static int lupb_map_pairs(lua_State *L) { - lupb_map_check(L, 1); size_t *iter = lua_newuserdata(L, sizeof(*iter)); + lupb_map_check(L, 1); *iter = UPB_MAP_BEGIN; lua_pushvalue(L, 1); diff --git a/upb/bindings/stdc++/string.h b/upb/bindings/stdc++/string.h deleted file mode 100644 index c3465481a3..0000000000 --- a/upb/bindings/stdc++/string.h +++ /dev/null @@ -1,69 +0,0 @@ - -#ifndef UPB_STDCPP_H_ -#define UPB_STDCPP_H_ - -#include "upb/sink.h" - -#include "upb/port_def.inc" - -namespace upb { - -template -class FillStringHandler { - public: - static void SetHandler(upb_byteshandler* handler) { - upb_byteshandler_setstartstr(handler, &FillStringHandler::StartString, - NULL); - upb_byteshandler_setstring(handler, &FillStringHandler::StringBuf, NULL); - } - - private: - // TODO(haberman): add UpbBind/UpbMakeHandler support to BytesHandler so these - // can be prettier callbacks. - static void* StartString(void *c, const void *hd, size_t size) { - UPB_UNUSED(hd); - UPB_UNUSED(size); - - T* str = static_cast(c); - str->clear(); - return c; - } - - static size_t StringBuf(void* c, const void* hd, const char* buf, size_t n, - const upb_bufhandle* h) { - UPB_UNUSED(hd); - UPB_UNUSED(h); - - T* str = static_cast(c); - try { - str->append(buf, n); - return n; - } catch (const std::exception&) { - return 0; - } - } -}; - -class StringSink { - public: - template - explicit StringSink(T* target) { - // TODO(haberman): we need to avoid rebuilding a new handler every time, - // but with class globals disallowed for google3 C++ this is tricky. - upb_byteshandler_init(&handler_); - FillStringHandler::SetHandler(&handler_); - input_.Reset(&handler_, target); - } - - BytesSink input() { return input_; } - - private: - upb_byteshandler handler_; - BytesSink input_; -}; - -} // namespace upb - -#include "upb/port_undef.inc" - -#endif // UPB_STDCPP_H_ diff --git a/upb/json_decode.c b/upb/json_decode.c index 15d305bb31..3b2548c251 100644 --- a/upb/json_decode.c +++ b/upb/json_decode.c @@ -1096,6 +1096,7 @@ static void jsondec_duration(jsondec *d, upb_msg *msg, const upb_msgdef *m) { upb_strview str = jsondec_string(d); const char *ptr = str.data; const char *end = ptr + str.size; + const int64_t max = (uint64_t)3652500 * 86400; /* "3.000000001s", "3s", etc. */ ptr = jsondec_buftoint64(d, ptr, end, &seconds.int64_val); @@ -1105,7 +1106,7 @@ static void jsondec_duration(jsondec *d, upb_msg *msg, const upb_msgdef *m) { jsondec_err(d, "Malformed duration"); } - if (seconds.int64_val < -315576000000LL || seconds.int64_val > 315576000000LL) { + if (seconds.int64_val < -max || seconds.int64_val > max) { jsondec_err(d, "Duration out of range"); } diff --git a/upb/upb.h b/upb/upb.h index f5c440e0ee..85205a5a76 100644 --- a/upb/upb.h +++ b/upb/upb.h @@ -301,7 +301,7 @@ UPB_INLINE uint32_t _upb_be_swap32(uint32_t val) { return val; } else { return ((val & 0xff) << 24) | ((val & 0xff00) << 8) | - ((val & 0xff0000ULL) >> 8) | ((val & 0xff000000ULL) >> 24); + ((val & 0xff0000) >> 8) | ((val & 0xff000000) >> 24); } } @@ -309,11 +309,7 @@ UPB_INLINE uint64_t _upb_be_swap64(uint64_t val) { if (_upb_isle()) { return val; } else { - return ((val & 0xff) << 56) | ((val & 0xff00) << 40) | - ((val & 0xff0000) << 24) | ((val & 0xff000000) << 8) | - ((val & 0xff00000000ULL) >> 8) | ((val & 0xff0000000000ULL) >> 24) | - ((val & 0xff000000000000ULL) >> 40) | - ((val & 0xff00000000000000ULL) >> 56); + return ((uint64_t)_upb_be_swap32(val) << 32) | _upb_be_swap32(val >> 32); } } diff --git a/upbc/BUILD b/upbc/BUILD new file mode 100644 index 0000000000..23897c9ac4 --- /dev/null +++ b/upbc/BUILD @@ -0,0 +1,35 @@ +load( + "//bazel:build_defs.bzl", + "UPB_DEFAULT_CPPOPTS", +) + +licenses(["notice"]) + +cc_library( + name = "upbc_generator", + srcs = [ + "generator.cc", + "message_layout.cc", + "message_layout.h", + ], + hdrs = ["generator.h"], + copts = UPB_DEFAULT_CPPOPTS, + deps = [ + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/strings", + "@com_google_protobuf//:protobuf", + "@com_google_protobuf//:protoc_lib", + ], +) + +cc_binary( + name = "protoc-gen-upb", + srcs = ["main.cc"], + copts = UPB_DEFAULT_CPPOPTS, + visibility = ["//visibility:public"], + deps = [ + ":upbc_generator", + "@com_google_protobuf//:protoc_lib", + ], +)